Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues 8 Pull requests Projects Releases Packages Wiki Activity Actions

updated 'config.ini'

Browse source
This commit is contained in:
partisan 2024-11-26 07:46:03 +01:00
parent 28f71271d7
commit be4f86580e
13 changed files with 635 additions and 208 deletions
Download patch file Download diff file Expand all files Collapse all files

127
cache-images.go
View file

@ -13,6 +13,7 @@ import (
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
@ -23,9 +24,9 @@ import (
)
var (
cachingImages = make(map[string]*sync.Mutex)
cachingImagesMu sync.Mutex
cachingSemaphore = make(chan struct{}, 100) // Limit to concurrent downloads
cachingImages = make(map[string]*sync.Mutex)
cachingImagesMu sync.Mutex
// cachingSemaphore = make(chan struct{}, 100) // Limit to concurrent downloads
invalidImageIDs = make(map[string]struct{})
invalidImageIDsMu sync.Mutex
@ -35,8 +36,6 @@ var (
)
func cacheImage(imageURL, imageID string, isThumbnail bool) (string, bool, error) {
cacheDir := "image_cache"
if imageURL == "" {
recordInvalidImageID(imageID)
return "", false, fmt.Errorf("empty image URL for image ID %s", imageID)
@ -50,7 +49,7 @@ func cacheImage(imageURL, imageID string, isThumbnail bool) (string, bool, error
filename = fmt.Sprintf("%s_full.webp", imageID)
}
cachedImagePath := filepath.Join(cacheDir, filename)
cachedImagePath := filepath.Join(config.DriveCache.Path, filename)
tempImagePath := cachedImagePath + ".tmp"
// Check if the image is already cached
@ -74,8 +73,9 @@ func cacheImage(imageURL, imageID string, isThumbnail bool) (string, bool, error
return cachedImagePath, true, nil
}
cachingSemaphore <- struct{}{} // Acquire a token
defer func() { <-cachingSemaphore }() // Release the token
// // Limit max concurrent downloads
// cachingSemaphore <- struct{}{} // Acquire a token
// defer func() { <-cachingSemaphore }() // Release the token
// Create a custom http.Client that skips SSL certificate verification
client := &http.Client{
@ -109,11 +109,6 @@ func cacheImage(imageURL, imageID string, isThumbnail bool) (string, bool, error
// Handle SVG files directly
if contentType == "image/svg+xml" {
// Ensure the cache directory exists
if _, err := os.Stat(cacheDir); os.IsNotExist(err) {
os.Mkdir(cacheDir, os.ModePerm)
}
// Save the SVG file as-is to the temp path
err = os.WriteFile(tempImagePath, data, 0644)
if err != nil {
@ -161,10 +156,11 @@ func cacheImage(imageURL, imageID string, isThumbnail bool) (string, bool, error
return "", false, fmt.Errorf("failed to decode image: %v", err)
}
// Ensure the cache directory exists
if _, err := os.Stat(cacheDir); os.IsNotExist(err) {
os.Mkdir(cacheDir, os.ModePerm)
}
// This is not working
// // Ensure the cache directory exists
// if _, err := os.Stat(config.DriveCache.Path); os.IsNotExist(err) {
// os.Mkdir(config.DriveCache.Path, os.ModePerm)
// }
// Open the temp file for writing
outFile, err := os.Create(tempImagePath)
@ -220,14 +216,19 @@ func handleImageServe(w http.ResponseWriter, r *http.Request) {
imageID = parts[0]
imageType = parts[1]
cacheDir := "image_cache"
filename := fmt.Sprintf("%s_%s.webp", imageID, imageType)
cachedImagePath := filepath.Join(cacheDir, filename)
cachedImagePath := filepath.Join(config.DriveCache.Path, filename)
if hasExtension && imageType == "thumb" {
// Requesting cached thumbnail image
// Requesting cached image (thumbnail or full)
if _, err := os.Stat(cachedImagePath); err == nil {
// Cached image exists, serve it
// Update the modification time to now
err := os.Chtimes(cachedImagePath, time.Now(), time.Now())
if err != nil {
printWarn("Failed to update modification time for %s: %v", cachedImagePath, err)
}
// Determine content type based on file extension
contentType := "image/webp"
w.Header().Set("Content-Type", contentType)
w.Header().Set("Cache-Control", "public, max-age=31536000")
@ -235,12 +236,12 @@ func handleImageServe(w http.ResponseWriter, r *http.Request) {
return
} else {
// Cached image not found
if config.HardCacheEnabled {
if config.DriveCacheEnabled {
// Thumbnail should be cached, but not found
serveMissingImage(w, r)
return
}
// Else, proceed to proxy (if HardCacheEnabled is false)
// Else, proceed to proxy if caching is disabled
}
}
@ -260,7 +261,7 @@ func handleImageServe(w http.ResponseWriter, r *http.Request) {
}
// For thumbnails, if HardCacheEnabled is true, and image not cached, serve missing image
if imageType == "thumb" && config.HardCacheEnabled {
if imageType == "thumb" && config.DriveCacheEnabled {
// Thumbnail should be cached, but not found
serveMissingImage(w, r)
return
@ -311,14 +312,13 @@ func handleImageStatus(w http.ResponseWriter, r *http.Request) {
}
// Check for cached full or thumbnail images
cacheDir := "image_cache"
extensions := []string{"webp", "svg"} // Extensions without leading dots
imageReady := false
// Check thumbnail first
for _, ext := range extensions {
thumbFilename := fmt.Sprintf("%s_thumb.%s", id, ext)
thumbPath := filepath.Join(cacheDir, thumbFilename)
thumbPath := filepath.Join(config.DriveCache.Path, thumbFilename)
if _, err := os.Stat(thumbPath); err == nil {
statusMap[id] = fmt.Sprintf("/image/%s_thumb.%s", id, ext)
@ -331,7 +331,7 @@ func handleImageStatus(w http.ResponseWriter, r *http.Request) {
if !imageReady {
for _, ext := range extensions {
fullFilename := fmt.Sprintf("%s_full.%s", id, ext)
fullPath := filepath.Join(cacheDir, fullFilename)
fullPath := filepath.Join(config.DriveCache.Path, fullFilename)
if _, err := os.Stat(fullPath); err == nil {
statusMap[id] = fmt.Sprintf("/image/%s_full.%s", id, ext)
@ -343,7 +343,7 @@ func handleImageStatus(w http.ResponseWriter, r *http.Request) {
// If neither is ready
if !imageReady {
if !config.HardCacheEnabled {
if !config.DriveCacheEnabled {
// Hard cache is disabled; use the proxy URL
statusMap[id] = fmt.Sprintf("/image/%s_thumb", id)
} else {
@ -424,6 +424,77 @@ func removeImageResultFromCache(query string, page int, safe bool, lang string,
}
}
func cleanExpiredCachedImages() {
if config.DriveCache.Duration <= 0 && config.DriveCache.MaxUsageBytes <= 0 {
return // No cleanup needed if both duration and max usage are disabled
}
ticker := time.NewTicker(1 * time.Hour)
defer ticker.Stop()
for range ticker.C {
cleanupCache()
}
}
func cleanupCache() {
files, err := os.ReadDir(config.DriveCache.Path)
if err != nil {
printErr("Failed to read DriveCache directory: %v", err)
return
}
var totalSize uint64
fileInfos := make([]os.FileInfo, 0, len(files))
for _, file := range files {
info, err := file.Info()
if err != nil {
continue
}
filePath := filepath.Join(config.DriveCache.Path, file.Name())
// Check for expired files based on modification time
if config.DriveCache.Duration > 0 && time.Since(info.ModTime()) > config.DriveCache.Duration {
if err := os.Remove(filePath); err == nil {
printDebug("Removed expired cache file: %s", filePath)
} else {
printErr("Failed to remove expired cache file: %s", filePath)
}
continue // Skip adding this file to the list
}
// Accumulate total size and store file info for potential deletion
totalSize += uint64(info.Size())
fileInfos = append(fileInfos, info)
}
// If total size exceeds MaxUsageBytes, delete least recently used files
if config.DriveCache.MaxUsageBytes > 0 && totalSize > config.DriveCache.MaxUsageBytes {
// Sort files by last access time (oldest first)
sort.Slice(fileInfos, func(i, j int) bool {
return fileInfos[i].ModTime().Before(fileInfos[j].ModTime())
})
for _, info := range fileInfos {
if totalSize <= config.DriveCache.MaxUsageBytes {
break
}
filePath := filepath.Join(config.DriveCache.Path, info.Name())
fileSize := uint64(info.Size())
if err := os.Remove(filePath); err == nil {
totalSize -= fileSize
printDebug("Removed cache file to reduce size: %s", filePath)
} else {
printErr("Failed to remove cache file: %s", filePath)
}
}
}
}
func getContentType(ext string) string {
switch strings.ToLower(ext) {
case "svg":