added: serving missing.svg on error instead of an invalid image URL
This commit is contained in:
partisan
parent
1721db85a7
commit
787816d0ab
7 changed files with 263 additions and 57 deletions
164
cache-images.go
164
cache-images.go
|
|
@ -2,6 +2,7 @@ package main
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"image"
|
||||
|
|
@ -14,6 +15,7 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/chai2010/webp"
|
||||
"golang.org/x/image/bmp"
|
||||
|
|
@ -24,15 +26,18 @@ var (
|
|||
cachingImages = make(map[string]*sync.Mutex)
|
||||
cachingImagesMu sync.Mutex
|
||||
cachingSemaphore = make(chan struct{}, 10) // Limit to 10 concurrent downloads
|
||||
|
||||
invalidImageIDs = make(map[string]struct{})
|
||||
invalidImageIDsMu sync.Mutex
|
||||
)
|
||||
|
||||
func cacheImage(imageURL, filename string) (string, error) {
|
||||
func cacheImage(imageURL, filename, imageID string) (string, bool, error) {
|
||||
cacheDir := "image_cache"
|
||||
cachedImagePath := filepath.Join(cacheDir, filename)
|
||||
|
||||
// Check if the image is already cached
|
||||
if _, err := os.Stat(cachedImagePath); err == nil {
|
||||
return cachedImagePath, nil
|
||||
return cachedImagePath, true, nil
|
||||
}
|
||||
|
||||
// Ensure only one goroutine caches the same image
|
||||
|
|
@ -48,31 +53,40 @@ func cacheImage(imageURL, filename string) (string, error) {
|
|||
|
||||
// Double-check if the image was cached while waiting
|
||||
if _, err := os.Stat(cachedImagePath); err == nil {
|
||||
return cachedImagePath, nil
|
||||
return cachedImagePath, true, nil
|
||||
}
|
||||
|
||||
cachingSemaphore <- struct{}{} // Acquire a token
|
||||
defer func() { <-cachingSemaphore }() // Release the token
|
||||
|
||||
// Download the image
|
||||
resp, err := http.Get(imageURL)
|
||||
// Create a custom http.Client that skips SSL certificate verification
|
||||
client := &http.Client{
|
||||
Timeout: 15 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
},
|
||||
}
|
||||
|
||||
// Download the image using the custom client
|
||||
resp, err := client.Get(imageURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
recordInvalidImageID(imageID)
|
||||
return "", false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read the image data into a byte slice
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
recordInvalidImageID(imageID)
|
||||
return "", false, err
|
||||
}
|
||||
|
||||
// Detect the content type
|
||||
// Check if the response is actually an image
|
||||
contentType := http.DetectContentType(data)
|
||||
|
||||
// If content type is HTML, skip caching
|
||||
if strings.HasPrefix(contentType, "text/html") {
|
||||
return "", fmt.Errorf("URL returned HTML content instead of an image: %s", imageURL)
|
||||
if !strings.HasPrefix(contentType, "image/") {
|
||||
recordInvalidImageID(imageID)
|
||||
return "", false, fmt.Errorf("URL did not return an image: %s", imageURL)
|
||||
}
|
||||
|
||||
// Handle SVG files directly
|
||||
|
|
@ -85,7 +99,8 @@ func cacheImage(imageURL, filename string) (string, error) {
|
|||
// Save the SVG file as-is
|
||||
err = os.WriteFile(cachedImagePath, data, 0644)
|
||||
if err != nil {
|
||||
return "", err
|
||||
recordInvalidImageID(imageID)
|
||||
return "", false, err
|
||||
}
|
||||
|
||||
// Clean up mutex
|
||||
|
|
@ -93,7 +108,7 @@ func cacheImage(imageURL, filename string) (string, error) {
|
|||
delete(cachingImages, imageURL)
|
||||
cachingImagesMu.Unlock()
|
||||
|
||||
return cachedImagePath, nil
|
||||
return cachedImagePath, true, nil
|
||||
}
|
||||
|
||||
// Decode the image based on the content type
|
||||
|
|
@ -112,11 +127,13 @@ func cacheImage(imageURL, filename string) (string, error) {
|
|||
case "image/tiff":
|
||||
img, err = tiff.Decode(bytes.NewReader(data))
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported image type: %s", contentType)
|
||||
recordInvalidImageID(imageID)
|
||||
return "", false, fmt.Errorf("unsupported image type: %s", contentType)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to decode image: %v", err)
|
||||
recordInvalidImageID(imageID)
|
||||
return "", false, fmt.Errorf("failed to decode image: %v", err)
|
||||
}
|
||||
|
||||
// Ensure the cache directory exists
|
||||
|
|
@ -127,7 +144,8 @@ func cacheImage(imageURL, filename string) (string, error) {
|
|||
// Open the cached file for writing
|
||||
outFile, err := os.Create(cachedImagePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
recordInvalidImageID(imageID)
|
||||
return "", false, err
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
|
|
@ -135,7 +153,8 @@ func cacheImage(imageURL, filename string) (string, error) {
|
|||
options := &webp.Options{Lossless: false, Quality: 80}
|
||||
err = webp.Encode(outFile, img, options)
|
||||
if err != nil {
|
||||
return "", err
|
||||
recordInvalidImageID(imageID)
|
||||
return "", false, err
|
||||
}
|
||||
|
||||
// Clean up mutex
|
||||
|
|
@ -143,7 +162,7 @@ func cacheImage(imageURL, filename string) (string, error) {
|
|||
delete(cachingImages, imageURL)
|
||||
cachingImagesMu.Unlock()
|
||||
|
||||
return cachedImagePath, nil
|
||||
return cachedImagePath, true, nil
|
||||
}
|
||||
|
||||
func handleCachedImages(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
@ -152,21 +171,15 @@ func handleCachedImages(w http.ResponseWriter, r *http.Request) {
|
|||
cachedImagePath := filepath.Join(cacheDir, imageName)
|
||||
|
||||
if _, err := os.Stat(cachedImagePath); os.IsNotExist(err) {
|
||||
// Serve placeholder image with no-store headers
|
||||
placeholderPath := "static/images/placeholder.webp"
|
||||
placeholderContentType := "image/webp"
|
||||
|
||||
// You can also check for SVG placeholder if needed
|
||||
if strings.HasSuffix(imageName, ".svg") {
|
||||
placeholderPath = "static/images/placeholder.svg"
|
||||
placeholderContentType = "image/svg+xml"
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", placeholderContentType)
|
||||
w.Header().Set("Cache-Control", "no-store, must-revalidate")
|
||||
w.Header().Set("Pragma", "no-cache")
|
||||
w.Header().Set("Expires", "0")
|
||||
http.ServeFile(w, r, placeholderPath)
|
||||
printDebug("Cached image not found: %s, serving missing.svg", cachedImagePath)
|
||||
// Serve missing image
|
||||
missingImagePath := filepath.Join("static", "images", "missing.svg")
|
||||
w.Header().Set("Content-Type", "image/svg+xml")
|
||||
http.ServeFile(w, r, missingImagePath)
|
||||
return
|
||||
} else if err != nil {
|
||||
printWarn("Error checking image file: %v", err)
|
||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -199,13 +212,21 @@ func handleImageStatus(w http.ResponseWriter, r *http.Request) {
|
|||
ids := strings.Split(imageIDs, ",")
|
||||
|
||||
statusMap := make(map[string]string)
|
||||
|
||||
cacheDir := "image_cache"
|
||||
|
||||
printDebug("Received image status request for IDs: %v", ids)
|
||||
printDebug("Status map: %v", statusMap)
|
||||
|
||||
invalidImageIDsMu.Lock()
|
||||
defer invalidImageIDsMu.Unlock()
|
||||
|
||||
for _, id := range ids {
|
||||
// Check if the image ID is in the invalidImageIDs map
|
||||
if _, invalid := invalidImageIDs[id]; invalid {
|
||||
// Image is invalid, set status to "missing"
|
||||
statusMap[id] = "/static/images/missing.svg"
|
||||
continue
|
||||
}
|
||||
|
||||
// Check for different possible extensions
|
||||
extensions := []string{".webp", ".svg"}
|
||||
var cachedImagePath string
|
||||
|
|
@ -224,11 +245,80 @@ func handleImageStatus(w http.ResponseWriter, r *http.Request) {
|
|||
if found {
|
||||
statusMap[id] = cachedImagePath
|
||||
} else {
|
||||
// Image is not ready
|
||||
// Image is not ready yet
|
||||
statusMap[id] = ""
|
||||
}
|
||||
}
|
||||
|
||||
printDebug("Status map: %v", statusMap)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(statusMap)
|
||||
}
|
||||
|
||||
func recordInvalidImageID(imageID string) {
|
||||
invalidImageIDsMu.Lock()
|
||||
defer invalidImageIDsMu.Unlock()
|
||||
invalidImageIDs[imageID] = struct{}{}
|
||||
printDebug("Recorded invalid image ID: %s", imageID)
|
||||
}
|
||||
|
||||
func filterValidImages(imageResults []ImageSearchResult) []ImageSearchResult {
|
||||
invalidImageIDsMu.Lock()
|
||||
defer invalidImageIDsMu.Unlock()
|
||||
|
||||
var filteredResults []ImageSearchResult
|
||||
for _, img := range imageResults {
|
||||
if _, invalid := invalidImageIDs[img.ID]; !invalid {
|
||||
filteredResults = append(filteredResults, img)
|
||||
} else {
|
||||
printDebug("Filtering out invalid image ID: %s", img.ID)
|
||||
}
|
||||
}
|
||||
return filteredResults
|
||||
}
|
||||
|
||||
func removeImageResultFromCache(query string, page int, safe bool, lang string, imageID string) {
|
||||
cacheKey := CacheKey{
|
||||
Query: query,
|
||||
Page: page,
|
||||
Safe: safe,
|
||||
Lang: lang,
|
||||
Type: "image",
|
||||
}
|
||||
|
||||
rc := resultsCache
|
||||
|
||||
rc.mu.Lock()
|
||||
defer rc.mu.Unlock()
|
||||
|
||||
keyStr := rc.keyToString(cacheKey)
|
||||
item, exists := rc.results[keyStr]
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
// Filter out the image with the given ID
|
||||
var newResults []SearchResult
|
||||
for _, r := range item.Results {
|
||||
if imgResult, ok := r.(ImageSearchResult); ok {
|
||||
if imgResult.ID != imageID {
|
||||
newResults = append(newResults, r)
|
||||
} else {
|
||||
printDebug("Removing invalid image ID from cache: %s", imageID)
|
||||
}
|
||||
} else {
|
||||
newResults = append(newResults, r)
|
||||
}
|
||||
}
|
||||
|
||||
// Update or delete the cache entry
|
||||
if len(newResults) > 0 {
|
||||
rc.results[keyStr] = CachedItem{
|
||||
Results: newResults,
|
||||
StoredTime: item.StoredTime,
|
||||
}
|
||||
} else {
|
||||
delete(rc.results, keyStr)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue