added: serving missing.svg on error instead of an invalid image URL

cache-images.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"image"
@@ -14,6 +15,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/chai2010/webp"
 	"golang.org/x/image/bmp"
@@ -24,15 +26,18 @@ var (
 	cachingImages    = make(map[string]*sync.Mutex)
 	cachingImagesMu  sync.Mutex
 	cachingSemaphore = make(chan struct{}, 10) // Limit to 10 concurrent downloads
+
+	invalidImageIDs   = make(map[string]struct{})
+	invalidImageIDsMu sync.Mutex
 )
 
-func cacheImage(imageURL, filename string) (string, error) {
+func cacheImage(imageURL, filename, imageID string) (string, bool, error) {
 	cacheDir := "image_cache"
 	cachedImagePath := filepath.Join(cacheDir, filename)
 
 	// Check if the image is already cached
 	if _, err := os.Stat(cachedImagePath); err == nil {
-		return cachedImagePath, nil
+		return cachedImagePath, true, nil
 	}
 
 	// Ensure only one goroutine caches the same image
@@ -48,31 +53,40 @@ func cacheImage(imageURL, filename string) (string, error) {
 
 	// Double-check if the image was cached while waiting
 	if _, err := os.Stat(cachedImagePath); err == nil {
-		return cachedImagePath, nil
+		return cachedImagePath, true, nil
 	}
 
 	cachingSemaphore <- struct{}{}        // Acquire a token
 	defer func() { <-cachingSemaphore }() // Release the token
 
-	// Download the image
+	// Create a custom http.Client that skips SSL certificate verification
-	resp, err := http.Get(imageURL)
+	client := &http.Client{
+		Timeout: 15 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+
+	// Download the image using the custom client
+	resp, err := client.Get(imageURL)
 	if err != nil {
-		return "", err
+		recordInvalidImageID(imageID)
+		return "", false, err
 	}
 	defer resp.Body.Close()
 
 	// Read the image data into a byte slice
 	data, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return "", err
+		recordInvalidImageID(imageID)
+		return "", false, err
 	}
 
-	// Detect the content type
+	// Check if the response is actually an image
 	contentType := http.DetectContentType(data)
-
+	if !strings.HasPrefix(contentType, "image/") {
-	// If content type is HTML, skip caching
+		recordInvalidImageID(imageID)
-	if strings.HasPrefix(contentType, "text/html") {
+		return "", false, fmt.Errorf("URL did not return an image: %s", imageURL)
-		return "", fmt.Errorf("URL returned HTML content instead of an image: %s", imageURL)
 	}
 
 	// Handle SVG files directly
@@ -85,7 +99,8 @@ func cacheImage(imageURL, filename string) (string, error) {
 		// Save the SVG file as-is
 		err = os.WriteFile(cachedImagePath, data, 0644)
 		if err != nil {
-			return "", err
+			recordInvalidImageID(imageID)
+			return "", false, err
 		}
 
 		// Clean up mutex
@@ -93,7 +108,7 @@ func cacheImage(imageURL, filename string) (string, error) {
 		delete(cachingImages, imageURL)
 		cachingImagesMu.Unlock()
 
-		return cachedImagePath, nil
+		return cachedImagePath, true, nil
 	}
 
 	// Decode the image based on the content type
@@ -112,11 +127,13 @@ func cacheImage(imageURL, filename string) (string, error) {
 	case "image/tiff":
 		img, err = tiff.Decode(bytes.NewReader(data))
 	default:
-		return "", fmt.Errorf("unsupported image type: %s", contentType)
+		recordInvalidImageID(imageID)
+		return "", false, fmt.Errorf("unsupported image type: %s", contentType)
 	}
 
 	if err != nil {
-		return "", fmt.Errorf("failed to decode image: %v", err)
+		recordInvalidImageID(imageID)
+		return "", false, fmt.Errorf("failed to decode image: %v", err)
 	}
 
 	// Ensure the cache directory exists
@@ -127,7 +144,8 @@ func cacheImage(imageURL, filename string) (string, error) {
 	// Open the cached file for writing
 	outFile, err := os.Create(cachedImagePath)
 	if err != nil {
-		return "", err
+		recordInvalidImageID(imageID)
+		return "", false, err
 	}
 	defer outFile.Close()
 
@@ -135,7 +153,8 @@ func cacheImage(imageURL, filename string) (string, error) {
 	options := &webp.Options{Lossless: false, Quality: 80}
 	err = webp.Encode(outFile, img, options)
 	if err != nil {
-		return "", err
+		recordInvalidImageID(imageID)
+		return "", false, err
 	}
 
 	// Clean up mutex
@@ -143,7 +162,7 @@ func cacheImage(imageURL, filename string) (string, error) {
 	delete(cachingImages, imageURL)
 	cachingImagesMu.Unlock()
 
-	return cachedImagePath, nil
+	return cachedImagePath, true, nil
 }
 
 func handleCachedImages(w http.ResponseWriter, r *http.Request) {
@@ -152,21 +171,15 @@ func handleCachedImages(w http.ResponseWriter, r *http.Request) {
 	cachedImagePath := filepath.Join(cacheDir, imageName)
 
 	if _, err := os.Stat(cachedImagePath); os.IsNotExist(err) {
-		// Serve placeholder image with no-store headers
+		printDebug("Cached image not found: %s, serving missing.svg", cachedImagePath)
-		placeholderPath := "static/images/placeholder.webp"
+		// Serve missing image
-		placeholderContentType := "image/webp"
+		missingImagePath := filepath.Join("static", "images", "missing.svg")
-
+		w.Header().Set("Content-Type", "image/svg+xml")
-		// You can also check for SVG placeholder if needed
+		http.ServeFile(w, r, missingImagePath)
-		if strings.HasSuffix(imageName, ".svg") {
+		return
-			placeholderPath = "static/images/placeholder.svg"
+	} else if err != nil {
-			placeholderContentType = "image/svg+xml"
+		printWarn("Error checking image file: %v", err)
-		}
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-
-		w.Header().Set("Content-Type", placeholderContentType)
-		w.Header().Set("Cache-Control", "no-store, must-revalidate")
-		w.Header().Set("Pragma", "no-cache")
-		w.Header().Set("Expires", "0")
-		http.ServeFile(w, r, placeholderPath)
 		return
 	}
 
@@ -199,13 +212,21 @@ func handleImageStatus(w http.ResponseWriter, r *http.Request) {
 	ids := strings.Split(imageIDs, ",")
 
 	statusMap := make(map[string]string)
-
 	cacheDir := "image_cache"
 
 	printDebug("Received image status request for IDs: %v", ids)
-	printDebug("Status map: %v", statusMap)
+
+	invalidImageIDsMu.Lock()
+	defer invalidImageIDsMu.Unlock()
 
 	for _, id := range ids {
+		// Check if the image ID is in the invalidImageIDs map
+		if _, invalid := invalidImageIDs[id]; invalid {
+			// Image is invalid, set status to "missing"
+			statusMap[id] = "/static/images/missing.svg"
+			continue
+		}
+
 		// Check for different possible extensions
 		extensions := []string{".webp", ".svg"}
 		var cachedImagePath string
@@ -224,11 +245,80 @@ func handleImageStatus(w http.ResponseWriter, r *http.Request) {
 		if found {
 			statusMap[id] = cachedImagePath
 		} else {
-			// Image is not ready
+			// Image is not ready yet
 			statusMap[id] = ""
 		}
 	}
 
+	printDebug("Status map: %v", statusMap)
+
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(statusMap)
 }
+
+func recordInvalidImageID(imageID string) {
+	invalidImageIDsMu.Lock()
+	defer invalidImageIDsMu.Unlock()
+	invalidImageIDs[imageID] = struct{}{}
+	printDebug("Recorded invalid image ID: %s", imageID)
+}
+
+func filterValidImages(imageResults []ImageSearchResult) []ImageSearchResult {
+	invalidImageIDsMu.Lock()
+	defer invalidImageIDsMu.Unlock()
+
+	var filteredResults []ImageSearchResult
+	for _, img := range imageResults {
+		if _, invalid := invalidImageIDs[img.ID]; !invalid {
+			filteredResults = append(filteredResults, img)
+		} else {
+			printDebug("Filtering out invalid image ID: %s", img.ID)
+		}
+	}
+	return filteredResults
+}
+
+func removeImageResultFromCache(query string, page int, safe bool, lang string, imageID string) {
+	cacheKey := CacheKey{
+		Query: query,
+		Page:  page,
+		Safe:  safe,
+		Lang:  lang,
+		Type:  "image",
+	}
+
+	rc := resultsCache
+
+	rc.mu.Lock()
+	defer rc.mu.Unlock()
+
+	keyStr := rc.keyToString(cacheKey)
+	item, exists := rc.results[keyStr]
+	if !exists {
+		return
+	}
+
+	// Filter out the image with the given ID
+	var newResults []SearchResult
+	for _, r := range item.Results {
+		if imgResult, ok := r.(ImageSearchResult); ok {
+			if imgResult.ID != imageID {
+				newResults = append(newResults, r)
+			} else {
+				printDebug("Removing invalid image ID from cache: %s", imageID)
+			}
+		} else {
+			newResults = append(newResults, r)
+		}
+	}
+
+	// Update or delete the cache entry
+	if len(newResults) > 0 {
+		rc.results[keyStr] = CachedItem{
+			Results:    newResults,
+			StoredTime: item.StoredTime,
+		}
+	} else {
+		delete(rc.results, keyStr)
+	}
+}

imageproxy.go

@@ -3,6 +3,8 @@ package main
 import (
 	"io"
 	"net/http"
+	"path/filepath"
+	"strings"
 )
 
 func handleImageProxy(w http.ResponseWriter, r *http.Request) {
@@ -17,29 +19,41 @@ func handleImageProxy(w http.ResponseWriter, r *http.Request) {
 	resp, err := http.Get(imageURL)
 	if err != nil {
 		printWarn("Error fetching image: %v", err)
-		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		serveMissingImage(w, r)
 		return
 	}
 	defer resp.Body.Close()
 
 	// Check if the request was successful
 	if resp.StatusCode != http.StatusOK {
-		http.Error(w, "Failed to fetch image", http.StatusBadGateway)
+		serveMissingImage(w, r)
 		return
 	}
 
 	// Set the Content-Type header to the type of the fetched image
 	contentType := resp.Header.Get("Content-Type")
-	if contentType != "" {
+	if contentType != "" && strings.HasPrefix(contentType, "image/") {
 		w.Header().Set("Content-Type", contentType)
 	} else {
-		// Default to octet-stream if Content-Type is not available
+		serveMissingImage(w, r)
-		w.Header().Set("Content-Type", "application/octet-stream")
+		return
 	}
 
 	// Write the image content to the response
 	if _, err := io.Copy(w, resp.Body); err != nil {
 		printWarn("Error writing image to response: %v", err)
-		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		// Serve missing.svg
+		// Note: At this point, headers are already sent, so serving missing.svg won't work.
+		// It's better to just log the error here.
 	}
 }
+
+// Serve missing.svg
+func serveMissingImage(w http.ResponseWriter, r *http.Request) {
+	missingImagePath := filepath.Join("static", "images", "missing.svg")
+	w.Header().Set("Content-Type", "image/svg+xml")
+	w.Header().Set("Cache-Control", "no-store, must-revalidate")
+	w.Header().Set("Pragma", "no-cache")
+	w.Header().Set("Expires", "0")
+	http.ServeFile(w, r, missingImagePath)
+}

images-bing.go

@@ -68,8 +68,8 @@ func PerformBingImageSearch(query, safe, lang string, page int) ([]ImageSearchRe
 			mediaURL, ok := data["murl"].(string)
 			if ok {
 				// Apply the image proxy
-				proxiedFullURL := "/imgproxy?url=" + imgSrc
+				proxiedFullURL := "/imgproxy?url=" + mediaURL
-				proxiedThumbURL := "/imgproxy?url=" + mediaURL
+				proxiedThumbURL := "/imgproxy?url=" + imgSrc
 				results = append(results, ImageSearchResult{
 					Thumb:      imgSrc,
 					Title:      strings.TrimSpace(title),

images-deviantart.go

@@ -149,7 +149,7 @@ func PerformDeviantArtImageSearch(query, safe, lang string, page int) ([]ImageSe
 			go func(imgSrc, resultURL, title string) {
 				defer wg.Done()
 				// Verify if the image URL is accessible
-				if isValidImageURL(imgSrc, DeviantArtImageUserAgent, resultURL) {
+				if DeviantArtisValidImageURL(imgSrc, DeviantArtImageUserAgent, resultURL) {
 					resultsChan <- ImageSearchResult{
 						Title:      strings.TrimSpace(title),
 						Full:       imgSrc,
@@ -201,7 +201,7 @@ func buildDeviantArtSearchURL(query string, page int) string {
 }
 
 // isValidImageURL checks if the image URL is accessible with the provided User-Agent
-func isValidImageURL(imgSrc, userAgent, referer string) bool {
+func DeviantArtisValidImageURL(imgSrc, userAgent, referer string) bool {
 	client := &http.Client{}
 	req, err := http.NewRequest("HEAD", imgSrc, nil)
 	if err != nil {

images.go

@@ -68,16 +68,18 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
 		if results == nil {
 			combinedResults = fetchImageResults(query, safe, lang, page)
 			if len(combinedResults) > 0 {
+				combinedResults = filterValidImages(combinedResults)
 				resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
 			}
 		} else {
 			_, _, imageResults := convertToSpecificResults(results)
-			combinedResults = imageResults
+			combinedResults = filterValidImages(imageResults)
 		}
 	case <-time.After(2 * time.Second):
 		printInfo("Cache check timeout")
 		combinedResults = fetchImageResults(query, safe, lang, page)
 		if len(combinedResults) > 0 {
+			combinedResults = filterValidImages(combinedResults)
 			resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
 		}
 	}
@@ -87,6 +89,7 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
 
 func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
 	var results []ImageSearchResult
+	safeBool := safe == "active"
 
 	for _, engine := range imageSearchEngines {
 		printInfo("Using image search engine: %s", engine.Name)
@@ -107,20 +110,28 @@ func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
 				hash := hex.EncodeToString(hasher.Sum(nil))
 				filename := hash + ".webp"
 
-				// Set the Full URL to point to the cached image path
-				cacheURL := "/image_cache/" + filename
-				imageResult.ProxyFull = cacheURL
-
 				// Assign the ID
 				imageResult.ID = hash
 
-				// Start caching in the background
+				// Set the ProxyFull URL
-				go func(originalURL, filename string) {
+				imageResult.ProxyFull = "/image_cache/" + filename
-					_, err := cacheImage(originalURL, filename)
+
+				// Start caching and validation in the background
+				go func(imgResult ImageSearchResult, originalURL, filename string) {
+					_, success, err := cacheImage(originalURL, filename, imgResult.ID)
 					if err != nil {
 						printWarn("Failed to cache image %s: %v", originalURL, err)
 					}
-				}(imageResult.Full, filename)
+					if !success {
+						// Remove the image result from the cache
+						removeImageResultFromCache(query, page, safeBool, lang, imgResult.ID)
+					}
+				}(imageResult, imageResult.Full, filename)
+
+			} else {
+				// When hard cache is not enabled, use the imgproxy URLs
+				imageResult.ProxyThumb = "/imgproxy?url=" + imageResult.Thumb // Proxied thumbnail
+				imageResult.ProxyFull = "/imgproxy?url=" + imageResult.Full   // Proxied full-size image
 			}
 			results = append(results, imageResult)
 		}
@@ -151,3 +162,82 @@ func wrapImageSearchFunc(f func(string, string, string, int) ([]ImageSearchResul
 		return searchResults, duration, nil
 	}
 }
+
+// func isValidImageURL(imageURL string) bool {
+// 	client := &http.Client{
+// 		Timeout: 10 * time.Second,
+// 		Transport: &http.Transport{
+// 			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+// 		},
+// 	}
+
+// 	req, err := http.NewRequest("GET", imageURL, nil)
+// 	if err != nil {
+// 		return false
+// 	}
+
+// 	// Set headers to mimic a real browser
+// 	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "+
+// 		"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36")
+// 	req.Header.Set("Accept", "image/webp,image/*,*/*;q=0.8")
+// 	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+// 	req.Header.Set("Referer", imageURL) // Some servers require a referer
+
+// 	resp, err := client.Do(req)
+// 	if err != nil {
+// 		return false
+// 	}
+// 	defer resp.Body.Close()
+
+// 	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
+// 		return false
+// 	}
+
+// 	// Limit the amount of data read to 10KB
+// 	limitedReader := io.LimitReader(resp.Body, 10240) // 10KB
+
+// 	// Attempt to decode image configuration
+// 	_, _, err = image.DecodeConfig(limitedReader)
+// 	if err != nil {
+// 		return false
+// 	}
+
+// 	return true
+// }
+
+// // This function can be used alternatively to isValidImageURL(), Its slower but reliable
+// func isImageAccessible(imageURL string) bool {
+// 	client := &http.Client{
+// 		Timeout: 5 * time.Second,
+// 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+// 			if len(via) >= 10 {
+// 				return http.ErrUseLastResponse
+// 			}
+// 			return nil
+// 		},
+// 	}
+
+// 	resp, err := client.Get(imageURL)
+// 	if err != nil {
+// 		return false
+// 	}
+// 	defer resp.Body.Close()
+
+// 	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
+// 		return false
+// 	}
+
+// 	// Read the entire image data
+// 	data, err := io.ReadAll(resp.Body)
+// 	if err != nil {
+// 		return false
+// 	}
+
+// 	// Try to decode the image
+// 	_, _, err = image.Decode(bytes.NewReader(data))
+// 	if err != nil {
+// 		return false
+// 	}
+
+// 	return true
+// }

static/images/missing.svg

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg width="800px" height="800px" viewBox="0 0 24 24" id="image" data-name="multi color" class="icon multi-color" xmlns="http://www.w3.org/2000/svg">
+  <polygon id="tertiary-fill" points="3.29 19.71 9 14 11 16 14 13 20.71 19.71 3.29 19.71" style="stroke-width: 2; fill: rgb(235, 235, 235); fill-opacity: 0.5;"/>
+  <path id="primary-stroke" d="M20,20H4a1,1,0,0,1-1-1V5A1,1,0,0,1,4,4H20a1,1,0,0,1,1,1V19A1,1,0,0,1,20,20Zm.71-.29L14,13l-3,3L9,14,3.29,19.71Z" style="fill: none; stroke-linecap: round; stroke-linejoin: round; stroke-width: 2; stroke: rgb(15, 15, 15); stroke-opacity: 0.5;"/>
+  <circle id="secondary-fill" cx="11" cy="9" r="1" style="fill: rgb(44, 169, 188); stroke-linecap: round; stroke-linejoin: round; stroke-width: 2; fill-opacity: 0; stroke: rgb(245, 142, 23); stroke-opacity: 0.5;"/>
+  <path style="fill: rgb(216, 216, 216); stroke-linecap: round; stroke: rgb(204, 0, 0);" d="M 8.918 9.102 L 14.942 15.125"/>
+  <path style="fill: rgb(216, 216, 216); stroke-linecap: round; stroke: rgb(204, 0, 0); transform-origin: 11.959px 12.143px;" d="M 8.85 15.254 L 15.07 9.033" transform="matrix(-1, 0, 0, -1, 0.000003, 0.000001)"/>
+</svg>

static/images/placeholder.svg

@@ -1,2 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
-<svg width="800px" height="800px" viewBox="0 0 24 24" id="image" data-name="multi color" xmlns="http://www.w3.org/2000/svg" class="icon multi-color"><polygon id="tertiary-fill" points="3.29 19.71 9 14 11 16 14 13 20.71 19.71 3.29 19.71" style="fill: #b7b7b7; stroke-width: 2;"></polygon><path id="primary-stroke" d="M20,20H4a1,1,0,0,1-1-1V5A1,1,0,0,1,4,4H20a1,1,0,0,1,1,1V19A1,1,0,0,1,20,20Zm.71-.29L14,13l-3,3L9,14,3.29,19.71Z" style="fill: none; stroke: rgb(0, 0, 0); stroke-linecap: round; stroke-linejoin: round; stroke-width: 2;"></path><circle id="secondary-fill" cx="11" cy="9" r="1" style="fill: rgb(44, 169, 188); stroke: rgb(246, 146, 30); stroke-linecap: round; stroke-linejoin: round; stroke-width: 2;"></circle></svg>
+<svg width="800px" height="800px" viewBox="0 0 24 24" id="image" data-name="multi color" class="icon multi-color" xmlns="http://www.w3.org/2000/svg">
+  <polygon id="tertiary-fill" points="3.29 19.71 9 14 11 16 14 13 20.71 19.71 3.29 19.71" style="stroke-width: 2; fill: rgb(235, 235, 235);"/>
+  <path id="primary-stroke" d="M20,20H4a1,1,0,0,1-1-1V5A1,1,0,0,1,4,4H20a1,1,0,0,1,1,1V19A1,1,0,0,1,20,20Zm.71-.29L14,13l-3,3L9,14,3.29,19.71Z" style="fill: none; stroke-linecap: round; stroke-linejoin: round; stroke-width: 2; stroke: rgb(15, 15, 15);"/>
+  <circle id="secondary-fill" cx="11" cy="9" r="1" style="fill: rgb(44, 169, 188); stroke-linecap: round; stroke-linejoin: round; stroke-width: 2; fill-opacity: 0; stroke: rgb(245, 142, 23);"/>
+</svg>