Improved icon fetching with DriveCache disabled

cache-images.go

@@ -150,17 +150,6 @@ func cacheImage(imageURL, imageID string, imageType string) (string, bool, error
 		return "", false, err
 	}
 
-	if err != nil {
-		recordInvalidImageID(imageID)
-		return "", false, fmt.Errorf("failed to decode image: %v", err)
-	}
-
-	// This is not working
-	// // Ensure the cache directory exists
-	// if _, err := os.Stat(config.DriveCache.Path); os.IsNotExist(err) {
-	// 	os.Mkdir(config.DriveCache.Path, os.ModePerm)
-	// }
-
 	// Open the temp file for writing
 	outFile, err := os.Create(tempImagePath)
 	if err != nil {
@@ -194,7 +183,7 @@ func cacheImage(imageURL, imageID string, imageType string) (string, bool, error
 }
 
 func handleImageServe(w http.ResponseWriter, r *http.Request) {
-	// Extract the image ID and type from the URL
+	// Extract image ID and type from URL
 	imageName := filepath.Base(r.URL.Path)
 	idType := imageName
 
@@ -202,7 +191,6 @@ func handleImageServe(w http.ResponseWriter, r *http.Request) {
 
 	hasExtension := false
 	if strings.HasSuffix(idType, ".webp") {
-		// Cached image, remove extension
 		idType = strings.TrimSuffix(idType, ".webp")
 		hasExtension = true
 	}
@@ -216,79 +204,111 @@ func handleImageServe(w http.ResponseWriter, r *http.Request) {
 	imageType = parts[1]
 
 	filename := fmt.Sprintf("%s_%s.webp", imageID, imageType)
-	// Adjust to read from config.DriveCache.Path / images
 	cachedImagePath := filepath.Join(config.DriveCache.Path, "images", filename)
 
-	if hasExtension && (imageType == "thumb" || imageType == "icon") {
+	// --- 1. PENDING: Is image currently being cached? Return 202 Accepted ---
+	imageKey := fmt.Sprintf("%s_%s", imageID, imageType)
+	imageURLMapMu.RLock()
+	imageURL, exists := imageURLMap[imageKey]
+	imageURLMapMu.RUnlock()
+
+	if exists {
+		cachingImagesMu.Lock()
+		_, isCaching := cachingImages[imageURL]
+		cachingImagesMu.Unlock()
+		if isCaching {
+			// Image is still being processed
+			w.WriteHeader(http.StatusAccepted) // 202
+			return
+		}
+	}
+
+	// --- 2. INVALID: Is image known as invalid? Return fallback with 410 ---
+	invalidImageIDsMu.Lock()
+	_, isInvalid := invalidImageIDs[imageID]
+	invalidImageIDsMu.Unlock()
+	if isInvalid {
+		if imageType == "icon" {
+			serveGlobeImage(w)
+		} else {
+			serveMissingImage(w)
+		}
+		return
+	}
+
+	// --- 3. READY: Serve cached file if available ---
+	if hasExtension && (imageType == "thumb" || imageType == "icon" || imageType == "full") {
 		if _, err := os.Stat(cachedImagePath); err == nil {
-			// Update the modification time
 			_ = os.Chtimes(cachedImagePath, time.Now(), time.Now())
 			w.Header().Set("Content-Type", "image/webp")
 			w.Header().Set("Cache-Control", "public, max-age=31536000")
 			http.ServeFile(w, r, cachedImagePath)
 			return
-		} else {
-			if config.DriveCacheEnabled {
+		} else if config.DriveCacheEnabled {
+			// With cache enabled, if cache file is missing, return fallback (410)
 			if imageType == "icon" {
-					serveGlobeImage(w, r)
+				serveGlobeImage(w)
 			} else {
-					serveMissingImage(w, r)
+				serveMissingImage(w)
 			}
 			return
 		}
 	}
-	}
-
-	// For full images, proceed to proxy the image
-
-	// Image not cached or caching not enabled
-	imageKey := fmt.Sprintf("%s_%s", imageID, imageType)
-
-	imageURLMapMu.RLock()
-	imageURL, exists := imageURLMap[imageKey]
-	imageURLMapMu.RUnlock()
 
+	// --- 4. MISSING MAPPING: No source image URL known, fallback with 410 ---
 	if !exists {
-		// Cannot find original URL, serve missing image
-		serveMissingImage(w, r)
+		serveMissingImage(w)
 		return
 	}
 
-	// For thumbnails, if HardCacheEnabled is true, and image not cached, serve missing image
-	if imageType == "thumb" && config.DriveCacheEnabled {
-		// Thumbnail should be cached, but not found
-		serveMissingImage(w, r)
-		return
-	}
-
-	// For full images, proceed to proxy the image
-
-	// Fetch the image from the original URL
+	// --- 5. PROXY ICON: If not cached, and icon requested, proxy original ---
+	if imageType == "icon" && !hasExtension {
 		resp, err := http.Get(imageURL)
 		if err != nil {
-		printWarn("Error fetching image: %v", err)
 			recordInvalidImageID(imageID)
-		serveMissingImage(w, r)
+			serveGlobeImage(w)
 			return
 		}
 		defer resp.Body.Close()
 
-	// Check if the request was successful
-	if resp.StatusCode != http.StatusOK {
-		serveMissingImage(w, r)
-		return
-	}
-
-	// Set the Content-Type header to the type of the fetched image
 		contentType := resp.Header.Get("Content-Type")
 		if contentType != "" && strings.HasPrefix(contentType, "image/") {
 			w.Header().Set("Content-Type", contentType)
 		} else {
-		serveMissingImage(w, r)
+			serveGlobeImage(w)
+			return
+		}
+		_, _ = io.Copy(w, resp.Body)
 		return
 	}
 
-	// Write the image content to the response
+	// --- 6. PROXY THUMB (if cache disabled): With cache ON, must be on disk ---
+	if imageType == "thumb" && config.DriveCacheEnabled {
+		serveMissingImage(w)
+		return
+	}
+
+	// --- 7. PROXY FULL: For full images, proxy directly ---
+	resp, err := http.Get(imageURL)
+	if err != nil {
+		printWarn("Error fetching image: %v", err)
+		recordInvalidImageID(imageID)
+		serveMissingImage(w)
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		serveMissingImage(w)
+		return
+	}
+	contentType := resp.Header.Get("Content-Type")
+	if contentType != "" && strings.HasPrefix(contentType, "image/") {
+		w.Header().Set("Content-Type", contentType)
+	} else {
+		serveMissingImage(w)
+		return
+	}
 	if _, err := io.Copy(w, resp.Body); err != nil {
 		printWarn("Error writing image to response: %v", err)
 	}
@@ -536,20 +556,31 @@ func safeDecodeImage(contentType string, data []byte) (img image.Image, err erro
 }
 
 // Serve missing.svg
-func serveMissingImage(w http.ResponseWriter, r *http.Request) {
+func serveMissingImage(w http.ResponseWriter) {
 	missingImagePath := filepath.Join("static", "images", "missing.svg")
+
+	// Set error code FIRST
+	w.WriteHeader(http.StatusGone)
+
+	// Now read the file and write it manually, to avoid conflict with http.ServeFile
+	data, err := os.ReadFile(missingImagePath)
+	if err != nil {
+		http.Error(w, "globe.svg not found", http.StatusInternalServerError)
+		return
+	}
+
 	w.Header().Set("Content-Type", "image/svg+xml")
 	w.Header().Set("Cache-Control", "no-store, must-revalidate")
 	w.Header().Set("Pragma", "no-cache")
 	w.Header().Set("Expires", "0")
-	http.ServeFile(w, r, missingImagePath)
+	_, _ = w.Write(data)
 }
 
-func serveGlobeImage(w http.ResponseWriter, r *http.Request) {
+func serveGlobeImage(w http.ResponseWriter) {
 	globePath := filepath.Join("static", "images", "globe.svg")
 
 	// Set error code FIRST
-	w.WriteHeader(http.StatusNotFound)
+	w.WriteHeader(http.StatusGone)
 
 	// Now read the file and write it manually, to avoid conflict with http.ServeFile
 	data, err := os.ReadFile(globePath)

common.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	"crypto/rand"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"html/template"
@@ -110,15 +108,15 @@ func renderTemplate(w http.ResponseWriter, tmplName string, data map[string]inte
 	}
 }
 
-// Randoms string generator used for auth code
-func generateStrongRandomString(length int) string {
-	bytes := make([]byte, length)
-	_, err := rand.Read(bytes)
-	if err != nil {
-		printErr("Error generating random string: %v", err)
-	}
-	return base64.URLEncoding.EncodeToString(bytes)[:length]
-}
+// // Randoms string generator used for auth code
+// func generateStrongRandomString(length int) string {
+// 	bytes := make([]byte, length)
+// 	_, err := rand.Read(bytes)
+// 	if err != nil {
+// 		printErr("Error generating random string: %v", err)
+// 	}
+// 	return base64.URLEncoding.EncodeToString(bytes)[:length]
+// }
 
 // Checks if the URL already includes a protocol
 func hasProtocol(url string) bool {

favicon.go

@@ -52,9 +52,12 @@ type faviconDownloadRequest struct {
 
 func init() {
 	// Start 5 worker goroutines to process favicon downloads
+
+	if !config.DriveCacheEnabled {
 		for i := 0; i < 5; i++ {
 			go faviconDownloadWorker()
 		}
+	}
 }
 
 func faviconDownloadWorker() {
@@ -72,7 +75,7 @@ func faviconIDFromURL(rawURL string) string {
 
 // Resolves favicon URL using multiple methods
 func resolveFaviconURL(rawFavicon, pageURL string) (faviconURL, cacheID string) {
-	cacheID = faviconIDFromURL(pageURL)
+	// cacheID = faviconIDFromURL(pageURL)
 
 	// Handle data URLs first
 	if strings.HasPrefix(rawFavicon, "data:image") {
@@ -136,6 +139,7 @@ func findFaviconInHeaders(pageURL string) string {
 		Timeout: 3 * time.Second, // like 3 seconds for favicon should be enough
 		Transport: &http.Transport{
 			TLSClientConfig:   &tls.Config{InsecureSkipVerify: true},
+			DisableKeepAlives: true,
 		},
 	}
 
@@ -214,7 +218,7 @@ func checkURLExists(url string) bool {
 	}
 
 	// Add User-Agent
-	userAgent, err := GetUserAgent("Text-Search-Brave")
+	userAgent, err := GetUserAgent("Text-Search-Favicons")
 	if err != nil {
 		printWarn("Error getting User-Agent: %v", err)
 	}
@@ -321,10 +325,6 @@ func getFaviconProxyURL(rawFavicon, pageURL string) string {
 	filename := fmt.Sprintf("%s_icon.webp", cacheID)
 	cachedPath := filepath.Join(config.DriveCache.Path, "images", filename)
 
-	if _, err := os.Stat(cachedPath); err == nil {
-		return fmt.Sprintf("/image/%s_icon.webp", cacheID)
-	}
-
 	// Resolve URL
 	faviconURL, _ := resolveFaviconURL(rawFavicon, pageURL)
 	if faviconURL == "" {
@@ -333,6 +333,14 @@ func getFaviconProxyURL(rawFavicon, pageURL string) string {
 	}
 
 	// Check if already downloading
+	imageURLMapMu.Lock()
+	imageURLMap[fmt.Sprintf("%s_icon", cacheID)] = faviconURL
+	imageURLMapMu.Unlock()
+
+	if config.DriveCacheEnabled {
+		if _, err := os.Stat(cachedPath); err == nil {
+			return fmt.Sprintf("/image/%s_icon.webp", cacheID)
+		}
 		faviconCache.RLock()
 		downloading := faviconCache.m[cacheID]
 		faviconCache.RUnlock()
@@ -342,14 +350,16 @@ func getFaviconProxyURL(rawFavicon, pageURL string) string {
 			faviconCache.m[cacheID] = true
 			faviconCache.Unlock()
 
-		// Send to download queue instead of starting goroutine
 			faviconDownloadQueue <- faviconDownloadRequest{
 				faviconURL: faviconURL,
 				pageURL:    pageURL,
 				cacheID:    cacheID,
 			}
 		}
+		return fmt.Sprintf("/image/%s_icon.webp", cacheID)
+	}
 
+	// Always proxy if cache is off
 	return fmt.Sprintf("/image/%s_icon.webp", cacheID)
 }
 
@@ -451,7 +461,7 @@ func cacheFavicon(imageURL, imageID string) (string, bool, error) {
 		}
 
 		// Add User-Agent
-		userAgent, err := GetUserAgent("Text-Search-Brave")
+		userAgent, err := GetUserAgent("Text-Search-Favicons")
 		if err != nil {
 			printWarn("Error getting User-Agent: %v", err)
 		}

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/fyne-io/image v0.1.1
 	github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f
 	golang.org/x/net v0.33.0
+	golang.org/x/text v0.21.0
 )
 
 require (
@@ -64,6 +65,5 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.etcd.io/bbolt v1.3.11 // indirect
 	golang.org/x/sys v0.28.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
 	google.golang.org/protobuf v1.36.0 // indirect
 )

static/js/dynamicscrollingtext.js

@@ -34,19 +34,19 @@
     }
 
     // Handle image/favicon loading errors
-    function handleImageError(imgElement, retryCount = 8, retryDelay = 500) {
+    function handleImageError(imgElement, retryCount = 10, retryDelay = 200) {
         const isFavicon = !!imgElement.closest('.favicon-wrapper');
         const container = imgElement.closest(type === 'image' ? '.image' : '.result_item');
         const titleSelector = type === 'image' ? '.img_title' : '.result-url';
         const title = container?.querySelector(titleSelector);
         const fullURL = imgElement.getAttribute('data-full');
 
-        if (retryCount > 0 && !imgElement.dataset.checked404) {
-            imgElement.dataset.checked404 = '1'; // avoid infinite loop
+        if (retryCount > 0 && !imgElement.dataset.checked410) {
+            imgElement.dataset.checked410 = '1'; // avoid infinite loop
 
             fetch(fullURL, { method: 'HEAD' })
                 .then(res => {
-                    if (res.status === 404) {
+                    if (res.status === 410) {
                         fallbackToGlobe(imgElement);
                     } else {
                         setTimeout(() => {
@@ -61,8 +61,15 @@
         } else {
             fallbackToGlobe(imgElement);
         }
+    }
 
     function fallbackToGlobe(imgElement) {
+        const type = document.getElementById('template-data').getAttribute('data-type');
+        const isFavicon = !!imgElement.closest('.favicon-wrapper');
+        const container = imgElement.closest(type === 'image' ? '.image' : '.result_item');
+        const titleSelector = type === 'image' ? '.img_title' : '.result-url';
+        const title = container?.querySelector(titleSelector);
+
         imgElement.closest('.favicon-wrapper')?.classList.remove('loading');
         if (title) title.classList.remove('title-loading');
 
@@ -82,7 +89,6 @@
             container?.remove();
         }
     }
-    }
 
     // Shared configuration
     const statusCheckInterval = 500;
@@ -124,20 +130,82 @@
 
         addLoadingEffects(imgElement);
 
-        if (hardCacheEnabled) {
-            imgElement.src = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';
-            imgElement.onerror = () => handleImageError(imgElement, 3, 1000);
+        const tryLoadImage = (attempts = 25, delay = 300) => {
+            fetch(`/image_status?image_ids=${id}`)
+            .then(res => res.json())
+            .then(map => {
+                const url = map[id];
+                if (!url) {
+                    if (attempts > 0) {
+                        // Exponential backoff with jitter
+                        const nextDelay = Math.min(delay * 2 + Math.random() * 100, 5000);
+                        setTimeout(() => tryLoadImage(attempts - 1, nextDelay), nextDelay);
                     } else {
+                        fallbackToGlobe(imgElement);
+                    }
+                } else if (url.endsWith('globe.svg') || url.endsWith('missing.svg')) {
+                    fallbackToGlobe(imgElement);
+                } else {
+                    // Remove cache buster to leverage browser caching
+                    const newImg = imgElement.cloneNode();
+                    newImg.src = url;
+                    newImg.onload = () => removeLoadingEffects(newImg);
+                    // Add retry mechanism for final load
+                    newImg.onerror = () => handleImageError(newImg, 3, 500);
+                    imgElement.parentNode.replaceChild(newImg, imgElement);
+                }
+            })
+            .catch(() => {
+                if (attempts > 0) {
+                    const nextDelay = Math.min(delay * 2, 5000);
+                    setTimeout(() => tryLoadImage(attempts - 1, nextDelay), nextDelay);
+                } else {
+                    fallbackToGlobe(imgElement);
+                }
+            });
+        };
+
+        // Initial load attempt with retry handler
         imgElement.src = imgElement.getAttribute('data-full');
         imgElement.onload = () => removeLoadingEffects(imgElement);
-            imgElement.onerror = () => handleImageError(imgElement, 3, 1000);
+        // Start polling immediately instead of waiting for error
+        setTimeout(() => tryLoadImage(), 500); // Start polling after initial load attempt
+
+        // Store reference in tracking map
+        if (!mediaMap.has(id)) mediaMap.set(id, []);
+        mediaMap.get(id).push(imgElement);
     }
 
-        // Track it
-        if (!mediaMap.has(id)) {
-            mediaMap.set(id, []);
+    function pollFaviconUntilReady(imgElement, id, retries = 8, delay = 700) {
+        let attempts = 0;
+        function poll() {
+            fetch(`/image_status?image_ids=${id}`)
+                .then(res => res.json())
+                .then(map => {
+                    const url = map[id];
+                    if (url && !url.endsWith('globe.svg') && !url.endsWith('missing.svg')) {
+
+                        const newImg = imgElement.cloneNode();
+                        newImg.src = url + "?v=" + Date.now();
+                        newImg.onload = () => removeLoadingEffects(newImg);
+                        newImg.onerror = () => fallbackToGlobe(newImg);
+                        imgElement.parentNode.replaceChild(newImg, imgElement);
+                    } else if (attempts < retries) {
+                        attempts++;
+                        setTimeout(poll, delay);
+                    } else {
+                        fallbackToGlobe(imgElement);
                     }
-        mediaMap.get(id).push(imgElement);
+                }).catch(() => {
+                    if (attempts < retries) {
+                        attempts++;
+                        setTimeout(poll, delay);
+                    } else {
+                        fallbackToGlobe(imgElement);
+                    }
+                });
+        }
+        poll();
     }
 
     // Check status of all tracked media elements
@@ -182,9 +250,10 @@
                 }
             }
     
-            mediaMap.clear();
-            for (const [id, imgs] of stillPending) {
-                mediaMap.set(id, imgs);
+            for (const id of Array.from(mediaMap.keys())) {
+                if (!stillPending.has(id)) {
+                    mediaMap.delete(id);
+                }
             }
         };
     

templates/text.html

@@ -265,8 +265,8 @@
         <div class="result_item">
             <div class="result_header">
                 <div class="favicon-container">
-                    <img src="/static/images/placeholder.svg" data-id="{{ .FaviconID }}"
-                        data-full="/image/{{ .FaviconID }}_icon.webp" alt="🌐" class="favicon placeholder-img" />
+                    <img src="/static/images/globe.svg" data-id="{{ .FaviconID }}"
+                        data-full="/image/{{ .FaviconID }}_icon" alt="🌐" class="favicon placeholder-img" />
                 </div>
                 <div class="result-url single-line-ellipsis">
                     {{ .PrettyLink.Domain }}