Improved icon fetching with DriveCache disabled

cache-images.go

@@ -150,17 +150,6 @@ func cacheImage(imageURL, imageID string, imageType string) (string, bool, error
 		return "", false, err
 	}
 
-	if err != nil {
-		recordInvalidImageID(imageID)
-		return "", false, fmt.Errorf("failed to decode image: %v", err)
-	}
-
-	// This is not working
-	// // Ensure the cache directory exists
-	// if _, err := os.Stat(config.DriveCache.Path); os.IsNotExist(err) {
-	// 	os.Mkdir(config.DriveCache.Path, os.ModePerm)
-	// }
-
 	// Open the temp file for writing
 	outFile, err := os.Create(tempImagePath)
 	if err != nil {
@@ -194,7 +183,7 @@ func cacheImage(imageURL, imageID string, imageType string) (string, bool, error
 }
 
 func handleImageServe(w http.ResponseWriter, r *http.Request) {
-	// Extract the image ID and type from the URL
+	// Extract image ID and type from URL
 	imageName := filepath.Base(r.URL.Path)
 	idType := imageName
 
@@ -202,7 +191,6 @@ func handleImageServe(w http.ResponseWriter, r *http.Request) {
 
 	hasExtension := false
 	if strings.HasSuffix(idType, ".webp") {
-		// Cached image, remove extension
 		idType = strings.TrimSuffix(idType, ".webp")
 		hasExtension = true
 	}
@@ -216,79 +204,111 @@ func handleImageServe(w http.ResponseWriter, r *http.Request) {
 	imageType = parts[1]
 
 	filename := fmt.Sprintf("%s_%s.webp", imageID, imageType)
-	// Adjust to read from config.DriveCache.Path / images
 	cachedImagePath := filepath.Join(config.DriveCache.Path, "images", filename)
 
-	if hasExtension && (imageType == "thumb" || imageType == "icon") {
+	// --- 1. PENDING: Is image currently being cached? Return 202 Accepted ---
+	imageKey := fmt.Sprintf("%s_%s", imageID, imageType)
+	imageURLMapMu.RLock()
+	imageURL, exists := imageURLMap[imageKey]
+	imageURLMapMu.RUnlock()
+
+	if exists {
+		cachingImagesMu.Lock()
+		_, isCaching := cachingImages[imageURL]
+		cachingImagesMu.Unlock()
+		if isCaching {
+			// Image is still being processed
+			w.WriteHeader(http.StatusAccepted) // 202
+			return
+		}
+	}
+
+	// --- 2. INVALID: Is image known as invalid? Return fallback with 410 ---
+	invalidImageIDsMu.Lock()
+	_, isInvalid := invalidImageIDs[imageID]
+	invalidImageIDsMu.Unlock()
+	if isInvalid {
+		if imageType == "icon" {
+			serveGlobeImage(w)
+		} else {
+			serveMissingImage(w)
+		}
+		return
+	}
+
+	// --- 3. READY: Serve cached file if available ---
+	if hasExtension && (imageType == "thumb" || imageType == "icon" || imageType == "full") {
 		if _, err := os.Stat(cachedImagePath); err == nil {
-			// Update the modification time
 			_ = os.Chtimes(cachedImagePath, time.Now(), time.Now())
 			w.Header().Set("Content-Type", "image/webp")
 			w.Header().Set("Cache-Control", "public, max-age=31536000")
 			http.ServeFile(w, r, cachedImagePath)
 			return
-		} else {
-			if config.DriveCacheEnabled {
-				if imageType == "icon" {
-					serveGlobeImage(w, r)
-				} else {
-					serveMissingImage(w, r)
-				}
-				return
+		} else if config.DriveCacheEnabled {
+			// With cache enabled, if cache file is missing, return fallback (410)
+			if imageType == "icon" {
+				serveGlobeImage(w)
+			} else {
+				serveMissingImage(w)
 			}
+			return
 		}
 	}
 
-	// For full images, proceed to proxy the image
-
-	// Image not cached or caching not enabled
-	imageKey := fmt.Sprintf("%s_%s", imageID, imageType)
-
-	imageURLMapMu.RLock()
-	imageURL, exists := imageURLMap[imageKey]
-	imageURLMapMu.RUnlock()
-
+	// --- 4. MISSING MAPPING: No source image URL known, fallback with 410 ---
 	if !exists {
-		// Cannot find original URL, serve missing image
-		serveMissingImage(w, r)
+		serveMissingImage(w)
 		return
 	}
 
-	// For thumbnails, if HardCacheEnabled is true, and image not cached, serve missing image
+	// --- 5. PROXY ICON: If not cached, and icon requested, proxy original ---
+	if imageType == "icon" && !hasExtension {
+		resp, err := http.Get(imageURL)
+		if err != nil {
+			recordInvalidImageID(imageID)
+			serveGlobeImage(w)
+			return
+		}
+		defer resp.Body.Close()
+
+		contentType := resp.Header.Get("Content-Type")
+		if contentType != "" && strings.HasPrefix(contentType, "image/") {
+			w.Header().Set("Content-Type", contentType)
+		} else {
+			serveGlobeImage(w)
+			return
+		}
+		_, _ = io.Copy(w, resp.Body)
+		return
+	}
+
+	// --- 6. PROXY THUMB (if cache disabled): With cache ON, must be on disk ---
 	if imageType == "thumb" && config.DriveCacheEnabled {
-		// Thumbnail should be cached, but not found
-		serveMissingImage(w, r)
+		serveMissingImage(w)
 		return
 	}
 
-	// For full images, proceed to proxy the image
-
-	// Fetch the image from the original URL
+	// --- 7. PROXY FULL: For full images, proxy directly ---
 	resp, err := http.Get(imageURL)
 	if err != nil {
 		printWarn("Error fetching image: %v", err)
 		recordInvalidImageID(imageID)
-		serveMissingImage(w, r)
+		serveMissingImage(w)
 		return
 	}
 	defer resp.Body.Close()
 
-	// Check if the request was successful
 	if resp.StatusCode != http.StatusOK {
-		serveMissingImage(w, r)
+		serveMissingImage(w)
 		return
 	}
-
-	// Set the Content-Type header to the type of the fetched image
 	contentType := resp.Header.Get("Content-Type")
 	if contentType != "" && strings.HasPrefix(contentType, "image/") {
 		w.Header().Set("Content-Type", contentType)
 	} else {
-		serveMissingImage(w, r)
+		serveMissingImage(w)
 		return
 	}
-
-	// Write the image content to the response
 	if _, err := io.Copy(w, resp.Body); err != nil {
 		printWarn("Error writing image to response: %v", err)
 	}
@@ -536,20 +556,31 @@ func safeDecodeImage(contentType string, data []byte) (img image.Image, err erro
 }
 
 // Serve missing.svg
-func serveMissingImage(w http.ResponseWriter, r *http.Request) {
+func serveMissingImage(w http.ResponseWriter) {
 	missingImagePath := filepath.Join("static", "images", "missing.svg")
+
+	// Set error code FIRST
+	w.WriteHeader(http.StatusGone)
+
+	// Now read the file and write it manually, to avoid conflict with http.ServeFile
+	data, err := os.ReadFile(missingImagePath)
+	if err != nil {
+		http.Error(w, "globe.svg not found", http.StatusInternalServerError)
+		return
+	}
+
 	w.Header().Set("Content-Type", "image/svg+xml")
 	w.Header().Set("Cache-Control", "no-store, must-revalidate")
 	w.Header().Set("Pragma", "no-cache")
 	w.Header().Set("Expires", "0")
-	http.ServeFile(w, r, missingImagePath)
+	_, _ = w.Write(data)
 }
 
-func serveGlobeImage(w http.ResponseWriter, r *http.Request) {
+func serveGlobeImage(w http.ResponseWriter) {
 	globePath := filepath.Join("static", "images", "globe.svg")
 
 	// Set error code FIRST
-	w.WriteHeader(http.StatusNotFound)
+	w.WriteHeader(http.StatusGone)
 
 	// Now read the file and write it manually, to avoid conflict with http.ServeFile
 	data, err := os.ReadFile(globePath)

common.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	"crypto/rand"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"html/template"
@@ -110,15 +108,15 @@ func renderTemplate(w http.ResponseWriter, tmplName string, data map[string]inte
 	}
 }
 
-// Randoms string generator used for auth code
-func generateStrongRandomString(length int) string {
-	bytes := make([]byte, length)
-	_, err := rand.Read(bytes)
-	if err != nil {
-		printErr("Error generating random string: %v", err)
-	}
-	return base64.URLEncoding.EncodeToString(bytes)[:length]
-}
+// // Randoms string generator used for auth code
+// func generateStrongRandomString(length int) string {
+// 	bytes := make([]byte, length)
+// 	_, err := rand.Read(bytes)
+// 	if err != nil {
+// 		printErr("Error generating random string: %v", err)
+// 	}
+// 	return base64.URLEncoding.EncodeToString(bytes)[:length]
+// }
 
 // Checks if the URL already includes a protocol
 func hasProtocol(url string) bool {

favicon.go

@@ -52,8 +52,11 @@ type faviconDownloadRequest struct {
 
 func init() {
 	// Start 5 worker goroutines to process favicon downloads
-	for i := 0; i < 5; i++ {
-		go faviconDownloadWorker()
+
+	if !config.DriveCacheEnabled {
+		for i := 0; i < 5; i++ {
+			go faviconDownloadWorker()
+		}
 	}
 }
 
@@ -72,7 +75,7 @@ func faviconIDFromURL(rawURL string) string {
 
 // Resolves favicon URL using multiple methods
 func resolveFaviconURL(rawFavicon, pageURL string) (faviconURL, cacheID string) {
-	cacheID = faviconIDFromURL(pageURL)
+	// cacheID = faviconIDFromURL(pageURL)
 
 	// Handle data URLs first
 	if strings.HasPrefix(rawFavicon, "data:image") {
@@ -135,7 +138,8 @@ func findFaviconInHeaders(pageURL string) string {
 	client := &http.Client{
 		Timeout: 3 * time.Second, // like 3 seconds for favicon should be enough
 		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			TLSClientConfig:   &tls.Config{InsecureSkipVerify: true},
+			DisableKeepAlives: true,
 		},
 	}
 
@@ -214,7 +218,7 @@ func checkURLExists(url string) bool {
 	}
 
 	// Add User-Agent
-	userAgent, err := GetUserAgent("Text-Search-Brave")
+	userAgent, err := GetUserAgent("Text-Search-Favicons")
 	if err != nil {
 		printWarn("Error getting User-Agent: %v", err)
 	}
@@ -321,10 +325,6 @@ func getFaviconProxyURL(rawFavicon, pageURL string) string {
 	filename := fmt.Sprintf("%s_icon.webp", cacheID)
 	cachedPath := filepath.Join(config.DriveCache.Path, "images", filename)
 
-	if _, err := os.Stat(cachedPath); err == nil {
-		return fmt.Sprintf("/image/%s_icon.webp", cacheID)
-	}
-
 	// Resolve URL
 	faviconURL, _ := resolveFaviconURL(rawFavicon, pageURL)
 	if faviconURL == "" {
@@ -333,23 +333,33 @@ func getFaviconProxyURL(rawFavicon, pageURL string) string {
 	}
 
 	// Check if already downloading
-	faviconCache.RLock()
-	downloading := faviconCache.m[cacheID]
-	faviconCache.RUnlock()
+	imageURLMapMu.Lock()
+	imageURLMap[fmt.Sprintf("%s_icon", cacheID)] = faviconURL
+	imageURLMapMu.Unlock()
 
-	if !downloading {
-		faviconCache.Lock()
-		faviconCache.m[cacheID] = true
-		faviconCache.Unlock()
-
-		// Send to download queue instead of starting goroutine
-		faviconDownloadQueue <- faviconDownloadRequest{
-			faviconURL: faviconURL,
-			pageURL:    pageURL,
-			cacheID:    cacheID,
+	if config.DriveCacheEnabled {
+		if _, err := os.Stat(cachedPath); err == nil {
+			return fmt.Sprintf("/image/%s_icon.webp", cacheID)
 		}
+		faviconCache.RLock()
+		downloading := faviconCache.m[cacheID]
+		faviconCache.RUnlock()
+
+		if !downloading {
+			faviconCache.Lock()
+			faviconCache.m[cacheID] = true
+			faviconCache.Unlock()
+
+			faviconDownloadQueue <- faviconDownloadRequest{
+				faviconURL: faviconURL,
+				pageURL:    pageURL,
+				cacheID:    cacheID,
+			}
+		}
+		return fmt.Sprintf("/image/%s_icon.webp", cacheID)
 	}
 
+	// Always proxy if cache is off
 	return fmt.Sprintf("/image/%s_icon.webp", cacheID)
 }
 
@@ -451,7 +461,7 @@ func cacheFavicon(imageURL, imageID string) (string, bool, error) {
 		}
 
 		// Add User-Agent
-		userAgent, err := GetUserAgent("Text-Search-Brave")
+		userAgent, err := GetUserAgent("Text-Search-Favicons")
 		if err != nil {
 			printWarn("Error getting User-Agent: %v", err)
 		}

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/fyne-io/image v0.1.1
 	github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f
 	golang.org/x/net v0.33.0
+	golang.org/x/text v0.21.0
 )
 
 require (
@@ -64,6 +65,5 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.etcd.io/bbolt v1.3.11 // indirect
 	golang.org/x/sys v0.28.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
 	google.golang.org/protobuf v1.36.0 // indirect
 )

static/js/dynamicscrollingtext.js

@@ -34,19 +34,19 @@
     }
 
     // Handle image/favicon loading errors
-    function handleImageError(imgElement, retryCount = 8, retryDelay = 500) {
+    function handleImageError(imgElement, retryCount = 10, retryDelay = 200) {
         const isFavicon = !!imgElement.closest('.favicon-wrapper');
         const container = imgElement.closest(type === 'image' ? '.image' : '.result_item');
         const titleSelector = type === 'image' ? '.img_title' : '.result-url';
         const title = container?.querySelector(titleSelector);
         const fullURL = imgElement.getAttribute('data-full');
 
-        if (retryCount > 0 && !imgElement.dataset.checked404) {
-            imgElement.dataset.checked404 = '1'; // avoid infinite loop
+        if (retryCount > 0 && !imgElement.dataset.checked410) {
+            imgElement.dataset.checked410 = '1'; // avoid infinite loop
 
             fetch(fullURL, { method: 'HEAD' })
                 .then(res => {
-                    if (res.status === 404) {
+                    if (res.status === 410) {
                         fallbackToGlobe(imgElement);
                     } else {
                         setTimeout(() => {
@@ -61,26 +61,32 @@
         } else {
             fallbackToGlobe(imgElement);
         }
+    }
 
-        function fallbackToGlobe(imgElement) {
-            imgElement.closest('.favicon-wrapper')?.classList.remove('loading');
-            if (title) title.classList.remove('title-loading');
+    function fallbackToGlobe(imgElement) {
+        const type = document.getElementById('template-data').getAttribute('data-type');
+        const isFavicon = !!imgElement.closest('.favicon-wrapper');
+        const container = imgElement.closest(type === 'image' ? '.image' : '.result_item');
+        const titleSelector = type === 'image' ? '.img_title' : '.result-url';
+        const title = container?.querySelector(titleSelector);
 
-            if (isFavicon) {
-                const wrapper = imgElement.closest('.favicon-wrapper') || imgElement.parentElement;
-                const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
-                svg.setAttribute("xmlns", "http://www.w3.org/2000/svg");
-                svg.setAttribute("viewBox", "0 -960 960 960");
-                svg.setAttribute("height", imgElement.height || "16");
-                svg.setAttribute("width", imgElement.width || "16");
-                svg.setAttribute("fill", "currentColor");
-                svg.classList.add("favicon", "globe-fallback");
+        imgElement.closest('.favicon-wrapper')?.classList.remove('loading');
+        if (title) title.classList.remove('title-loading');
+
+        if (isFavicon) {
+            const wrapper = imgElement.closest('.favicon-wrapper') || imgElement.parentElement;
+            const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
+            svg.setAttribute("xmlns", "http://www.w3.org/2000/svg");
+            svg.setAttribute("viewBox", "0 -960 960 960");
+            svg.setAttribute("height", imgElement.height || "16");
+            svg.setAttribute("width", imgElement.width || "16");
+            svg.setAttribute("fill", "currentColor");
+            svg.classList.add("favicon", "globe-fallback");
                 svg.innerHTML = `<path d="M480-80q-82 0-155-31.5t-127.5-86Q143-252 111.5-325T80-480q0-83 31.5-155.5t86-127Q252-817 325-848.5T480-880q83 0 155.5 31.5t127 86q54.5 54.5 86 127T880-480q0 82-31.5 155t-86 127.5q-54.5 54.5-127 86T480-80Zm0-82q26-36 45-75t31-83H404q12 44 31 83t45 75Zm-104-16q-18-33-31.5-68.5T322-320H204q29 50 72.5 87t99.5 55Zm208 0q56-18 99.5-55t72.5-87H638q-9 38-22.5 73.5T584-178ZM170-400h136q-3-20-4.5-39.5T300-480q0-21 1.5-40.5T306-560H170q-5 20-7.5 39.5T160-480q0 21 2.5 40.5T170-400Zm216 0h188q3-20 4.5-39.5T580-480q0-21-1.5-40.5T574-560H386q-3 20-4.5 39.5T380-480q0 21 1.5 40.5T386-400Zm268 0h136q5-20 7.5-39.5T800-480q0-21-2.5-40.5T790-560H654q3 20 4.5 39.5T660-480q0 21-1.5 40.5T654-400Zm-16-240h118q-29-50-72.5-87T584-782q18 33 31.5 68.5T638-640Zm-234 0h152q-12-44-31-83t-45-75q-26 36-45 75t-31 83Zm-200 0h118q9-38 22.5-73.5T376-782q-56 18-99.5 55T204-640Z"/>`;
-                imgElement.remove();
-                wrapper.appendChild(svg);
-            } else if (type === 'image') {
-                container?.remove();
-            }
+            imgElement.remove();
+            wrapper.appendChild(svg);
+        } else if (type === 'image') {
+            container?.remove();
         }
     }
 
@@ -124,22 +130,84 @@
 
         addLoadingEffects(imgElement);
 
-        if (hardCacheEnabled) {
-            imgElement.src = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';
-            imgElement.onerror = () => handleImageError(imgElement, 3, 1000);
-        } else {
-            imgElement.src = imgElement.getAttribute('data-full');
-            imgElement.onload = () => removeLoadingEffects(imgElement);
-            imgElement.onerror = () => handleImageError(imgElement, 3, 1000);
-        }
+        const tryLoadImage = (attempts = 25, delay = 300) => {
+            fetch(`/image_status?image_ids=${id}`)
+            .then(res => res.json())
+            .then(map => {
+                const url = map[id];
+                if (!url) {
+                    if (attempts > 0) {
+                        // Exponential backoff with jitter
+                        const nextDelay = Math.min(delay * 2 + Math.random() * 100, 5000);
+                        setTimeout(() => tryLoadImage(attempts - 1, nextDelay), nextDelay);
+                    } else {
+                        fallbackToGlobe(imgElement);
+                    }
+                } else if (url.endsWith('globe.svg') || url.endsWith('missing.svg')) {
+                    fallbackToGlobe(imgElement);
+                } else {
+                    // Remove cache buster to leverage browser caching
+                    const newImg = imgElement.cloneNode();
+                    newImg.src = url;
+                    newImg.onload = () => removeLoadingEffects(newImg);
+                    // Add retry mechanism for final load
+                    newImg.onerror = () => handleImageError(newImg, 3, 500);
+                    imgElement.parentNode.replaceChild(newImg, imgElement);
+                }
+            })
+            .catch(() => {
+                if (attempts > 0) {
+                    const nextDelay = Math.min(delay * 2, 5000);
+                    setTimeout(() => tryLoadImage(attempts - 1, nextDelay), nextDelay);
+                } else {
+                    fallbackToGlobe(imgElement);
+                }
+            });
+        };
 
-        // Track it
-        if (!mediaMap.has(id)) {
-            mediaMap.set(id, []);
-        }
+        // Initial load attempt with retry handler
+        imgElement.src = imgElement.getAttribute('data-full');
+        imgElement.onload = () => removeLoadingEffects(imgElement);
+        // Start polling immediately instead of waiting for error
+        setTimeout(() => tryLoadImage(), 500); // Start polling after initial load attempt
+
+        // Store reference in tracking map
+        if (!mediaMap.has(id)) mediaMap.set(id, []);
         mediaMap.get(id).push(imgElement);
     }
 
+    function pollFaviconUntilReady(imgElement, id, retries = 8, delay = 700) {
+        let attempts = 0;
+        function poll() {
+            fetch(`/image_status?image_ids=${id}`)
+                .then(res => res.json())
+                .then(map => {
+                    const url = map[id];
+                    if (url && !url.endsWith('globe.svg') && !url.endsWith('missing.svg')) {
+
+                        const newImg = imgElement.cloneNode();
+                        newImg.src = url + "?v=" + Date.now();
+                        newImg.onload = () => removeLoadingEffects(newImg);
+                        newImg.onerror = () => fallbackToGlobe(newImg);
+                        imgElement.parentNode.replaceChild(newImg, imgElement);
+                    } else if (attempts < retries) {
+                        attempts++;
+                        setTimeout(poll, delay);
+                    } else {
+                        fallbackToGlobe(imgElement);
+                    }
+                }).catch(() => {
+                    if (attempts < retries) {
+                        attempts++;
+                        setTimeout(poll, delay);
+                    } else {
+                        fallbackToGlobe(imgElement);
+                    }
+                });
+        }
+        poll();
+    }
+
     // Check status of all tracked media elements
     function checkMediaStatus() {
         const allIds = Array.from(mediaMap.keys());
@@ -182,9 +250,10 @@
                 }
             }
     
-            mediaMap.clear();
-            for (const [id, imgs] of stillPending) {
-                mediaMap.set(id, imgs);
+            for (const id of Array.from(mediaMap.keys())) {
+                if (!stillPending.has(id)) {
+                    mediaMap.delete(id);
+                }
             }
         };
     

templates/text.html

@@ -265,8 +265,8 @@
         <div class="result_item">
             <div class="result_header">
                 <div class="favicon-container">
-                    <img src="/static/images/placeholder.svg" data-id="{{ .FaviconID }}"
-                        data-full="/image/{{ .FaviconID }}_icon.webp" alt="🌐" class="favicon placeholder-img" />
+                    <img src="/static/images/globe.svg" data-id="{{ .FaviconID }}"
+                        data-full="/image/{{ .FaviconID }}_icon" alt="🌐" class="favicon placeholder-img" />
                 </div>
                 <div class="result-url single-line-ellipsis">
                     {{ .PrettyLink.Domain }}