diff --git a/cache-images.go b/cache-images.go
index 4e75b58..398907c 100644
--- a/cache-images.go
+++ b/cache-images.go
@@ -150,17 +150,6 @@ func cacheImage(imageURL, imageID string, imageType string) (string, bool, error
return "", false, err
}
- if err != nil {
- recordInvalidImageID(imageID)
- return "", false, fmt.Errorf("failed to decode image: %v", err)
- }
-
- // This is not working
- // // Ensure the cache directory exists
- // if _, err := os.Stat(config.DriveCache.Path); os.IsNotExist(err) {
- // os.Mkdir(config.DriveCache.Path, os.ModePerm)
- // }
-
// Open the temp file for writing
outFile, err := os.Create(tempImagePath)
if err != nil {
@@ -194,7 +183,7 @@ func cacheImage(imageURL, imageID string, imageType string) (string, bool, error
}
func handleImageServe(w http.ResponseWriter, r *http.Request) {
- // Extract the image ID and type from the URL
+ // Extract image ID and type from URL
imageName := filepath.Base(r.URL.Path)
idType := imageName
@@ -202,7 +191,6 @@ func handleImageServe(w http.ResponseWriter, r *http.Request) {
hasExtension := false
if strings.HasSuffix(idType, ".webp") {
- // Cached image, remove extension
idType = strings.TrimSuffix(idType, ".webp")
hasExtension = true
}
@@ -216,79 +204,111 @@ func handleImageServe(w http.ResponseWriter, r *http.Request) {
imageType = parts[1]
filename := fmt.Sprintf("%s_%s.webp", imageID, imageType)
- // Adjust to read from config.DriveCache.Path / images
cachedImagePath := filepath.Join(config.DriveCache.Path, "images", filename)
- if hasExtension && (imageType == "thumb" || imageType == "icon") {
+ // --- 1. PENDING: Is image currently being cached? Return 202 Accepted ---
+ imageKey := fmt.Sprintf("%s_%s", imageID, imageType)
+ imageURLMapMu.RLock()
+ imageURL, exists := imageURLMap[imageKey]
+ imageURLMapMu.RUnlock()
+
+ if exists {
+ cachingImagesMu.Lock()
+ _, isCaching := cachingImages[imageURL]
+ cachingImagesMu.Unlock()
+ if isCaching {
+ // Image is still being processed
+ w.WriteHeader(http.StatusAccepted) // 202
+ return
+ }
+ }
+
+ // --- 2. INVALID: Is image known as invalid? Return fallback with 410 ---
+ invalidImageIDsMu.Lock()
+ _, isInvalid := invalidImageIDs[imageID]
+ invalidImageIDsMu.Unlock()
+ if isInvalid {
+ if imageType == "icon" {
+ serveGlobeImage(w)
+ } else {
+ serveMissingImage(w)
+ }
+ return
+ }
+
+ // --- 3. READY: Serve cached file if available ---
+ if hasExtension && (imageType == "thumb" || imageType == "icon" || imageType == "full") {
if _, err := os.Stat(cachedImagePath); err == nil {
- // Update the modification time
_ = os.Chtimes(cachedImagePath, time.Now(), time.Now())
w.Header().Set("Content-Type", "image/webp")
w.Header().Set("Cache-Control", "public, max-age=31536000")
http.ServeFile(w, r, cachedImagePath)
return
- } else {
- if config.DriveCacheEnabled {
- if imageType == "icon" {
- serveGlobeImage(w, r)
- } else {
- serveMissingImage(w, r)
- }
- return
+ } else if config.DriveCacheEnabled {
+ // With cache enabled, if cache file is missing, return fallback (410)
+ if imageType == "icon" {
+ serveGlobeImage(w)
+ } else {
+ serveMissingImage(w)
}
+ return
}
}
- // For full images, proceed to proxy the image
-
- // Image not cached or caching not enabled
- imageKey := fmt.Sprintf("%s_%s", imageID, imageType)
-
- imageURLMapMu.RLock()
- imageURL, exists := imageURLMap[imageKey]
- imageURLMapMu.RUnlock()
-
+ // --- 4. MISSING MAPPING: No source image URL known, fallback with 410 ---
if !exists {
- // Cannot find original URL, serve missing image
- serveMissingImage(w, r)
+ serveMissingImage(w)
return
}
- // For thumbnails, if HardCacheEnabled is true, and image not cached, serve missing image
+ // --- 5. PROXY ICON: If not cached, and icon requested, proxy original ---
+ if imageType == "icon" && !hasExtension {
+ resp, err := http.Get(imageURL)
+ if err != nil {
+ recordInvalidImageID(imageID)
+ serveGlobeImage(w)
+ return
+ }
+ defer resp.Body.Close()
+
+ contentType := resp.Header.Get("Content-Type")
+ if contentType != "" && strings.HasPrefix(contentType, "image/") {
+ w.Header().Set("Content-Type", contentType)
+ } else {
+ serveGlobeImage(w)
+ return
+ }
+ _, _ = io.Copy(w, resp.Body)
+ return
+ }
+
+ // --- 6. PROXY THUMB (if cache disabled): With cache ON, must be on disk ---
if imageType == "thumb" && config.DriveCacheEnabled {
- // Thumbnail should be cached, but not found
- serveMissingImage(w, r)
+ serveMissingImage(w)
return
}
- // For full images, proceed to proxy the image
-
- // Fetch the image from the original URL
+ // --- 7. PROXY FULL: For full images, proxy directly ---
resp, err := http.Get(imageURL)
if err != nil {
printWarn("Error fetching image: %v", err)
recordInvalidImageID(imageID)
- serveMissingImage(w, r)
+ serveMissingImage(w)
return
}
defer resp.Body.Close()
- // Check if the request was successful
if resp.StatusCode != http.StatusOK {
- serveMissingImage(w, r)
+ serveMissingImage(w)
return
}
-
- // Set the Content-Type header to the type of the fetched image
contentType := resp.Header.Get("Content-Type")
if contentType != "" && strings.HasPrefix(contentType, "image/") {
w.Header().Set("Content-Type", contentType)
} else {
- serveMissingImage(w, r)
+ serveMissingImage(w)
return
}
-
- // Write the image content to the response
if _, err := io.Copy(w, resp.Body); err != nil {
printWarn("Error writing image to response: %v", err)
}
@@ -536,20 +556,31 @@ func safeDecodeImage(contentType string, data []byte) (img image.Image, err erro
}
// Serve missing.svg
-func serveMissingImage(w http.ResponseWriter, r *http.Request) {
+func serveMissingImage(w http.ResponseWriter) {
missingImagePath := filepath.Join("static", "images", "missing.svg")
+
+ // Set error code FIRST
+ w.WriteHeader(http.StatusGone)
+
+ // Now read the file and write it manually, to avoid conflict with http.ServeFile
+ data, err := os.ReadFile(missingImagePath)
+ if err != nil {
+ http.Error(w, "globe.svg not found", http.StatusInternalServerError)
+ return
+ }
+
w.Header().Set("Content-Type", "image/svg+xml")
w.Header().Set("Cache-Control", "no-store, must-revalidate")
w.Header().Set("Pragma", "no-cache")
w.Header().Set("Expires", "0")
- http.ServeFile(w, r, missingImagePath)
+ _, _ = w.Write(data)
}
-func serveGlobeImage(w http.ResponseWriter, r *http.Request) {
+func serveGlobeImage(w http.ResponseWriter) {
globePath := filepath.Join("static", "images", "globe.svg")
// Set error code FIRST
- w.WriteHeader(http.StatusNotFound)
+ w.WriteHeader(http.StatusGone)
// Now read the file and write it manually, to avoid conflict with http.ServeFile
data, err := os.ReadFile(globePath)
diff --git a/common.go b/common.go
old mode 100755
new mode 100644
index 06c47bf..c5f9336
--- a/common.go
+++ b/common.go
@@ -1,8 +1,6 @@
package main
import (
- "crypto/rand"
- "encoding/base64"
"encoding/json"
"fmt"
"html/template"
@@ -110,15 +108,15 @@ func renderTemplate(w http.ResponseWriter, tmplName string, data map[string]inte
}
}
-// Randoms string generator used for auth code
-func generateStrongRandomString(length int) string {
- bytes := make([]byte, length)
- _, err := rand.Read(bytes)
- if err != nil {
- printErr("Error generating random string: %v", err)
- }
- return base64.URLEncoding.EncodeToString(bytes)[:length]
-}
+// // Randoms string generator used for auth code
+// func generateStrongRandomString(length int) string {
+// bytes := make([]byte, length)
+// _, err := rand.Read(bytes)
+// if err != nil {
+// printErr("Error generating random string: %v", err)
+// }
+// return base64.URLEncoding.EncodeToString(bytes)[:length]
+// }
// Checks if the URL already includes a protocol
func hasProtocol(url string) bool {
diff --git a/favicon.go b/favicon.go
index cd08682..50a88fd 100644
--- a/favicon.go
+++ b/favicon.go
@@ -52,8 +52,11 @@ type faviconDownloadRequest struct {
func init() {
// Start 5 worker goroutines to process favicon downloads
- for i := 0; i < 5; i++ {
- go faviconDownloadWorker()
+
+ if !config.DriveCacheEnabled {
+ for i := 0; i < 5; i++ {
+ go faviconDownloadWorker()
+ }
}
}
@@ -72,7 +75,7 @@ func faviconIDFromURL(rawURL string) string {
// Resolves favicon URL using multiple methods
func resolveFaviconURL(rawFavicon, pageURL string) (faviconURL, cacheID string) {
- cacheID = faviconIDFromURL(pageURL)
+ // cacheID = faviconIDFromURL(pageURL)
// Handle data URLs first
if strings.HasPrefix(rawFavicon, "data:image") {
@@ -135,7 +138,8 @@ func findFaviconInHeaders(pageURL string) string {
client := &http.Client{
Timeout: 3 * time.Second, // like 3 seconds for favicon should be enough
Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ DisableKeepAlives: true,
},
}
@@ -214,7 +218,7 @@ func checkURLExists(url string) bool {
}
// Add User-Agent
- userAgent, err := GetUserAgent("Text-Search-Brave")
+ userAgent, err := GetUserAgent("Text-Search-Favicons")
if err != nil {
printWarn("Error getting User-Agent: %v", err)
}
@@ -321,10 +325,6 @@ func getFaviconProxyURL(rawFavicon, pageURL string) string {
filename := fmt.Sprintf("%s_icon.webp", cacheID)
cachedPath := filepath.Join(config.DriveCache.Path, "images", filename)
- if _, err := os.Stat(cachedPath); err == nil {
- return fmt.Sprintf("/image/%s_icon.webp", cacheID)
- }
-
// Resolve URL
faviconURL, _ := resolveFaviconURL(rawFavicon, pageURL)
if faviconURL == "" {
@@ -333,23 +333,33 @@ func getFaviconProxyURL(rawFavicon, pageURL string) string {
}
// Check if already downloading
- faviconCache.RLock()
- downloading := faviconCache.m[cacheID]
- faviconCache.RUnlock()
+ imageURLMapMu.Lock()
+ imageURLMap[fmt.Sprintf("%s_icon", cacheID)] = faviconURL
+ imageURLMapMu.Unlock()
- if !downloading {
- faviconCache.Lock()
- faviconCache.m[cacheID] = true
- faviconCache.Unlock()
-
- // Send to download queue instead of starting goroutine
- faviconDownloadQueue <- faviconDownloadRequest{
- faviconURL: faviconURL,
- pageURL: pageURL,
- cacheID: cacheID,
+ if config.DriveCacheEnabled {
+ if _, err := os.Stat(cachedPath); err == nil {
+ return fmt.Sprintf("/image/%s_icon.webp", cacheID)
}
+ faviconCache.RLock()
+ downloading := faviconCache.m[cacheID]
+ faviconCache.RUnlock()
+
+ if !downloading {
+ faviconCache.Lock()
+ faviconCache.m[cacheID] = true
+ faviconCache.Unlock()
+
+ faviconDownloadQueue <- faviconDownloadRequest{
+ faviconURL: faviconURL,
+ pageURL: pageURL,
+ cacheID: cacheID,
+ }
+ }
+ return fmt.Sprintf("/image/%s_icon.webp", cacheID)
}
+ // Always proxy if cache is off
return fmt.Sprintf("/image/%s_icon.webp", cacheID)
}
@@ -451,7 +461,7 @@ func cacheFavicon(imageURL, imageID string) (string, bool, error) {
}
// Add User-Agent
- userAgent, err := GetUserAgent("Text-Search-Brave")
+ userAgent, err := GetUserAgent("Text-Search-Favicons")
if err != nil {
printWarn("Error getting User-Agent: %v", err)
}
diff --git a/go.mod b/go.mod
index b088e24..09adc78 100644
--- a/go.mod
+++ b/go.mod
@@ -20,6 +20,7 @@ require (
github.com/fyne-io/image v0.1.1
github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f
golang.org/x/net v0.33.0
+ golang.org/x/text v0.21.0
)
require (
@@ -64,6 +65,5 @@ require (
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.etcd.io/bbolt v1.3.11 // indirect
golang.org/x/sys v0.28.0 // indirect
- golang.org/x/text v0.21.0 // indirect
google.golang.org/protobuf v1.36.0 // indirect
)
diff --git a/static/js/dynamicscrollingtext.js b/static/js/dynamicscrollingtext.js
index 98811b5..8ce89ba 100644
--- a/static/js/dynamicscrollingtext.js
+++ b/static/js/dynamicscrollingtext.js
@@ -34,19 +34,19 @@
}
// Handle image/favicon loading errors
- function handleImageError(imgElement, retryCount = 8, retryDelay = 500) {
+ function handleImageError(imgElement, retryCount = 10, retryDelay = 200) {
const isFavicon = !!imgElement.closest('.favicon-wrapper');
const container = imgElement.closest(type === 'image' ? '.image' : '.result_item');
const titleSelector = type === 'image' ? '.img_title' : '.result-url';
const title = container?.querySelector(titleSelector);
const fullURL = imgElement.getAttribute('data-full');
- if (retryCount > 0 && !imgElement.dataset.checked404) {
- imgElement.dataset.checked404 = '1'; // avoid infinite loop
+ if (retryCount > 0 && !imgElement.dataset.checked410) {
+ imgElement.dataset.checked410 = '1'; // avoid infinite loop
fetch(fullURL, { method: 'HEAD' })
.then(res => {
- if (res.status === 404) {
+ if (res.status === 410) {
fallbackToGlobe(imgElement);
} else {
setTimeout(() => {
@@ -61,26 +61,32 @@
} else {
fallbackToGlobe(imgElement);
}
+ }
- function fallbackToGlobe(imgElement) {
- imgElement.closest('.favicon-wrapper')?.classList.remove('loading');
- if (title) title.classList.remove('title-loading');
+ function fallbackToGlobe(imgElement) {
+ const type = document.getElementById('template-data').getAttribute('data-type');
+ const isFavicon = !!imgElement.closest('.favicon-wrapper');
+ const container = imgElement.closest(type === 'image' ? '.image' : '.result_item');
+ const titleSelector = type === 'image' ? '.img_title' : '.result-url';
+ const title = container?.querySelector(titleSelector);
- if (isFavicon) {
- const wrapper = imgElement.closest('.favicon-wrapper') || imgElement.parentElement;
- const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
- svg.setAttribute("xmlns", "http://www.w3.org/2000/svg");
- svg.setAttribute("viewBox", "0 -960 960 960");
- svg.setAttribute("height", imgElement.height || "16");
- svg.setAttribute("width", imgElement.width || "16");
- svg.setAttribute("fill", "currentColor");
- svg.classList.add("favicon", "globe-fallback");
+ imgElement.closest('.favicon-wrapper')?.classList.remove('loading');
+ if (title) title.classList.remove('title-loading');
+
+ if (isFavicon) {
+ const wrapper = imgElement.closest('.favicon-wrapper') || imgElement.parentElement;
+ const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
+ svg.setAttribute("xmlns", "http://www.w3.org/2000/svg");
+ svg.setAttribute("viewBox", "0 -960 960 960");
+ svg.setAttribute("height", imgElement.height || "16");
+ svg.setAttribute("width", imgElement.width || "16");
+ svg.setAttribute("fill", "currentColor");
+ svg.classList.add("favicon", "globe-fallback");
svg.innerHTML = ``;
- imgElement.remove();
- wrapper.appendChild(svg);
- } else if (type === 'image') {
- container?.remove();
- }
+ imgElement.remove();
+ wrapper.appendChild(svg);
+ } else if (type === 'image') {
+ container?.remove();
}
}
@@ -113,7 +119,7 @@
function registerMediaElement(imgElement) {
const id = imgElement.getAttribute('data-id');
if (!id) return;
-
+
let wrapper = imgElement.closest('.favicon-wrapper');
if (!wrapper) {
wrapper = document.createElement('span');
@@ -121,24 +127,86 @@
imgElement.replaceWith(wrapper);
wrapper.appendChild(imgElement);
}
-
+
addLoadingEffects(imgElement);
-
- if (hardCacheEnabled) {
- imgElement.src = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';
- imgElement.onerror = () => handleImageError(imgElement, 3, 1000);
- } else {
- imgElement.src = imgElement.getAttribute('data-full');
- imgElement.onload = () => removeLoadingEffects(imgElement);
- imgElement.onerror = () => handleImageError(imgElement, 3, 1000);
- }
-
- // Track it
- if (!mediaMap.has(id)) {
- mediaMap.set(id, []);
- }
+
+ const tryLoadImage = (attempts = 25, delay = 300) => {
+ fetch(`/image_status?image_ids=${id}`)
+ .then(res => res.json())
+ .then(map => {
+ const url = map[id];
+ if (!url) {
+ if (attempts > 0) {
+ // Exponential backoff with jitter
+ const nextDelay = Math.min(delay * 2 + Math.random() * 100, 5000);
+ setTimeout(() => tryLoadImage(attempts - 1, nextDelay), nextDelay);
+ } else {
+ fallbackToGlobe(imgElement);
+ }
+ } else if (url.endsWith('globe.svg') || url.endsWith('missing.svg')) {
+ fallbackToGlobe(imgElement);
+ } else {
+ // Remove cache buster to leverage browser caching
+ const newImg = imgElement.cloneNode();
+ newImg.src = url;
+ newImg.onload = () => removeLoadingEffects(newImg);
+ // Add retry mechanism for final load
+ newImg.onerror = () => handleImageError(newImg, 3, 500);
+ imgElement.parentNode.replaceChild(newImg, imgElement);
+ }
+ })
+ .catch(() => {
+ if (attempts > 0) {
+ const nextDelay = Math.min(delay * 2, 5000);
+ setTimeout(() => tryLoadImage(attempts - 1, nextDelay), nextDelay);
+ } else {
+ fallbackToGlobe(imgElement);
+ }
+ });
+ };
+
+ // Initial load attempt with retry handler
+ imgElement.src = imgElement.getAttribute('data-full');
+ imgElement.onload = () => removeLoadingEffects(imgElement);
+ // Start polling immediately instead of waiting for error
+ setTimeout(() => tryLoadImage(), 500); // Start polling after initial load attempt
+
+ // Store reference in tracking map
+ if (!mediaMap.has(id)) mediaMap.set(id, []);
mediaMap.get(id).push(imgElement);
- }
+ }
+
+ function pollFaviconUntilReady(imgElement, id, retries = 8, delay = 700) {
+ let attempts = 0;
+ function poll() {
+ fetch(`/image_status?image_ids=${id}`)
+ .then(res => res.json())
+ .then(map => {
+ const url = map[id];
+ if (url && !url.endsWith('globe.svg') && !url.endsWith('missing.svg')) {
+
+ const newImg = imgElement.cloneNode();
+ newImg.src = url + "?v=" + Date.now();
+ newImg.onload = () => removeLoadingEffects(newImg);
+ newImg.onerror = () => fallbackToGlobe(newImg);
+ imgElement.parentNode.replaceChild(newImg, imgElement);
+ } else if (attempts < retries) {
+ attempts++;
+ setTimeout(poll, delay);
+ } else {
+ fallbackToGlobe(imgElement);
+ }
+ }).catch(() => {
+ if (attempts < retries) {
+ attempts++;
+ setTimeout(poll, delay);
+ } else {
+ fallbackToGlobe(imgElement);
+ }
+ });
+ }
+ poll();
+ }
// Check status of all tracked media elements
function checkMediaStatus() {
@@ -182,9 +250,10 @@
}
}
- mediaMap.clear();
- for (const [id, imgs] of stillPending) {
- mediaMap.set(id, imgs);
+ for (const id of Array.from(mediaMap.keys())) {
+ if (!stillPending.has(id)) {
+ mediaMap.delete(id);
+ }
}
};
diff --git a/templates/text.html b/templates/text.html
old mode 100755
new mode 100644
index 795a00e..2521553
--- a/templates/text.html
+++ b/templates/text.html
@@ -265,8 +265,8 @@
+ 