clean up

cache-images.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/chai2010/webp"
+	"github.com/fyne-io/image/ico"
 	"golang.org/x/image/bmp"
 	"golang.org/x/image/tiff"
 )
@@ -139,6 +140,8 @@ func cacheImage(imageURL, imageID string, isThumbnail bool) (string, bool, error
 	// Decode the image based on the content type
 	var img image.Image
 	switch contentType {
+	case "image/x-icon", "image/vnd.microsoft.icon":
+		img, err = ico.Decode(bytes.NewReader(data))
 	case "image/jpeg":
 		img, err = jpeg.Decode(bytes.NewReader(data))
 	case "image/png":

common.go

@@ -8,6 +8,7 @@ import (
 	"html/template"
 	mathrand "math/rand"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 )
@@ -36,6 +37,12 @@ type SearchEngine struct {
 	Func func(string, string, string, int) ([]SearchResult, time.Duration, error)
 }
 
+type LinkParts struct {
+	Domain  template.HTML
+	Path    template.HTML
+	RootURL string // used by getFaviconProxyURL()
+}
+
 // Helper function to render templates without elapsed time measurement
 func renderTemplate(w http.ResponseWriter, tmplName string, data map[string]interface{}) {
 	// Generate icon paths for SVG and PNG, including a 1/10 chance for an alternate icon
@@ -125,3 +132,44 @@ func FormatElapsedTime(elapsed time.Duration) string {
 	}
 	return fmt.Sprintf("%.2f %s", elapsed.Seconds(), Translate("seconds"))
 }
+func FormatURLParts(rawURL string) (domain, path, rootURL string) {
+	parsed, err := url.Parse(rawURL)
+	if err != nil || parsed.Host == "" {
+		return "", "", ""
+	}
+
+	domain = parsed.Host
+	if strings.HasPrefix(domain, "www.") {
+		domain = domain[4:]
+	}
+
+	rootURL = parsed.Scheme + "://" + parsed.Host
+
+	path = strings.Trim(parsed.Path, "/")
+	pathSegments := strings.Split(path, "/")
+	var cleanSegments []string
+	for _, seg := range pathSegments {
+		if seg != "" {
+			cleanSegments = append(cleanSegments, seg)
+		}
+	}
+	path = strings.Join(cleanSegments, "/")
+	return domain, path, rootURL
+}
+
+func FormatLinkHTML(rawURL string) LinkParts {
+	domain, path, root := FormatURLParts(rawURL)
+
+	lp := LinkParts{
+		RootURL: root,
+	}
+
+	lp.Domain = template.HTML(fmt.Sprintf(`<span class="result-domain">%s</span>`, template.HTMLEscapeString(domain)))
+
+	if path != "" {
+		pathDisplay := strings.ReplaceAll(path, "/", " › ")
+		lp.Path = template.HTML(fmt.Sprintf(`<span class="result-path"> › %s</span>`, template.HTMLEscapeString(pathDisplay)))
+	}
+
+	return lp
+}

favicon.go

@@ -0,0 +1,574 @@
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/tls"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"image"
+	"image/gif"
+	"image/jpeg"
+	"image/png"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/chai2010/webp"
+	"github.com/fyne-io/image/ico"
+	"golang.org/x/image/bmp"
+	"golang.org/x/image/draw"
+	"golang.org/x/image/tiff"
+	"golang.org/x/net/html"
+)
+
+var (
+	faviconCache = struct {
+		sync.RWMutex
+		m map[string]bool // tracks in-progress downloads
+	}{m: make(map[string]bool)}
+
+	// Common favicon paths to try
+	commonFaviconPaths = []string{
+		"/favicon.ico",
+		"/favicon.png",
+		"/favicon.jpg",
+		"/favicon.jpeg",
+		"/favicon.webp",
+		"/apple-touch-icon.png",
+		"/apple-touch-icon-precomposed.png",
+	}
+
+	// Regex to extract favicon URLs from HTML
+	iconLinkRegex = regexp.MustCompile(`<link[^>]+rel=["'](?:icon|shortcut icon|apple-touch-icon)["'][^>]+href=["']([^"']+)["']`)
+)
+
+// Add this near the top with other vars
+var (
+	faviconDownloadQueue = make(chan faviconDownloadRequest, 1000)
+)
+
+type faviconDownloadRequest struct {
+	faviconURL string
+	pageURL    string
+	cacheID    string
+}
+
+func init() {
+	// Start 5 worker goroutines to process favicon downloads
+	for i := 0; i < 5; i++ {
+		go faviconDownloadWorker()
+	}
+}
+
+func faviconDownloadWorker() {
+	for req := range faviconDownloadQueue {
+		cacheFavicon(req.faviconURL, req.cacheID)
+	}
+}
+
+// Generates a cache ID from URL
+func faviconIDFromURL(rawURL string) string {
+	hasher := md5.New()
+	hasher.Write([]byte(rawURL))
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+// Resolves favicon URL using multiple methods
+func resolveFaviconURL(rawFavicon, pageURL string) (faviconURL, cacheID string) {
+	cacheID = faviconIDFromURL(pageURL)
+
+	// Handle data URLs first
+	if strings.HasPrefix(rawFavicon, "data:image") {
+		parts := strings.SplitN(rawFavicon, ";base64,", 2)
+		if len(parts) == 2 {
+			data, err := base64.StdEncoding.DecodeString(parts[1])
+			if err == nil {
+				hasher := md5.New()
+				hasher.Write(data)
+				return rawFavicon, hex.EncodeToString(hasher.Sum(nil))
+			}
+		}
+		return "", "" // Invalid data URL
+	}
+
+	// Existing URL handling logic
+	if rawFavicon != "" && strings.HasPrefix(rawFavicon, "http") {
+		cacheID = faviconIDFromURL(rawFavicon)
+		return rawFavicon, cacheID
+	}
+
+	parsedPage, err := url.Parse(pageURL)
+	if err != nil {
+		return "", ""
+	}
+
+	// Method 1: Parse HTML
+	if favicon := findFaviconInHTML(pageURL); favicon != "" {
+		if strings.HasPrefix(favicon, "http") {
+			return favicon, faviconIDFromURL(favicon)
+		}
+		resolved := resolveRelativeURL(parsedPage, favicon)
+		return resolved, faviconIDFromURL(resolved)
+	}
+
+	// Method 2: Common paths
+	for _, path := range commonFaviconPaths {
+		testURL := "https://" + parsedPage.Host + path
+		if checkURLExists(testURL) {
+			return testURL, faviconIDFromURL(testURL)
+		}
+	}
+
+	// Method 3: HTTP headers
+	if headerIcon := findFaviconInHeaders(pageURL); headerIcon != "" {
+		if strings.HasPrefix(headerIcon, "http") {
+			return headerIcon, faviconIDFromURL(headerIcon)
+		}
+		resolved := resolveRelativeURL(parsedPage, headerIcon)
+		return resolved, faviconIDFromURL(resolved)
+	}
+
+	// Fallback
+	fallbackURL := "https://" + parsedPage.Host + "/favicon.ico"
+	return fallbackURL, faviconIDFromURL(fallbackURL)
+}
+
+// Checks HTTP headers for favicon links
+func findFaviconInHeaders(pageURL string) string {
+	client := &http.Client{
+		Timeout: 3 * time.Second, // like 3 seconds for favicon should be enough
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+
+	req, err := http.NewRequest("HEAD", pageURL, nil)
+	if err != nil {
+		return ""
+	}
+
+	// Add User-Agent
+	userAgent, err := GetUserAgent("findFaviconInHeaders")
+	if err != nil {
+		printWarn("Error getting User-Agent: %v", err)
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return ""
+	}
+	defer resp.Body.Close()
+
+	// Check Link headers (common for favicons)
+	if links, ok := resp.Header["Link"]; ok {
+		for _, link := range links {
+			parts := strings.Split(link, ";")
+			if len(parts) < 2 {
+				continue
+			}
+
+			urlPart := strings.TrimSpace(parts[0])
+			if !strings.HasPrefix(urlPart, "<") || !strings.HasSuffix(urlPart, ">") {
+				continue
+			}
+
+			urlPart = urlPart[1 : len(urlPart)-1] // Remove < and >
+			for _, part := range parts[1:] {
+				part = strings.TrimSpace(part)
+				if strings.EqualFold(part, `rel="icon"`) ||
+					strings.EqualFold(part, `rel=icon`) ||
+					strings.EqualFold(part, `rel="shortcut icon"`) ||
+					strings.EqualFold(part, `rel=shortcut icon`) {
+					return urlPart
+				}
+			}
+		}
+	}
+
+	return ""
+}
+
+// Helper to resolve relative URLs
+func resolveRelativeURL(base *url.URL, relative string) string {
+	if strings.HasPrefix(relative, "http") {
+		return relative
+	}
+	if strings.HasPrefix(relative, "//") {
+		return base.Scheme + ":" + relative
+	}
+	if strings.HasPrefix(relative, "/") {
+		return base.Scheme + "://" + base.Host + relative
+	}
+	return base.Scheme + "://" + base.Host + base.Path + "/" + relative
+}
+
+// Checks if a URL exists (returns 200 OK)
+func checkURLExists(url string) bool {
+	client := &http.Client{
+		Timeout: 5 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+	req, err := http.NewRequest("HEAD", url, nil)
+	if err != nil {
+		return false
+	}
+
+	// Add User-Agent
+	userAgent, err := GetUserAgent("Text-Search-Brave")
+	if err != nil {
+		printWarn("Error getting User-Agent: %v", err)
+	}
+	req.Header.Set("checkURLExists", userAgent)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return false
+	}
+	resp.Body.Close()
+	return resp.StatusCode == http.StatusOK
+}
+
+// Fetches HTML and looks for favicon links
+func findFaviconInHTML(pageURL string) string {
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+
+	req, err := http.NewRequest("GET", pageURL, nil)
+	if err != nil {
+		return ""
+	}
+
+	// Add User-Agent
+	userAgent, err := GetUserAgent("findFaviconInHTML")
+	if err != nil {
+		printWarn("Error getting User-Agent: %v", err)
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return ""
+	}
+	defer resp.Body.Close()
+
+	// Check if this is an AMP page
+	isAMP := false
+	for _, attr := range resp.Header["Link"] {
+		if strings.Contains(attr, "rel=\"amphtml\"") {
+			isAMP = true
+			break
+		}
+	}
+
+	// Parse HTML
+	doc, err := html.Parse(resp.Body)
+	if err != nil {
+		return ""
+	}
+
+	var faviconURL string
+	var findLinks func(*html.Node)
+	findLinks = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "link" {
+			var rel, href string
+			for _, attr := range n.Attr {
+				switch attr.Key {
+				case "rel":
+					rel = attr.Val
+				case "href":
+					href = attr.Val
+				}
+			}
+
+			// Prioritize different favicon types
+			if href != "" {
+				switch rel {
+				case "icon", "shortcut icon", "apple-touch-icon", "apple-touch-icon-precomposed":
+					// For AMP pages, prefer the non-versioned URL if possible
+					if isAMP {
+						if u, err := url.Parse(href); err == nil {
+							u.RawQuery = "" // Remove query parameters
+							href = u.String()
+						}
+					}
+					if faviconURL == "" || // First found
+						rel == "apple-touch-icon" || // Prefer apple-touch-icon
+						rel == "icon" { // Then regular icon
+						faviconURL = href
+					}
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			findLinks(c)
+		}
+	}
+	findLinks(doc)
+
+	return faviconURL
+}
+
+func getFaviconProxyURL(rawFavicon, pageURL string) string {
+	if pageURL == "" {
+		return "/static/images/missing.svg"
+	}
+
+	cacheID := faviconIDFromURL(pageURL)
+	filename := fmt.Sprintf("%s_thumb.webp", cacheID)
+	cachedPath := filepath.Join(config.DriveCache.Path, "images", filename)
+
+	if _, err := os.Stat(cachedPath); err == nil {
+		return fmt.Sprintf("/image/%s_thumb.webp", cacheID)
+	}
+
+	// Resolve URL
+	faviconURL, _ := resolveFaviconURL(rawFavicon, pageURL)
+	if faviconURL == "" {
+		recordInvalidImageID(cacheID)
+		return "/static/images/missing.svg"
+	}
+
+	// Check if already downloading
+	faviconCache.RLock()
+	downloading := faviconCache.m[cacheID]
+	faviconCache.RUnlock()
+
+	if !downloading {
+		faviconCache.Lock()
+		faviconCache.m[cacheID] = true
+		faviconCache.Unlock()
+
+		// Send to download queue instead of starting goroutine
+		faviconDownloadQueue <- faviconDownloadRequest{
+			faviconURL: faviconURL,
+			pageURL:    pageURL,
+			cacheID:    cacheID,
+		}
+	}
+
+	return fmt.Sprintf("/image/%s_thumb.webp", cacheID)
+}
+
+// Caches favicon, always saving *_thumb.webp
+func cacheFavicon(imageURL, imageID string) (string, bool, error) {
+	// if imageURL == "" {
+	// 	recordInvalidImageID(imageID)
+	// 	return "", false, fmt.Errorf("empty image URL for image ID %s", imageID)
+	// }
+
+	// Debug
+	fmt.Printf("Downloading favicon [%s] for ID [%s]\n", imageURL, imageID)
+
+	filename := fmt.Sprintf("%s_thumb.webp", imageID)
+	imageCacheDir := filepath.Join(config.DriveCache.Path, "images")
+	if err := os.MkdirAll(imageCacheDir, 0755); err != nil {
+		return "", false, fmt.Errorf("couldn't create images folder: %v", err)
+	}
+	cachedImagePath := filepath.Join(imageCacheDir, filename)
+	tempImagePath := cachedImagePath + ".tmp"
+
+	// Already cached?
+	if _, err := os.Stat(cachedImagePath); err == nil {
+		return cachedImagePath, true, nil
+	}
+
+	cachingImagesMu.Lock()
+	if _, exists := cachingImages[imageURL]; !exists {
+		cachingImages[imageURL] = &sync.Mutex{}
+	}
+	mu := cachingImages[imageURL]
+	cachingImagesMu.Unlock()
+
+	mu.Lock()
+	defer mu.Unlock()
+
+	// Recheck after lock
+	if _, err := os.Stat(cachedImagePath); err == nil {
+		return cachedImagePath, true, nil
+	}
+
+	cachingSemaphore <- struct{}{}
+	defer func() { <-cachingSemaphore }()
+
+	var data []byte
+	var contentType string
+
+	// Handle data URLs
+	if strings.HasPrefix(imageURL, "data:") {
+		commaIndex := strings.Index(imageURL, ",")
+		if commaIndex == -1 {
+			recordInvalidImageID(imageID)
+			return "", false, fmt.Errorf("invalid data URL: no comma")
+		}
+		headerPart := imageURL[:commaIndex]
+		dataPart := imageURL[commaIndex+1:]
+
+		mediaType := "text/plain"
+		base64Encoded := false
+		if strings.HasPrefix(headerPart, "data:") {
+			mediaTypePart := headerPart[5:]
+			mediaTypeParts := strings.SplitN(mediaTypePart, ";", 2)
+			mediaType = mediaTypeParts[0]
+			if len(mediaTypeParts) > 1 {
+				for _, param := range strings.Split(mediaTypeParts[1], ";") {
+					param = strings.TrimSpace(param)
+					if param == "base64" {
+						base64Encoded = true
+					}
+				}
+			}
+		}
+
+		if base64Encoded {
+			data, _ = base64.StdEncoding.DecodeString(dataPart)
+		} else {
+			decodedStr, err := url.QueryUnescape(dataPart)
+			if err != nil {
+				data = []byte(dataPart)
+			} else {
+				data = []byte(decodedStr)
+			}
+		}
+
+		contentType = mediaType
+	} else {
+		// Download from HTTP URL
+		client := &http.Client{
+			Timeout: 15 * time.Second,
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+		}
+
+		req, err := http.NewRequest("GET", imageURL, nil)
+		if err != nil {
+			recordInvalidImageID(imageID)
+			return "", false, err
+		}
+
+		// Add User-Agent
+		userAgent, err := GetUserAgent("Text-Search-Brave")
+		if err != nil {
+			printWarn("Error getting User-Agent: %v", err)
+		}
+		req.Header.Set("User-Agent", userAgent)
+
+		resp, err := client.Do(req)
+		if err != nil {
+			recordInvalidImageID(imageID)
+			return "", false, err
+		}
+		defer resp.Body.Close()
+
+		data, err = io.ReadAll(resp.Body)
+		if err != nil {
+			recordInvalidImageID(imageID)
+			return "", false, err
+		}
+
+		contentType = http.DetectContentType(data)
+	}
+
+	if !strings.HasPrefix(contentType, "image/") {
+		recordInvalidImageID(imageID)
+		return "", false, fmt.Errorf("URL did not return an image: %s", imageURL)
+	}
+
+	// SVG special case
+	if contentType == "image/svg+xml" {
+		err := os.WriteFile(tempImagePath, data, 0644)
+		if err != nil {
+			recordInvalidImageID(imageID)
+			return "", false, err
+		}
+		err = os.Rename(tempImagePath, cachedImagePath)
+		if err != nil {
+			recordInvalidImageID(imageID)
+			return "", false, err
+		}
+		cachingImagesMu.Lock()
+		delete(cachingImages, imageURL)
+		cachingImagesMu.Unlock()
+		return cachedImagePath, true, nil
+	}
+
+	// Decode image
+	var img image.Image
+	var err error
+	switch contentType {
+	case "image/x-icon", "image/vnd.microsoft.icon":
+		img, err = ico.Decode(bytes.NewReader(data))
+	case "image/jpeg":
+		img, err = jpeg.Decode(bytes.NewReader(data))
+	case "image/png":
+		img, err = png.Decode(bytes.NewReader(data))
+	case "image/gif":
+		img, err = gif.Decode(bytes.NewReader(data))
+	case "image/webp":
+		img, err = webp.Decode(bytes.NewReader(data))
+	case "image/bmp":
+		img, err = bmp.Decode(bytes.NewReader(data))
+	case "image/tiff":
+		img, err = tiff.Decode(bytes.NewReader(data))
+	default:
+		recordInvalidImageID(imageID)
+		return "", false, fmt.Errorf("unsupported image type: %s", contentType)
+	}
+	if err != nil {
+		recordInvalidImageID(imageID)
+		return "", false, err
+	}
+
+	// Resize
+	maxSize := 16
+	width := img.Bounds().Dx()
+	height := img.Bounds().Dy()
+
+	if width > maxSize || height > maxSize {
+		dst := image.NewRGBA(image.Rect(0, 0, maxSize, maxSize))
+		draw.ApproxBiLinear.Scale(dst, dst.Bounds(), img, img.Bounds(), draw.Over, nil)
+		img = dst
+	}
+
+	// Save as WebP
+	outFile, err := os.Create(tempImagePath)
+	if err != nil {
+		recordInvalidImageID(imageID)
+		return "", false, err
+	}
+	defer outFile.Close()
+
+	options := &webp.Options{Lossless: false, Quality: 80}
+	err = webp.Encode(outFile, img, options)
+	if err != nil {
+		recordInvalidImageID(imageID)
+		return "", false, err
+	}
+
+	err = os.Rename(tempImagePath, cachedImagePath)
+	if err != nil {
+		recordInvalidImageID(imageID)
+		return "", false, err
+	}
+
+	cachingImagesMu.Lock()
+	delete(cachingImages, imageURL)
+	cachingImagesMu.Unlock()
+
+	return cachedImagePath, true, nil
+}

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/blevesearch/bleve/v2 v2.4.4
 	github.com/chromedp/cdproto v0.0.0-20241022234722-4d5d5faf59fb
 	github.com/chromedp/chromedp v0.11.2
+	github.com/fyne-io/image v0.1.1
 	github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f
 	golang.org/x/net v0.33.0
 )
@@ -55,11 +56,11 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/jsummers/gobmp v0.0.0-20230614200233-a9de23ed2e25 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.etcd.io/bbolt v1.3.11 // indirect
 	golang.org/x/sys v0.28.0 // indirect

go.sum

@@ -56,6 +56,8 @@ github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHG
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fyne-io/image v0.1.1 h1:WH0z4H7qfvNUw5l4p3bC1q70sa5+YWVt6HCj7y4VNyA=
+github.com/fyne-io/image v0.1.1/go.mod h1:xrfYBh6yspc+KjkgdZU/ifUC9sPA5Iv7WYUBzQKK7JM=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
 github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
@@ -84,6 +86,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jsummers/gobmp v0.0.0-20230614200233-a9de23ed2e25 h1:YLvr1eE6cdCqjOe972w/cYF+FjW34v27+9Vo5106B4M=
+github.com/jsummers/gobmp v0.0.0-20230614200233-a9de23ed2e25/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
 github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
 github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
 github.com/leonelquinteros/gotext v1.7.0 h1:jcJmF4AXqyamP7vuw2MMIKs+O3jAEmvrc5JQiI8Ht/8=
@@ -111,8 +115,8 @@ github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMT
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=

static/css/style-loadingcircle.css

@@ -0,0 +1,31 @@
+.favicon-wrapper {
+    position: relative;
+    display: inline-block;
+    width: 16px;
+    height: 16px;
+}
+
+.favicon-wrapper.loading img {
+    visibility: hidden; /* hide placeholder */
+}
+
+.favicon-wrapper.loading::after {
+    content: "";
+    position: absolute;
+    top: 50%;
+    left: 50%;
+    width: 14px;
+    height: 14px;
+    margin: -8px 0 0 -8px;
+    border: 2px solid var(--html-bg);
+    border-top-color: var(--fg);
+    border-radius: 50%;
+    animation: spin 0.7s linear infinite;
+    z-index: 2;
+}
+
+@keyframes spin {
+    to {
+        transform: rotate(360deg);
+    }
+}

static/css/style.css

@@ -1309,6 +1309,91 @@ p {
     text-shadow: 1px 1px 2px var(--border) !important; /* Adjust text shadow */
 }
 
+/* Favicon styling */
+.result_header {
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    margin-bottom: 4px;
+}
+
+.favicon-container {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: 18px;
+    border-radius: 8%;
+    flex-shrink: 0;
+}
+
+.favicon {
+    width: 16px;
+    height: 16px;
+    border-radius: 3px;
+    box-shadow: 0 0 2px rgba(0, 0, 0, 0.4);
+}
+
+/* Result link styling */
+.result-link {
+    color: var(--fg);
+    font-size: 14px;
+    text-decoration: none;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+}
+
+.result-link:hover {
+    text-decoration: underline;
+}
+
+/* Result item spacing */
+.result_item {
+    margin-bottom: 1.5rem;
+}
+
+.result-title h3 {
+    margin: 4px 0;
+    font-weight: 400;
+}
+
+.result-description {
+    margin: 4px 0 0 0;
+    color: var(--font-fg);
+    line-height: 1.4;
+}
+
+.results br {
+    display: none;
+}
+
+.result-url {
+    font-size: 14px;
+    color: var(--fg);
+    display: flex;
+    align-items: center;
+    flex-wrap: wrap;
+    gap: 4px;
+}
+
+.result-domain {
+    color: var(--fg);
+    font-weight: 600;
+    opacity: 0.85;
+}
+
+.result-path {
+    color: var(--font-fg);
+    font-weight: 500;
+    opacity: 0.8;
+}
+/* 
+.result-path::before {
+    content: "›";
+    margin: 0 4px;
+    opacity: 0.6;
+} */
+
 body, h1, p, a, input, button {
     color: var(--text-color); /* Applies the text color based on theme */
     background-color: var(--background-color); /* Applies the background color based on theme */

static/js/dynamicscrollingimages.js

@@ -2,6 +2,9 @@
 (function() {
     // Add loading effects to image and title
     function addLoadingEffects(imgElement) {
+        const container = imgElement.closest('.image');
+        if (!container) return; // avoid null dereference
+
         const title = imgElement.closest('.image').querySelector('.img_title');
         imgElement.classList.add('loading-image');
         title.classList.add('title-loading');

static/js/dynamicscrollingtext.js

@@ -0,0 +1,246 @@
+(function() {
+    // Get template data and configuration
+    const templateData = document.getElementById('template-data');
+    const type = templateData.getAttribute('data-type');
+    const hardCacheEnabled = templateData.getAttribute('data-hard-cache-enabled') === 'true';
+    
+    // Track all favicon/image elements and their IDs
+    let allMediaElements = [];
+    let allMediaIds = [];
+    let statusCheckTimeout = null;
+
+    // Add loading effects to image/favicon and associated text
+    function addLoadingEffects(imgElement) {
+        const container = imgElement.closest(type === 'image' ? '.image' : '.result_item');
+        if (!container) return;
+
+        const titleSelector = type === 'image' ? '.img_title' : '.result-url';
+        const title = container.querySelector(titleSelector);
+        imgElement.closest('.favicon-wrapper')?.classList.add('loading');
+        // if (title) title.classList.add('title-loading');
+    }
+
+    // Remove loading effects when image/favicon loads
+    function removeLoadingEffects(imgElement) {
+        const container = imgElement.closest(type === 'image' ? '.image' : '.result_item');
+        const titleSelector = type === 'image' ? '.img_title' : '.result-url';
+        const title = container?.querySelector(titleSelector);
+        imgElement.closest('.favicon-wrapper')?.classList.remove('loading');
+        if (title) title.classList.remove('title-loading');
+        
+        if (type === 'image' && imgElement.src.endsWith('/images/missing.svg')) {
+            container.remove();
+        }
+    }
+
+    // Handle image/favicon loading errors
+    function handleImageError(imgElement) {
+        const container = imgElement.closest(type === 'image' ? '.image' : '.result_item');
+        const titleSelector = type === 'image' ? '.img_title' : '.result-url';
+        const title = container?.querySelector(titleSelector);
+        
+        imgElement.closest('.favicon-wrapper')?.classList.remove('loading');
+        if (title) title.classList.remove('title-loading');
+        
+        if (type === 'image') {
+            container.style.display = 'none';
+        } else {
+            imgElement.src = '/static/images/missing.svg';
+        }
+    }
+
+    // Shared configuration
+    const statusCheckInterval = 500;
+    const scrollThreshold = 500;
+    const loadingIndicator = document.getElementById('message-bottom-right');
+    let loadingTimer;
+    let isFetching = false;
+    let page = parseInt(templateData.getAttribute('data-page')) || 1;
+    let query = templateData.getAttribute('data-query');
+    let noMoreImages = false;
+
+    function showLoadingMessage() {
+        loadingIndicator.classList.add('visible');
+    }
+    
+    function hideLoadingMessage() {
+        loadingIndicator.classList.remove('visible');
+    }
+
+    function ensureScrollable() {
+        if (noMoreImages) return;
+        if (document.body.scrollHeight <= window.innerHeight) {
+            fetchNextPage();
+        }
+    }
+
+    // Register a new media element for tracking
+    function registerMediaElement(imgElement) {
+        const id = imgElement.getAttribute('data-id');
+        if (!id || allMediaIds.includes(id)) return;
+
+        // Wrap the image in a .favicon-wrapper if not already
+        if (!imgElement.parentElement.classList.contains('favicon-wrapper')) {
+            const wrapper = document.createElement('span');
+            wrapper.classList.add('favicon-wrapper');
+            imgElement.parentElement.replaceChild(wrapper, imgElement);
+            wrapper.appendChild(imgElement);
+        }
+
+        // Track and style
+        allMediaElements.push(imgElement);
+        allMediaIds.push(id);
+        addLoadingEffects(imgElement);
+
+
+        if (!hardCacheEnabled) {
+            imgElement.src = ''; // don't show anything until actual URL arrives
+        }        
+        
+        // Schedule a status check if not already pending
+        if (!statusCheckTimeout) {
+            statusCheckTimeout = setTimeout(checkMediaStatus, statusCheckInterval);
+        }
+    }
+
+    // Check status of all tracked media elements
+    function checkMediaStatus() {
+        statusCheckTimeout = null;
+        
+        if (allMediaIds.length === 0) return;
+
+        // Group IDs to avoid very long URLs
+        const idGroups = [];
+        for (let i = 0; i < allMediaIds.length; i += 50) {
+            idGroups.push(allMediaIds.slice(i, i + 50));
+        }
+
+        const checkGroup = (group) => {
+            return fetch(`/image_status?image_ids=${group.join(',')}`)
+                .then(response => response.json())
+                .then(statusMap => {
+                    const pendingElements = [];
+                    const pendingIds = [];
+
+                    allMediaElements.forEach((imgElement, index) => {
+                        const id = allMediaIds[index];
+                        if (group.includes(id)) {
+                            if (statusMap[id]) {
+                                if (imgElement.src !== statusMap[id]) {
+                                    imgElement.src = statusMap[id];
+                                    imgElement.onload = () => removeLoadingEffects(imgElement);
+                                    imgElement.onerror = () => handleImageError(imgElement);
+                                }
+                            } else {
+                                pendingElements.push(imgElement);
+                                pendingIds.push(id);
+                            }
+                        }
+                    });
+
+                    // Update global arrays with remaining pending items
+                    allMediaElements = pendingElements;
+                    allMediaIds = pendingIds;
+                });
+        };
+
+        // Process all groups sequentially
+        const processGroups = async () => {
+            for (const group of idGroups) {
+                try {
+                    await checkGroup(group);
+                } catch (error) {
+                    console.error('Status check error:', error);
+                }
+            }
+
+            // If we still have pending items, schedule another check
+            if (allMediaIds.length > 0) {
+                statusCheckTimeout = setTimeout(checkMediaStatus, statusCheckInterval);
+            }
+        };
+
+        processGroups();
+    }
+
+    function fetchNextPage() {
+        if (isFetching || noMoreImages) return;
+        
+        loadingTimer = setTimeout(() => {
+            showLoadingMessage();
+        }, 150);
+        
+        isFetching = true;
+        page += 1;
+
+        fetch(`/search?q=${encodeURIComponent(query)}&t=${type}&p=${page}&ajax=true`)
+            .then(response => response.text())
+            .then(html => {
+                clearTimeout(loadingTimer);
+                hideLoadingMessage();
+
+                let tempDiv = document.createElement('div');
+                tempDiv.innerHTML = html;
+                let newItems = tempDiv.querySelectorAll(type === 'image' ? '.image' : '.result_item');
+
+                if (newItems.length > 0) {
+                    let resultsContainer = document.querySelector(type === 'image' ? '.images' : '.results');
+                    newItems.forEach(item => {
+                        let clonedItem = item.cloneNode(true);
+                        resultsContainer.appendChild(clonedItem);
+
+                        // Register any new media elements
+                        const img = clonedItem.querySelector('img[data-id]');
+                        if (img) {
+                            registerMediaElement(img);
+                        }
+                    });
+
+                    ensureScrollable();
+                } else {
+                    noMoreImages = true;
+                }
+                isFetching = false;
+            })
+            .catch(error => {
+                clearTimeout(loadingTimer);
+                hideLoadingMessage();
+                console.error('Fetch error:', error);
+                isFetching = false;
+            });
+    }
+
+    // Initialize all existing media elements
+    function initializeMediaElements() {
+        document.querySelectorAll('img[data-id]').forEach(img => {
+            registerMediaElement(img);
+        });
+
+        // Start periodic checks if hard cache is enabled
+        if (hardCacheEnabled && allMediaIds.length > 0) {
+            statusCheckTimeout = setTimeout(checkMediaStatus, statusCheckInterval);
+        }
+    }
+
+    // Initialize when DOM is ready
+    if (document.readyState === 'complete') {
+        initializeMediaElements();
+    } else {
+        window.addEventListener('load', initializeMediaElements);
+    }
+
+    // Infinite scroll handler
+    window.addEventListener('scroll', () => {
+        if (isFetching || noMoreImages) return;
+        if (window.innerHeight + window.scrollY >= document.body.offsetHeight - scrollThreshold) {
+            fetchNextPage();
+        }
+    });
+
+    // Clean up on page unload
+    window.addEventListener('beforeunload', () => {
+        if (statusCheckTimeout) {
+            clearTimeout(statusCheckTimeout);
+        }
+    });
+})();

templates/text.html

@@ -10,7 +10,7 @@
     <link rel="stylesheet" href="/static/css/style.css">
     <link rel="stylesheet" href="/static/css/{{.Theme}}.css">
     <link rel="stylesheet" href="/static/css/style-fonts.css">
-    <link rel="stylesheet" href="/static/css/style-loadingindicator.css">
+    <link rel="stylesheet" href="/static/css/style-loadingcircle.css">
     <link rel="stylesheet" href="/static/css/style-menu.css">
     <link rel="search" type="application/opensearchdescription+xml" title="{{ translate "site_name" }}" href="/opensearch.xml">    
     <!-- Icons -->
@@ -141,24 +141,41 @@
     <p class="fetched fetched_dif_files fetched_tor">{{ translate "fetched_in" .Fetched }}</p>
    
     <div class="results" id="results">
-        {{if .Results}}
-            {{range .Results}}
-                <div class="result_item">
-                    <a id="link" href="{{.URL}}">{{.URL}}</a>
-                    <a href="{{.URL}}"><h3>{{.Header}}</h3></a>
-                    <p>{{.Description}}</p>
+        {{ if .Results }}
+            {{ range .Results }}
+            <div class="result_item">
+                <div class="result_header">
+                    <div class="favicon-container">
+                        <img
+                            src="/static/images/placeholder.svg"
+                            data-id="{{ .FaviconID }}"
+                            data-full="/image/{{ .FaviconID }}_thumb.webp"
+                            alt="🌐"
+                            class="favicon placeholder-img"
+                        />
+                    </div>
+                    <div class="result-url">
+                        {{ .PrettyLink.Domain }}
+                        {{ if .PrettyLink.Path }}
+                            {{ .PrettyLink.Path }}
+                        {{ end }}
+                    </div>
                 </div>
-                <br>
-            {{end}}
-        {{else if .NoResults}}
+                <a href="{{ .URL }}" class="result-title"><h3>{{ .Header }}</h3></a>
+                <p class="result-description">{{ .Description }}</p>
+            </div>
+            {{ end }}
+        {{ else if .NoResults }}
             <div class="no-results-found">
                 {{ translate "no_results_found" .Query }}<br>
                 {{ translate "suggest_rephrase" }}
             </div>
-        {{else}}
-            <div class="no-results-found">{{ translate "no_more_results" }}</div>
-        {{end}}
-    </div>    
+        {{ else }}
+            <div class="no-results-found">
+                {{ translate "no_more_results" }}
+            </div>
+        {{ end }}
+    </div>     
     <div id="message-bottom-right" class="message-bottom-right">
         <span id="loading-text">{{ translate "searching_for_new_results" }}</span><span class="dot">.</span><span class="dot">.</span><span class="dot">.</span>
     </div>
@@ -177,7 +194,7 @@
         </form>
     </div>
     <div id="template-data" data-page="{{ .Page }}" data-query="{{ .Query }}" data-type="text"></div>
-    <script defer src="/static/js/dynamicscrolling.js"></script>
+    <script defer src="/static/js/dynamicscrollingtext.js"></script>
     <script defer src="/static/js/autocomplete.js"></script>
     <script defer src="/static/js/minimenu.js"></script>
     <script>

text.go

@@ -1,7 +1,10 @@
 package main
 
 import (
+	"fmt"
 	"net/http"
+	"os"
+	"path/filepath"
 	"time"
 )
 
@@ -33,7 +36,13 @@ func initTextEngines() {
 func HandleTextSearch(w http.ResponseWriter, settings UserSettings, query string, page int) {
 	startTime := time.Now()
 
-	cacheKey := CacheKey{Query: query, Page: page, Safe: settings.SafeSearch == "active", Lang: settings.SearchLanguage, Type: "text"}
+	cacheKey := CacheKey{
+		Query: query,
+		Page:  page,
+		Safe:  settings.SafeSearch == "active",
+		Lang:  settings.SearchLanguage,
+		Type:  "text",
+	}
 	combinedResults := getTextResultsFromCacheOrFetch(cacheKey, query, settings.SafeSearch, settings.SearchLanguage, page)
 
 	hasPrevPage := page > 1
@@ -46,27 +55,65 @@ func HandleTextSearch(w http.ResponseWriter, settings UserSettings, query string
 
 	elapsedTime := time.Since(startTime)
 
-	// Prepare the data to pass to the template
-	data := map[string]interface{}{
-		"Results":         combinedResults,
-		"Query":           query,
-		"Fetched":         FormatElapsedTime(elapsedTime),
-		"Page":            page,
-		"HasPrevPage":     page > 1,
-		"HasNextPage":     len(combinedResults) >= 50,
-		"NoResults":       len(combinedResults) == 0,
-		"LanguageOptions": languageOptions,
-		"CurrentLang":     settings.SearchLanguage,
-		"Theme":           settings.Theme,
-		"Safe":            settings.SafeSearch,
-		"IsThemeDark":     settings.IsThemeDark,
-		"Trans":           Translate,
+	// Simplified result structure without waiting for favicons
+	type DecoratedResult struct {
+		TextSearchResult
+		PrettyLink LinkParts
+		FaviconID  string // Just the ID, URL will be generated client-side
+	}
+
+	var decoratedResults []DecoratedResult
+	for _, r := range combinedResults {
+		if r.URL == "" {
+			continue
+		}
+
+		prettyLink := FormatLinkHTML(r.URL)
+		faviconID := faviconIDFromURL(prettyLink.RootURL)
+
+		decoratedResults = append(decoratedResults, DecoratedResult{
+			TextSearchResult: r,
+			PrettyLink:       prettyLink,
+			FaviconID:        faviconID,
+		})
+
+		// Start async favicon fetch if not already cached
+		go ensureFaviconIsCached(faviconID, prettyLink.RootURL)
+	}
+
+	data := map[string]interface{}{
+		"Results":          decoratedResults,
+		"Query":            query,
+		"Fetched":          FormatElapsedTime(elapsedTime),
+		"Page":             page,
+		"HasPrevPage":      hasPrevPage,
+		"HasNextPage":      len(combinedResults) >= 50,
+		"NoResults":        len(combinedResults) == 0,
+		"LanguageOptions":  languageOptions,
+		"CurrentLang":      settings.SearchLanguage,
+		"Theme":            settings.Theme,
+		"Safe":             settings.SafeSearch,
+		"IsThemeDark":      settings.IsThemeDark,
+		"Trans":            Translate,
+		"HardCacheEnabled": config.DriveCacheEnabled,
 	}
 
-	// Render the template without measuring time
 	renderTemplate(w, "text.html", data)
 }
 
+func ensureFaviconIsCached(faviconID, rootURL string) {
+	// Check if already exists in cache
+	filename := fmt.Sprintf("%s_thumb.webp", faviconID)
+	cachedPath := filepath.Join(config.DriveCache.Path, "images", filename)
+
+	if _, err := os.Stat(cachedPath); err == nil {
+		return // Already cached
+	}
+
+	// Not cached, initiate download
+	getFaviconProxyURL("", rootURL) // This will trigger async download
+}
+
 func getTextResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page int) []TextSearchResult {
 	cacheChan := make(chan []SearchResult)
 	var combinedResults []TextSearchResult