results fetching improvements

images.go

@@ -89,31 +89,33 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
 
 func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
 	var results []ImageSearchResult
+	engineCount := len(imageSearchEngines)
 	safeBool := safe == "active"
 
-	for _, engine := range imageSearchEngines {
+	// Determine the engine to use based on the page number
-		printInfo("Using image search engine: %s", engine.Name)
+	engineIndex := (page - 1) % engineCount
+	engine := imageSearchEngines[engineIndex]
 
-		searchResults, _, err := engine.Func(query, safe, lang, page)
+	// Calculate the specific page number for the selected engine
-		if err != nil {
+	enginePage := (page-1)/engineCount + 1
-			printWarn("Error performing image search with %s: %v", engine.Name, err)
-			continue
-		}
 
+	// Debug print to verify the selected engine and page
+	printInfo("Fetching image results for overall page %d using engine: %s (engine page %d)", page, engine.Name, enginePage)
+
+	// Fetch results from the selected engine
+	searchResults, _, err := engine.Func(query, safe, lang, enginePage)
+	if err != nil {
+		printWarn("Error performing image search with %s: %v", engine.Name, err)
+	} else {
 		for _, result := range searchResults {
 			imageResult := result.(ImageSearchResult)
-			if config.HardCacheEnabled == true {
+			if config.HardCacheEnabled {
-
+				// Generate hash and set up caching
-				// Generate hash from the original full-size image URL
 				hasher := md5.New()
 				hasher.Write([]byte(imageResult.Full))
 				hash := hex.EncodeToString(hasher.Sum(nil))
 				filename := hash + ".webp"
-
-				// Assign the ID
 				imageResult.ID = hash
-
-				// Set the ProxyFull URL
 				imageResult.ProxyFull = "/image_cache/" + filename
 
 				// Start caching and validation in the background
@@ -123,28 +125,68 @@ func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
 						printWarn("Failed to cache image %s: %v", originalURL, err)
 					}
 					if !success {
-						// Remove the image result from the cache
 						removeImageResultFromCache(query, page, safeBool, lang, imgResult.ID)
 					}
 				}(imageResult, imageResult.Full, filename)
-
 			} else {
-				// When hard cache is not enabled, use the imgproxy URLs
+				// Use proxied URLs when hard cache is disabled
-				imageResult.ProxyThumb = "/imgproxy?url=" + imageResult.Thumb // Proxied thumbnail
+				imageResult.ProxyThumb = "/imgproxy?url=" + imageResult.Thumb
-				imageResult.ProxyFull = "/imgproxy?url=" + imageResult.Full   // Proxied full-size image
+				imageResult.ProxyFull = "/imgproxy?url=" + imageResult.Full
 			}
 			results = append(results, imageResult)
 		}
+	}
 
-		if len(results) > 0 {
+	// Fallback mechanism if no results are fetched from the selected engine
-			break
+	if len(results) == 0 {
+		printWarn("No image results found with engine %s, trying other engines.", engine.Name)
+		for i := 1; i < engineCount; i++ {
+			nextEngine := imageSearchEngines[(engineIndex+i)%engineCount]
+			enginePage = (page-1)/engineCount + 1 // Recalculate page for next engine
+			printInfo("Trying next image search engine: %s (engine page %d)", nextEngine.Name, enginePage)
+
+			searchResults, _, err := nextEngine.Func(query, safe, lang, enginePage)
+			if err != nil {
+				printWarn("Error performing image search with %s: %v", nextEngine.Name, err)
+				continue
+			}
+			for _, result := range searchResults {
+				imageResult := result.(ImageSearchResult)
+				if config.HardCacheEnabled {
+					// Generate hash and set up caching
+					hasher := md5.New()
+					hasher.Write([]byte(imageResult.Full))
+					hash := hex.EncodeToString(hasher.Sum(nil))
+					filename := hash + ".webp"
+					imageResult.ID = hash
+					imageResult.ProxyFull = "/image_cache/" + filename
+
+					// Start caching and validation in the background
+					go func(imgResult ImageSearchResult, originalURL, filename string) {
+						_, success, err := cacheImage(originalURL, filename, imgResult.ID)
+						if err != nil {
+							printWarn("Failed to cache image %s: %v", originalURL, err)
+						}
+						if !success {
+							removeImageResultFromCache(query, page, safeBool, lang, imgResult.ID)
+						}
+					}(imageResult, imageResult.Full, filename)
+				} else {
+					// Use proxied URLs when hard cache is disabled
+					imageResult.ProxyThumb = "/imgproxy?url=" + imageResult.Thumb
+					imageResult.ProxyFull = "/imgproxy?url=" + imageResult.Full
+				}
+				results = append(results, imageResult)
+			}
+
+			if len(results) > 0 {
+				break
+			}
 		}
 	}
 
-	if len(results) == 0 {
+	// Final debug print to show the count of results fetched
-		printWarn("No image results found for query: %s, trying other nodes", query)
+	printInfo("Fetched %d image results for overall page %d", len(results), page)
-		results = tryOtherNodesForImageSearch(query, safe, lang, page, []string{hostID})
-	}
 
 	return results
 }

text.go

@@ -107,29 +107,47 @@ func prefetchPage(query, safe, lang string, page int) {
 
 func fetchTextResults(query, safe, lang string, page int) []TextSearchResult {
 	var results []TextSearchResult
+	engineCount := len(textSearchEngines)
 
-	for _, engine := range textSearchEngines {
+	// Determine which engine to use for the current page
-		printInfo("Using search engine: %s", engine.Name)
+	engineIndex := (page - 1) % engineCount
+	engine := textSearchEngines[engineIndex]
 
-		searchResults, _, err := engine.Func(query, safe, lang, page)
+	// Calculate the page number for this engine
-		if err != nil {
+	enginePage := (page-1)/engineCount + 1
-			printWarn("Error performing search with %s: %v", engine.Name, err)
-			continue
-		}
 
+	// Debug print to verify engine and page number being fetched
+	printInfo("Fetching results for overall page %d using engine: %s (engine page %d)", page, engine.Name, enginePage)
+
+	// Fetch results from the selected engine
+	searchResults, _, err := engine.Func(query, safe, lang, enginePage)
+	if err != nil {
+		printWarn("Error performing search with %s: %v", engine.Name, err)
+	} else {
 		results = append(results, validateResults(searchResults)...)
+	}
 
-		// If results are found, break out of the loop
+	// If no results are found with the selected engine, try the next in line
-		if len(results) > 0 {
+	if len(results) == 0 {
-			break
+		for i := 1; i < engineCount; i++ {
+			nextEngine := textSearchEngines[(engineIndex+i)%engineCount]
+			enginePage = (page-1)/engineCount + 1 // Recalculate for the new engine
+			printInfo("No results found, trying next engine: %s (engine page %d)", nextEngine.Name, enginePage)
+
+			searchResults, _, err := nextEngine.Func(query, safe, lang, enginePage)
+			if err != nil {
+				printWarn("Error performing search with %s: %v", nextEngine.Name, err)
+				continue
+			}
+			results = append(results, validateResults(searchResults)...)
+			if len(results) > 0 {
+				break
+			}
 		}
 	}
 
-	// If no results found after trying all engines
+	// Final debug print to display results count and source
-	if len(results) == 0 {
+	printInfo("Fetched %d results for overall page %d", len(results), page)
-		printWarn("No text results found for query: %s, trying other nodes", query)
-		results = tryOtherNodesForTextSearch(query, safe, lang, page, []string{hostID})
-	}
 
 	return results
 }