automatic reputation for search engines

This commit is contained in:
partisan 2024-06-14 17:56:20 +02:00
parent dd9ed4cc53
commit e3d568f6cb
9 changed files with 198 additions and 126 deletions

View file

@ -4,30 +4,17 @@ import (
"fmt"
"html/template"
"log"
"math/rand"
"net/http"
"sync"
"time"
)
var (
imageEngines []imageEngine
imageEngineLock sync.Mutex
)
type imageEngine struct {
Name string
Func func(string, string, string, int) ([]ImageSearchResult, error)
Weight int
}
var imageSearchEngines []SearchEngine
func init() {
imageEngines = []imageEngine{
{Name: "Qwant", Func: PerformQwantImageSearch, Weight: 1},
{Name: "Imgur", Func: PerformImgurImageSearch, Weight: 2},
imageSearchEngines = []SearchEngine{
{Name: "Qwant", Func: wrapImageSearchFunc(PerformQwantImageSearch), Weight: 1},
{Name: "Imgur", Func: wrapImageSearchFunc(PerformImgurImageSearch), Weight: 2},
}
rand.Seed(time.Now().UnixNano())
}
func handleImageSearch(w http.ResponseWriter, query, safe, lang string, page int) {
@ -111,17 +98,24 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
var results []ImageSearchResult
var err error
var duration time.Duration
for attempts := 0; attempts < len(imageEngines); attempts++ {
engine := selectImageEngine()
for attempts := 0; attempts < len(imageSearchEngines); attempts++ {
engine := selectSearchEngine(imageSearchEngines)
log.Printf("Using image search engine: %s", engine.Name)
results, err = engine.Func(query, safe, lang, page)
var searchResults []SearchResult
searchResults, duration, err = engine.Func(query, safe, lang, page)
updateEngineMetrics(&engine, duration, err == nil)
if err != nil {
log.Printf("Error performing image search with %s: %v", engine.Name, err)
continue
}
for _, result := range searchResults {
results = append(results, result.(ImageSearchResult))
}
if len(results) > 0 {
break
}
@ -130,30 +124,16 @@ func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
return results
}
func selectImageEngine() imageEngine {
imageEngineLock.Lock()
defer imageEngineLock.Unlock()
totalWeight := 0
for _, engine := range imageEngines {
totalWeight += engine.Weight
}
randValue := rand.Intn(totalWeight)
for _, engine := range imageEngines {
if randValue < engine.Weight {
// Adjust weights for load balancing
for i := range imageEngines {
if imageEngines[i].Name == engine.Name {
imageEngines[i].Weight = max(1, imageEngines[i].Weight-1)
} else {
imageEngines[i].Weight++
}
}
return engine
func wrapImageSearchFunc(f func(string, string, string, int) ([]ImageSearchResult, time.Duration, error)) func(string, string, string, int) ([]SearchResult, time.Duration, error) {
return func(query, safe, lang string, page int) ([]SearchResult, time.Duration, error) {
imageResults, duration, err := f(query, safe, lang, page)
if err != nil {
return nil, duration, err
}
randValue -= engine.Weight
searchResults := make([]SearchResult, len(imageResults))
for i, result := range imageResults {
searchResults[i] = result
}
return searchResults, duration, nil
}
return imageEngines[0] // fallback to the first engine
}