automatic reputation for search engines
This commit is contained in:
parent
dd9ed4cc53
commit
e3d568f6cb
9 changed files with 198 additions and 126 deletions
68
images.go
68
images.go
|
@ -4,30 +4,17 @@ import (
|
|||
"fmt"
|
||||
"html/template"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
imageEngines []imageEngine
|
||||
imageEngineLock sync.Mutex
|
||||
)
|
||||
|
||||
type imageEngine struct {
|
||||
Name string
|
||||
Func func(string, string, string, int) ([]ImageSearchResult, error)
|
||||
Weight int
|
||||
}
|
||||
var imageSearchEngines []SearchEngine
|
||||
|
||||
func init() {
|
||||
imageEngines = []imageEngine{
|
||||
{Name: "Qwant", Func: PerformQwantImageSearch, Weight: 1},
|
||||
{Name: "Imgur", Func: PerformImgurImageSearch, Weight: 2},
|
||||
imageSearchEngines = []SearchEngine{
|
||||
{Name: "Qwant", Func: wrapImageSearchFunc(PerformQwantImageSearch), Weight: 1},
|
||||
{Name: "Imgur", Func: wrapImageSearchFunc(PerformImgurImageSearch), Weight: 2},
|
||||
}
|
||||
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func handleImageSearch(w http.ResponseWriter, query, safe, lang string, page int) {
|
||||
|
@ -111,17 +98,24 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
|
|||
func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
|
||||
var results []ImageSearchResult
|
||||
var err error
|
||||
var duration time.Duration
|
||||
|
||||
for attempts := 0; attempts < len(imageEngines); attempts++ {
|
||||
engine := selectImageEngine()
|
||||
for attempts := 0; attempts < len(imageSearchEngines); attempts++ {
|
||||
engine := selectSearchEngine(imageSearchEngines)
|
||||
log.Printf("Using image search engine: %s", engine.Name)
|
||||
|
||||
results, err = engine.Func(query, safe, lang, page)
|
||||
var searchResults []SearchResult
|
||||
searchResults, duration, err = engine.Func(query, safe, lang, page)
|
||||
updateEngineMetrics(&engine, duration, err == nil)
|
||||
if err != nil {
|
||||
log.Printf("Error performing image search with %s: %v", engine.Name, err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, result := range searchResults {
|
||||
results = append(results, result.(ImageSearchResult))
|
||||
}
|
||||
|
||||
if len(results) > 0 {
|
||||
break
|
||||
}
|
||||
|
@ -130,30 +124,16 @@ func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
|
|||
return results
|
||||
}
|
||||
|
||||
func selectImageEngine() imageEngine {
|
||||
imageEngineLock.Lock()
|
||||
defer imageEngineLock.Unlock()
|
||||
|
||||
totalWeight := 0
|
||||
for _, engine := range imageEngines {
|
||||
totalWeight += engine.Weight
|
||||
}
|
||||
|
||||
randValue := rand.Intn(totalWeight)
|
||||
for _, engine := range imageEngines {
|
||||
if randValue < engine.Weight {
|
||||
// Adjust weights for load balancing
|
||||
for i := range imageEngines {
|
||||
if imageEngines[i].Name == engine.Name {
|
||||
imageEngines[i].Weight = max(1, imageEngines[i].Weight-1)
|
||||
} else {
|
||||
imageEngines[i].Weight++
|
||||
}
|
||||
}
|
||||
return engine
|
||||
func wrapImageSearchFunc(f func(string, string, string, int) ([]ImageSearchResult, time.Duration, error)) func(string, string, string, int) ([]SearchResult, time.Duration, error) {
|
||||
return func(query, safe, lang string, page int) ([]SearchResult, time.Duration, error) {
|
||||
imageResults, duration, err := f(query, safe, lang, page)
|
||||
if err != nil {
|
||||
return nil, duration, err
|
||||
}
|
||||
randValue -= engine.Weight
|
||||
searchResults := make([]SearchResult, len(imageResults))
|
||||
for i, result := range imageResults {
|
||||
searchResults[i] = result
|
||||
}
|
||||
return searchResults, duration, nil
|
||||
}
|
||||
|
||||
return imageEngines[0] // fallback to the first engine
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue