Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues 8 Pull requests Projects Releases Packages Wiki Activity Actions

fix for pages

Browse source
This commit is contained in:
partisan 2024-06-09 21:44:49 +02:00
parent 6885983576
commit a86b370f69
5 changed files with 99 additions and 157 deletions
Download patch file Download diff file Expand all files Collapse all files

174
text.go
View file

@ -1,44 +1,59 @@
package main
import (
"flag"
"fmt"
"html/template"
"log"
"math/rand"
"net/http"
"sort"
"sync"
"time"
)
var (
debugMode bool
debugMode bool
searchEngines []searchEngine
searchEngineLock sync.Mutex
)
type searchEngine struct {
Name string
Func func(string, string, string, int) ([]TextSearchResult, error)
Weight int
}
func init() {
flag.BoolVar(&debugMode, "debug", false, "enable debug mode")
flag.Parse()
debugMode = false
searchEngines = []searchEngine{
{Name: "Google", Func: PerformGoogleTextSearch, Weight: 1},
{Name: "LibreX", Func: PerformLibreXTextSearch, Weight: 2},
// {Name: "DuckDuckGo", Func: PerformDuckDuckGoTextSearch, Weight: 3}, // DuckDuckGo timeouts too fast and search results are trash
// {Name: "SearXNG", Func: PerformSearXNGTextSearch, Weight: 2}, // Uncomment when implemented
}
rand.Seed(time.Now().UnixNano())
}
func HandleTextSearch(w http.ResponseWriter, query, safe, lang string, page int) {
startTime := time.Now()
const resultsPerPage = 10
cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "true", Lang: lang, Type: "text"}
combinedResults := getTextResultsFromCacheOrFetch(cacheKey, query, safe, lang, page, resultsPerPage)
combinedResults := getTextResultsFromCacheOrFetch(cacheKey, query, safe, lang, page)
hasPrevPage := page > 1
hasNextPage := len(combinedResults) == resultsPerPage
hasNextPage := len(combinedResults) > 0
displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds(), page, hasPrevPage, hasNextPage)
// Always check and cache the next page if not enough results
if hasNextPage {
go cacheNextPageIfNotCached(query, safe, lang, page+1, resultsPerPage)
// Prefetch next and previous pages
go prefetchPage(query, safe, lang, page+1)
if hasPrevPage {
go prefetchPage(query, safe, lang, page-1)
}
}
func getTextResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page, resultsPerPage int) []TextSearchResult {
func getTextResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page int) []TextSearchResult {
cacheChan := make(chan []SearchResult)
var combinedResults []TextSearchResult
@ -56,7 +71,7 @@ func getTextResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string,
select {
case results := <-cacheChan:
if results == nil {
combinedResults = fetchTextResultsUntilFull(query, safe, lang, page, resultsPerPage)
combinedResults = fetchTextResults(query, safe, lang, page)
resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
} else {
textResults, _, _ := convertToSpecificResults(results)
@ -64,129 +79,70 @@ func getTextResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string,
}
case <-time.After(2 * time.Second):
log.Println("Cache check timeout")
combinedResults = fetchTextResultsUntilFull(query, safe, lang, page, resultsPerPage)
combinedResults = fetchTextResults(query, safe, lang, page)
resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
}
return combinedResults
}
func cacheNextPageIfNotCached(query, safe, lang string, page, resultsPerPage int) {
func prefetchPage(query, safe, lang string, page int) {
cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "true", Lang: lang, Type: "text"}
if _, exists := resultsCache.Get(cacheKey); !exists {
log.Printf("Next page %d not cached, caching now...", page)
nextPageResults := fetchTextResultsUntilFull(query, safe, lang, page, resultsPerPage)
resultsCache.Set(cacheKey, convertToSearchResults(nextPageResults))
log.Printf("Page %d not cached, caching now...", page)
pageResults := fetchTextResults(query, safe, lang, page)
resultsCache.Set(cacheKey, convertToSearchResults(pageResults))
} else {
log.Printf("Next page %d already cached", page)
log.Printf("Page %d already cached", page)
}
}
func fetchTextResultsUntilFull(query, safe, lang string, targetPage, resultsPerPage int) []TextSearchResult {
var combinedResults []TextSearchResult
currentPage := 1
resultsNeeded := targetPage * resultsPerPage
func fetchTextResults(query, safe, lang string, page int) []TextSearchResult {
engine := selectSearchEngine()
log.Printf("Using search engine: %s", engine.Name)
for len(combinedResults) < resultsNeeded {
cacheKey := CacheKey{Query: query, Page: targetPage, Safe: safe == "true", Lang: lang, Type: "text"}
cachedResults, exists := resultsCache.Get(cacheKey)
if exists {
textResults, _, _ := convertToSpecificResults(cachedResults)
combinedResults = append(combinedResults, textResults...)
} else {
results := fetchAndCacheTextResults(query, safe, lang, currentPage, resultsPerPage)
if len(results) == 0 {
break
}
combinedResults = append(combinedResults, results...)
resultsCache.Set(cacheKey, convertToSearchResults(results))
}
currentPage++
// Stop fetching if we have enough results for the target page and the next page
if len(combinedResults) >= resultsNeeded+resultsPerPage {
break
}
results, err := engine.Func(query, safe, lang, page)
if err != nil {
log.Printf("Error performing search with %s: %v", engine.Name, err)
return nil
}
startIndex := (targetPage - 1) * resultsPerPage
endIndex := startIndex + resultsPerPage
if startIndex >= len(combinedResults) {
return []TextSearchResult{}
}
if endIndex > len(combinedResults) {
endIndex = len(combinedResults)
}
return combinedResults[startIndex:endIndex]
return results
}
func fetchAndCacheTextResults(query, safe, lang string, page, resultsPerPage int) []TextSearchResult {
var combinedResults []TextSearchResult
var wg sync.WaitGroup
var mu sync.Mutex
func selectSearchEngine() searchEngine {
searchEngineLock.Lock()
defer searchEngineLock.Unlock()
resultsChan := make(chan []TextSearchResult)
searchFuncs := []struct {
Func func(string, string, string, int) ([]TextSearchResult, error)
Source string
}{
{PerformGoogleTextSearch, "Google"},
// {PerformLibreXTextSearch, "LibreX"},
// {PerformSearXNGTextSearch, "SearXNG"},
totalWeight := 0
for _, engine := range searchEngines {
totalWeight += engine.Weight
}
wg.Add(len(searchFuncs))
for _, searchFunc := range searchFuncs {
go func(searchFunc func(string, string, string, int) ([]TextSearchResult, error), source string) {
defer wg.Done()
results, err := searchFunc(query, safe, lang, page)
if err == nil {
for i := range results {
results[i].Source = source
randValue := rand.Intn(totalWeight)
for _, engine := range searchEngines {
if randValue < engine.Weight {
// Adjust weights for load balancing
for i := range searchEngines {
if searchEngines[i].Name == engine.Name {
searchEngines[i].Weight = max(1, searchEngines[i].Weight-1)
} else {
searchEngines[i].Weight++
}
resultsChan <- results
} else {
log.Printf("Error performing search from %s: %v", source, err)
}
}(searchFunc.Func, searchFunc.Source)
return engine
}
randValue -= engine.Weight
}
go func() {
wg.Wait()
close(resultsChan)
}()
for results := range resultsChan {
mu.Lock()
combinedResults = append(combinedResults, results...)
mu.Unlock()
}
sort.SliceStable(combinedResults, func(i, j int) bool {
return sourceOrder(combinedResults[i].Source) < sourceOrder(combinedResults[j].Source)
})
log.Printf("Fetched %d results for page %d", len(combinedResults), page)
return combinedResults
return searchEngines[0] // fallback to the first engine
}
func sourceOrder(source string) int {
switch source {
case "Google":
return 1
case "LibreX":
return 2
case "SearchXNG":
return 3
default:
return 4
func max(a, b int) int {
if a > b {
return a
}
return b
}
func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string, elapsed float64, page int, hasPrevPage, hasNextPage bool) {