Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues 5 Pull requests Projects Releases Packages Wiki Activity Actions

Changed self-crawling as experimental, cleanup unused features
Some checks failed
Run Integration Tests / test (push) Failing after 1m15s
Details

Browse source
This commit is contained in:
partisan 2025-06-08 22:12:15 +02:00
parent ca87df5df1
commit 49cb7bb94a
27 changed files with 1731 additions and 832 deletions
Download patch file Download diff file Expand all files Collapse all files

88
text.go
View file

@ -1,3 +1,6 @@
//go:build !experimental
// +build !experimental
package main
import (
@ -174,70 +177,47 @@ func prefetchPage(query, safe, lang string, page int) {
func fetchTextResults(query, safe, lang string, page int) []TextSearchResult {
var results []TextSearchResult
if !config.MetaSearchEnabled {
printDebug("Crawler is disabled; fetching from local index.")
// Crawler is enabled, so use the search engines
engineCount := len(textSearchEngines)
// Calculate the starting position based on the page number
indexedResults, err := SearchIndex(query, page, 10)
if err != nil {
printErr("Error searching the index: %v", err)
return results // Return empty results on error
}
// Determine which engine to use for the current page
engineIndex := (page - 1) % engineCount
engine := textSearchEngines[engineIndex]
// Convert indexed results to TextSearchResult format
for _, doc := range indexedResults {
results = append(results, TextSearchResult{
URL: doc.Link,
Header: doc.Title,
Description: doc.Description,
Source: doc.Tags,
})
}
// Calculate the page number for this engine
enginePage := (page-1)/engineCount + 1
return results
printDebug("Fetching results for overall page %d using engine: %s (engine page %d)", page, engine.Name, enginePage)
// Fetch results from the selected engine
searchResults, _, err := engine.Func(query, safe, lang, enginePage)
if err != nil {
printWarn("Error performing search with %s: %v", engine.Name, err)
} else {
// Crawler is enabled, so use the search engines
engineCount := len(textSearchEngines)
results = append(results, validateResults(searchResults)...)
}
// Determine which engine to use for the current page
engineIndex := (page - 1) % engineCount
engine := textSearchEngines[engineIndex]
// If no results are found with the selected engine, try the next in line
if len(results) == 0 {
for i := 1; i < engineCount; i++ {
nextEngine := textSearchEngines[(engineIndex+i)%engineCount]
enginePage = (page-1)/engineCount + 1
printInfo("No results found, trying next engine: %s (engine page %d)", nextEngine.Name, enginePage)
// Calculate the page number for this engine
enginePage := (page-1)/engineCount + 1
printDebug("Fetching results for overall page %d using engine: %s (engine page %d)", page, engine.Name, enginePage)
// Fetch results from the selected engine
searchResults, _, err := engine.Func(query, safe, lang, enginePage)
if err != nil {
printWarn("Error performing search with %s: %v", engine.Name, err)
} else {
searchResults, _, err := nextEngine.Func(query, safe, lang, enginePage)
if err != nil {
printWarn("Error performing search with %s: %v", nextEngine.Name, err)
continue
}
results = append(results, validateResults(searchResults)...)
}
// If no results are found with the selected engine, try the next in line
if len(results) == 0 {
for i := 1; i < engineCount; i++ {
nextEngine := textSearchEngines[(engineIndex+i)%engineCount]
enginePage = (page-1)/engineCount + 1
printInfo("No results found, trying next engine: %s (engine page %d)", nextEngine.Name, enginePage)
searchResults, _, err := nextEngine.Func(query, safe, lang, enginePage)
if err != nil {
printWarn("Error performing search with %s: %v", nextEngine.Name, err)
continue
}
results = append(results, validateResults(searchResults)...)
if len(results) > 0 {
break
}
if len(results) > 0 {
break
}
}
printInfo("Fetched %d results for overall page %d", len(results), page)
return results
}
printInfo("Fetched %d results for overall page %d", len(results), page)
return results
}
func validateResults(searchResults []SearchResult) []TextSearchResult {