Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues 8 Pull requests Projects Releases Packages Wiki Activity Actions

caching for "files" and images + moved search result types

Browse source
This commit is contained in:
partisan 2024-05-24 14:07:16 +02:00
parent 5a66f61a4c
commit 7668ee5eca
7 changed files with 253 additions and 124 deletions
Download patch file Download diff file Expand all files Collapse all files

45
text.go
View file

@ -12,8 +12,7 @@ import (
)
var (
debugMode bool
resultsCache = NewResultsCache(6 * time.Hour) // Cache with 6-hour expiration
debugMode bool
)
func init() {
@ -25,8 +24,8 @@ func HandleTextSearch(w http.ResponseWriter, query, safe, lang string, page int)
startTime := time.Now()
const resultsPerPage = 10
cacheKey := CacheKey{Query: query, Page: page, Safe: safe, Lang: lang}
combinedResults := getResultsFromCacheOrFetch(cacheKey, query, safe, lang, page, resultsPerPage)
cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "true", Lang: lang, Type: "text"}
combinedResults := getTextResultsFromCacheOrFetch(cacheKey, query, safe, lang, page, resultsPerPage)
hasPrevPage := page > 1
hasNextPage := len(combinedResults) == resultsPerPage
@ -39,8 +38,8 @@ func HandleTextSearch(w http.ResponseWriter, query, safe, lang string, page int)
}
}
func getResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page, resultsPerPage int) []TextSearchResult {
cacheChan := make(chan []TextSearchResult)
func getTextResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page, resultsPerPage int) []TextSearchResult {
cacheChan := make(chan []SearchResult)
var combinedResults []TextSearchResult
go func() {
@ -55,48 +54,52 @@ func getResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, pag
}()
select {
case combinedResults = <-cacheChan:
if combinedResults == nil {
combinedResults = fetchResultsUntilFull(query, safe, lang, page, resultsPerPage)
resultsCache.Set(cacheKey, combinedResults)
case results := <-cacheChan:
if results == nil {
combinedResults = fetchTextResultsUntilFull(query, safe, lang, page, resultsPerPage)
resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
} else {
textResults, _, _ := convertToSpecificResults(results)
combinedResults = textResults
}
case <-time.After(2 * time.Second):
log.Println("Cache check timeout")
combinedResults = fetchResultsUntilFull(query, safe, lang, page, resultsPerPage)
resultsCache.Set(cacheKey, combinedResults)
combinedResults = fetchTextResultsUntilFull(query, safe, lang, page, resultsPerPage)
resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
}
return combinedResults
}
func cacheNextPageIfNotCached(query, safe, lang string, page, resultsPerPage int) {
cacheKey := CacheKey{Query: query, Page: page, Safe: safe, Lang: lang}
cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "true", Lang: lang, Type: "text"}
if _, exists := resultsCache.Get(cacheKey); !exists {
log.Printf("Next page %d not cached, caching now...", page)
nextPageResults := fetchResultsUntilFull(query, safe, lang, page, resultsPerPage)
resultsCache.Set(cacheKey, nextPageResults)
nextPageResults := fetchTextResultsUntilFull(query, safe, lang, page, resultsPerPage)
resultsCache.Set(cacheKey, convertToSearchResults(nextPageResults))
} else {
log.Printf("Next page %d already cached", page)
}
}
func fetchResultsUntilFull(query, safe, lang string, targetPage, resultsPerPage int) []TextSearchResult {
func fetchTextResultsUntilFull(query, safe, lang string, targetPage, resultsPerPage int) []TextSearchResult {
var combinedResults []TextSearchResult
currentPage := 1
resultsNeeded := targetPage * resultsPerPage
for len(combinedResults) < resultsNeeded {
cacheKey := CacheKey{Query: query, Page: currentPage, Safe: safe, Lang: lang}
cacheKey := CacheKey{Query: query, Page: targetPage, Safe: safe == "true", Lang: lang, Type: "text"}
cachedResults, exists := resultsCache.Get(cacheKey)
if exists {
combinedResults = append(combinedResults, cachedResults...)
textResults, _, _ := convertToSpecificResults(cachedResults)
combinedResults = append(combinedResults, textResults...)
} else {
results := fetchAndCacheResults(query, safe, lang, currentPage, resultsPerPage)
results := fetchAndCacheTextResults(query, safe, lang, currentPage, resultsPerPage)
if len(results) == 0 {
break
}
combinedResults = append(combinedResults, results...)
resultsCache.Set(cacheKey, results)
resultsCache.Set(cacheKey, convertToSearchResults(results))
}
currentPage++
@ -120,7 +123,7 @@ func fetchResultsUntilFull(query, safe, lang string, targetPage, resultsPerPage
return combinedResults[startIndex:endIndex]
}
func fetchAndCacheResults(query, safe, lang string, page, resultsPerPage int) []TextSearchResult {
func fetchAndCacheTextResults(query, safe, lang string, page, resultsPerPage int) []TextSearchResult {
var combinedResults []TextSearchResult
var wg sync.WaitGroup
var mu sync.Mutex