Improve hot start by preselecting working video search instance
Some checks are pending
Run Integration Tests / test (push) Waiting to run

This commit is contained in:
partisan 2025-06-10 19:33:26 +02:00
parent 1ec633470b
commit 8f31f0b2eb
3 changed files with 77 additions and 18 deletions

View file

@ -392,6 +392,20 @@ func loadConfig() Config {
filesList := strings.Split(getConfigValueString(cfg.Section("MetaSearch").Key("Files"), strings.Join(defaultConfig.MetaSearch.Files, ",")), ",")
videoList := strings.Split(getConfigValueString(cfg.Section("MetaSearch").Key("Video"), strings.Join(defaultConfig.MetaSearch.Video, ",")), ",")
// Load default values for MetaSearch if they are empty
if len(textList) == 1 && textList[0] == "" {
textList = defaultConfig.MetaSearch.Text
}
if len(imageList) == 1 && imageList[0] == "" {
imageList = defaultConfig.MetaSearch.Image
}
if len(filesList) == 1 && filesList[0] == "" {
filesList = defaultConfig.MetaSearch.Files
}
if len(videoList) == 1 && videoList[0] == "" {
videoList = defaultConfig.MetaSearch.Video
}
// Indexing
concurrentStandardCrawlers := getConfigValue(cfg.Section("Indexer").Key("ConcurrentStandardCrawlers"), defaultConfig.ConcurrentStandardCrawlers, strconv.Atoi)
concurrentChromeCrawlers := getConfigValue(cfg.Section("Indexer").Key("ConcurrentChromeCrawlers"), defaultConfig.ConcurrentChromeCrawlers, strconv.Atoi)

View file

@ -135,10 +135,11 @@ func fetchImageResults(query, safe, lang string, page int, synchronous bool, thu
return results
}
if len(imageSearchEngines) == 0 {
printWarn("No image search engines configured in imageSearchEngines")
return nil
}
// This will not happen as during config load there is check to have at least something in search engine list
// if len(imageSearchEngines) == 0 {
// printWarn("No image search engines configured in imageSearchEngines")
// return nil
// }
engineCount := len(imageSearchEngines)

View file

@ -12,6 +12,7 @@ import (
const retryDuration = 12 * time.Hour // Retry duration for unresponding piped instances
var (
preferredInstance string
pipedInstances = []string{}
disabledInstances = make(map[string]bool)
mu sync.Mutex
@ -19,6 +20,8 @@ var (
func initPipedInstances() {
pipedInstances = config.MetaSearch.Video
go checkDisabledInstancesPeriodically()
go selectInitialPipedInstance()
}
// VideoAPIResponse matches the structure of the JSON response from the Piped API
@ -67,10 +70,6 @@ func formatDuration(seconds int) string {
return fmt.Sprintf("%02d:%02d", minutes, seconds)
}
func init() {
go checkDisabledInstancesPeriodically()
}
func checkDisabledInstancesPeriodically() {
checkAndReactivateInstances() // Initial immediate check
ticker := time.NewTicker(retryDuration)
@ -81,6 +80,28 @@ func checkDisabledInstancesPeriodically() {
}
}
func selectInitialPipedInstance() {
for _, inst := range pipedInstances {
if testInstanceAvailability(inst) {
mu.Lock()
preferredInstance = inst
mu.Unlock()
printInfo("Selected preferred piped instance: %s", inst)
return
}
}
printWarn("No available piped instances found during initial scan.")
}
func disableInstance(inst string) {
mu.Lock()
defer mu.Unlock()
disabledInstances[inst] = true
if preferredInstance == inst {
preferredInstance = ""
}
}
func checkAndReactivateInstances() {
mu.Lock()
defer mu.Unlock()
@ -107,32 +128,55 @@ func testInstanceAvailability(instance string) bool {
}
func makeHTMLRequest(query, safe, lang string, page int) (*VideoAPIResponse, error) {
mu.Lock()
instance := preferredInstance
mu.Unlock()
if instance != "" && !disabledInstances[instance] {
url := fmt.Sprintf("https://%s/search?q=%s&filter=all&safe=%s&lang=%s&page=%d",
instance, url.QueryEscape(query), safe, lang, page)
resp, err := http.Get(url)
if err == nil && resp.StatusCode == http.StatusOK {
defer resp.Body.Close()
var apiResp VideoAPIResponse
if err := json.NewDecoder(resp.Body).Decode(&apiResp); err == nil {
return &apiResp, nil
}
}
printWarn("Preferred instance %s failed, falling back to other instances", instance)
disableInstance(instance)
}
// Fallback loop
var lastError error
mu.Lock()
defer mu.Unlock()
for _, instance := range pipedInstances {
if disabledInstances[instance] {
continue // Skip this instance because it's still disabled
for _, inst := range pipedInstances {
if disabledInstances[inst] {
continue
}
url := fmt.Sprintf("https://%s/search?q=%s&filter=all&safe=%s&lang=%s&page=%d", instance, url.QueryEscape(query), safe, lang, page)
url := fmt.Sprintf("https://%s/search?q=%s&filter=all&safe=%s&lang=%s&page=%d",
inst, url.QueryEscape(query), safe, lang, page)
resp, err := http.Get(url)
if err != nil || resp.StatusCode != http.StatusOK {
printInfo("Disabling instance %s due to error or status code: %v", instance, err)
disabledInstances[instance] = true
lastError = fmt.Errorf("error making request to %s: %w", instance, err)
printInfo("Disabling instance %s due to error or status code: %v", inst, err)
disabledInstances[inst] = true
lastError = fmt.Errorf("error from %s: %w", inst, err)
continue
}
defer resp.Body.Close()
var apiResp VideoAPIResponse
if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
lastError = fmt.Errorf("error decoding response from %s: %w", instance, err)
lastError = fmt.Errorf("decode error from %s: %w", inst, err)
continue
}
// Store new preferred instance
preferredInstance = inst
return &apiResp, nil
}
return nil, fmt.Errorf("all instances failed, last error: %v", lastError)
}