Improve hot start by preselecting working video search instance
Some checks are pending
Run Integration Tests / test (push) Waiting to run
Some checks are pending
Run Integration Tests / test (push) Waiting to run
This commit is contained in:
parent
1ec633470b
commit
8f31f0b2eb
3 changed files with 77 additions and 18 deletions
14
config.go
14
config.go
|
@ -392,6 +392,20 @@ func loadConfig() Config {
|
|||
filesList := strings.Split(getConfigValueString(cfg.Section("MetaSearch").Key("Files"), strings.Join(defaultConfig.MetaSearch.Files, ",")), ",")
|
||||
videoList := strings.Split(getConfigValueString(cfg.Section("MetaSearch").Key("Video"), strings.Join(defaultConfig.MetaSearch.Video, ",")), ",")
|
||||
|
||||
// Load default values for MetaSearch if they are empty
|
||||
if len(textList) == 1 && textList[0] == "" {
|
||||
textList = defaultConfig.MetaSearch.Text
|
||||
}
|
||||
if len(imageList) == 1 && imageList[0] == "" {
|
||||
imageList = defaultConfig.MetaSearch.Image
|
||||
}
|
||||
if len(filesList) == 1 && filesList[0] == "" {
|
||||
filesList = defaultConfig.MetaSearch.Files
|
||||
}
|
||||
if len(videoList) == 1 && videoList[0] == "" {
|
||||
videoList = defaultConfig.MetaSearch.Video
|
||||
}
|
||||
|
||||
// Indexing
|
||||
concurrentStandardCrawlers := getConfigValue(cfg.Section("Indexer").Key("ConcurrentStandardCrawlers"), defaultConfig.ConcurrentStandardCrawlers, strconv.Atoi)
|
||||
concurrentChromeCrawlers := getConfigValue(cfg.Section("Indexer").Key("ConcurrentChromeCrawlers"), defaultConfig.ConcurrentChromeCrawlers, strconv.Atoi)
|
||||
|
|
|
@ -135,10 +135,11 @@ func fetchImageResults(query, safe, lang string, page int, synchronous bool, thu
|
|||
return results
|
||||
}
|
||||
|
||||
if len(imageSearchEngines) == 0 {
|
||||
printWarn("No image search engines configured in imageSearchEngines")
|
||||
return nil
|
||||
}
|
||||
// This will not happen as during config load there is check to have at least something in search engine list
|
||||
// if len(imageSearchEngines) == 0 {
|
||||
// printWarn("No image search engines configured in imageSearchEngines")
|
||||
// return nil
|
||||
// }
|
||||
|
||||
engineCount := len(imageSearchEngines)
|
||||
|
||||
|
|
72
video.go
72
video.go
|
@ -12,6 +12,7 @@ import (
|
|||
const retryDuration = 12 * time.Hour // Retry duration for unresponding piped instances
|
||||
|
||||
var (
|
||||
preferredInstance string
|
||||
pipedInstances = []string{}
|
||||
disabledInstances = make(map[string]bool)
|
||||
mu sync.Mutex
|
||||
|
@ -19,6 +20,8 @@ var (
|
|||
|
||||
func initPipedInstances() {
|
||||
pipedInstances = config.MetaSearch.Video
|
||||
go checkDisabledInstancesPeriodically()
|
||||
go selectInitialPipedInstance()
|
||||
}
|
||||
|
||||
// VideoAPIResponse matches the structure of the JSON response from the Piped API
|
||||
|
@ -67,10 +70,6 @@ func formatDuration(seconds int) string {
|
|||
return fmt.Sprintf("%02d:%02d", minutes, seconds)
|
||||
}
|
||||
|
||||
func init() {
|
||||
go checkDisabledInstancesPeriodically()
|
||||
}
|
||||
|
||||
func checkDisabledInstancesPeriodically() {
|
||||
checkAndReactivateInstances() // Initial immediate check
|
||||
ticker := time.NewTicker(retryDuration)
|
||||
|
@ -81,6 +80,28 @@ func checkDisabledInstancesPeriodically() {
|
|||
}
|
||||
}
|
||||
|
||||
func selectInitialPipedInstance() {
|
||||
for _, inst := range pipedInstances {
|
||||
if testInstanceAvailability(inst) {
|
||||
mu.Lock()
|
||||
preferredInstance = inst
|
||||
mu.Unlock()
|
||||
printInfo("Selected preferred piped instance: %s", inst)
|
||||
return
|
||||
}
|
||||
}
|
||||
printWarn("No available piped instances found during initial scan.")
|
||||
}
|
||||
|
||||
func disableInstance(inst string) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
disabledInstances[inst] = true
|
||||
if preferredInstance == inst {
|
||||
preferredInstance = ""
|
||||
}
|
||||
}
|
||||
|
||||
func checkAndReactivateInstances() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
@ -107,32 +128,55 @@ func testInstanceAvailability(instance string) bool {
|
|||
}
|
||||
|
||||
func makeHTMLRequest(query, safe, lang string, page int) (*VideoAPIResponse, error) {
|
||||
mu.Lock()
|
||||
instance := preferredInstance
|
||||
mu.Unlock()
|
||||
|
||||
if instance != "" && !disabledInstances[instance] {
|
||||
url := fmt.Sprintf("https://%s/search?q=%s&filter=all&safe=%s&lang=%s&page=%d",
|
||||
instance, url.QueryEscape(query), safe, lang, page)
|
||||
resp, err := http.Get(url)
|
||||
if err == nil && resp.StatusCode == http.StatusOK {
|
||||
defer resp.Body.Close()
|
||||
var apiResp VideoAPIResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&apiResp); err == nil {
|
||||
return &apiResp, nil
|
||||
}
|
||||
}
|
||||
printWarn("Preferred instance %s failed, falling back to other instances", instance)
|
||||
disableInstance(instance)
|
||||
}
|
||||
|
||||
// Fallback loop
|
||||
var lastError error
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
for _, instance := range pipedInstances {
|
||||
if disabledInstances[instance] {
|
||||
continue // Skip this instance because it's still disabled
|
||||
for _, inst := range pipedInstances {
|
||||
if disabledInstances[inst] {
|
||||
continue
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("https://%s/search?q=%s&filter=all&safe=%s&lang=%s&page=%d", instance, url.QueryEscape(query), safe, lang, page)
|
||||
url := fmt.Sprintf("https://%s/search?q=%s&filter=all&safe=%s&lang=%s&page=%d",
|
||||
inst, url.QueryEscape(query), safe, lang, page)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil || resp.StatusCode != http.StatusOK {
|
||||
printInfo("Disabling instance %s due to error or status code: %v", instance, err)
|
||||
disabledInstances[instance] = true
|
||||
lastError = fmt.Errorf("error making request to %s: %w", instance, err)
|
||||
printInfo("Disabling instance %s due to error or status code: %v", inst, err)
|
||||
disabledInstances[inst] = true
|
||||
lastError = fmt.Errorf("error from %s: %w", inst, err)
|
||||
continue
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
var apiResp VideoAPIResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
|
||||
lastError = fmt.Errorf("error decoding response from %s: %w", instance, err)
|
||||
lastError = fmt.Errorf("decode error from %s: %w", inst, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Store new preferred instance
|
||||
preferredInstance = inst
|
||||
return &apiResp, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("all instances failed, last error: %v", lastError)
|
||||
}
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue