Improve hot start by preselecting working video search instance
Some checks are pending
Run Integration Tests / test (push) Waiting to run
Some checks are pending
Run Integration Tests / test (push) Waiting to run
This commit is contained in:
parent
1ec633470b
commit
8f31f0b2eb
3 changed files with 77 additions and 18 deletions
14
config.go
14
config.go
|
@ -392,6 +392,20 @@ func loadConfig() Config {
|
||||||
filesList := strings.Split(getConfigValueString(cfg.Section("MetaSearch").Key("Files"), strings.Join(defaultConfig.MetaSearch.Files, ",")), ",")
|
filesList := strings.Split(getConfigValueString(cfg.Section("MetaSearch").Key("Files"), strings.Join(defaultConfig.MetaSearch.Files, ",")), ",")
|
||||||
videoList := strings.Split(getConfigValueString(cfg.Section("MetaSearch").Key("Video"), strings.Join(defaultConfig.MetaSearch.Video, ",")), ",")
|
videoList := strings.Split(getConfigValueString(cfg.Section("MetaSearch").Key("Video"), strings.Join(defaultConfig.MetaSearch.Video, ",")), ",")
|
||||||
|
|
||||||
|
// Load default values for MetaSearch if they are empty
|
||||||
|
if len(textList) == 1 && textList[0] == "" {
|
||||||
|
textList = defaultConfig.MetaSearch.Text
|
||||||
|
}
|
||||||
|
if len(imageList) == 1 && imageList[0] == "" {
|
||||||
|
imageList = defaultConfig.MetaSearch.Image
|
||||||
|
}
|
||||||
|
if len(filesList) == 1 && filesList[0] == "" {
|
||||||
|
filesList = defaultConfig.MetaSearch.Files
|
||||||
|
}
|
||||||
|
if len(videoList) == 1 && videoList[0] == "" {
|
||||||
|
videoList = defaultConfig.MetaSearch.Video
|
||||||
|
}
|
||||||
|
|
||||||
// Indexing
|
// Indexing
|
||||||
concurrentStandardCrawlers := getConfigValue(cfg.Section("Indexer").Key("ConcurrentStandardCrawlers"), defaultConfig.ConcurrentStandardCrawlers, strconv.Atoi)
|
concurrentStandardCrawlers := getConfigValue(cfg.Section("Indexer").Key("ConcurrentStandardCrawlers"), defaultConfig.ConcurrentStandardCrawlers, strconv.Atoi)
|
||||||
concurrentChromeCrawlers := getConfigValue(cfg.Section("Indexer").Key("ConcurrentChromeCrawlers"), defaultConfig.ConcurrentChromeCrawlers, strconv.Atoi)
|
concurrentChromeCrawlers := getConfigValue(cfg.Section("Indexer").Key("ConcurrentChromeCrawlers"), defaultConfig.ConcurrentChromeCrawlers, strconv.Atoi)
|
||||||
|
|
|
@ -135,10 +135,11 @@ func fetchImageResults(query, safe, lang string, page int, synchronous bool, thu
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(imageSearchEngines) == 0 {
|
// This will not happen as during config load there is check to have at least something in search engine list
|
||||||
printWarn("No image search engines configured in imageSearchEngines")
|
// if len(imageSearchEngines) == 0 {
|
||||||
return nil
|
// printWarn("No image search engines configured in imageSearchEngines")
|
||||||
}
|
// return nil
|
||||||
|
// }
|
||||||
|
|
||||||
engineCount := len(imageSearchEngines)
|
engineCount := len(imageSearchEngines)
|
||||||
|
|
||||||
|
|
72
video.go
72
video.go
|
@ -12,6 +12,7 @@ import (
|
||||||
const retryDuration = 12 * time.Hour // Retry duration for unresponding piped instances
|
const retryDuration = 12 * time.Hour // Retry duration for unresponding piped instances
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
preferredInstance string
|
||||||
pipedInstances = []string{}
|
pipedInstances = []string{}
|
||||||
disabledInstances = make(map[string]bool)
|
disabledInstances = make(map[string]bool)
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
|
@ -19,6 +20,8 @@ var (
|
||||||
|
|
||||||
func initPipedInstances() {
|
func initPipedInstances() {
|
||||||
pipedInstances = config.MetaSearch.Video
|
pipedInstances = config.MetaSearch.Video
|
||||||
|
go checkDisabledInstancesPeriodically()
|
||||||
|
go selectInitialPipedInstance()
|
||||||
}
|
}
|
||||||
|
|
||||||
// VideoAPIResponse matches the structure of the JSON response from the Piped API
|
// VideoAPIResponse matches the structure of the JSON response from the Piped API
|
||||||
|
@ -67,10 +70,6 @@ func formatDuration(seconds int) string {
|
||||||
return fmt.Sprintf("%02d:%02d", minutes, seconds)
|
return fmt.Sprintf("%02d:%02d", minutes, seconds)
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
|
||||||
go checkDisabledInstancesPeriodically()
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkDisabledInstancesPeriodically() {
|
func checkDisabledInstancesPeriodically() {
|
||||||
checkAndReactivateInstances() // Initial immediate check
|
checkAndReactivateInstances() // Initial immediate check
|
||||||
ticker := time.NewTicker(retryDuration)
|
ticker := time.NewTicker(retryDuration)
|
||||||
|
@ -81,6 +80,28 @@ func checkDisabledInstancesPeriodically() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func selectInitialPipedInstance() {
|
||||||
|
for _, inst := range pipedInstances {
|
||||||
|
if testInstanceAvailability(inst) {
|
||||||
|
mu.Lock()
|
||||||
|
preferredInstance = inst
|
||||||
|
mu.Unlock()
|
||||||
|
printInfo("Selected preferred piped instance: %s", inst)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
printWarn("No available piped instances found during initial scan.")
|
||||||
|
}
|
||||||
|
|
||||||
|
func disableInstance(inst string) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
disabledInstances[inst] = true
|
||||||
|
if preferredInstance == inst {
|
||||||
|
preferredInstance = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func checkAndReactivateInstances() {
|
func checkAndReactivateInstances() {
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
defer mu.Unlock()
|
defer mu.Unlock()
|
||||||
|
@ -107,32 +128,55 @@ func testInstanceAvailability(instance string) bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func makeHTMLRequest(query, safe, lang string, page int) (*VideoAPIResponse, error) {
|
func makeHTMLRequest(query, safe, lang string, page int) (*VideoAPIResponse, error) {
|
||||||
|
mu.Lock()
|
||||||
|
instance := preferredInstance
|
||||||
|
mu.Unlock()
|
||||||
|
|
||||||
|
if instance != "" && !disabledInstances[instance] {
|
||||||
|
url := fmt.Sprintf("https://%s/search?q=%s&filter=all&safe=%s&lang=%s&page=%d",
|
||||||
|
instance, url.QueryEscape(query), safe, lang, page)
|
||||||
|
resp, err := http.Get(url)
|
||||||
|
if err == nil && resp.StatusCode == http.StatusOK {
|
||||||
|
defer resp.Body.Close()
|
||||||
|
var apiResp VideoAPIResponse
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&apiResp); err == nil {
|
||||||
|
return &apiResp, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
printWarn("Preferred instance %s failed, falling back to other instances", instance)
|
||||||
|
disableInstance(instance)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback loop
|
||||||
var lastError error
|
var lastError error
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
defer mu.Unlock()
|
defer mu.Unlock()
|
||||||
|
for _, inst := range pipedInstances {
|
||||||
for _, instance := range pipedInstances {
|
if disabledInstances[inst] {
|
||||||
if disabledInstances[instance] {
|
continue
|
||||||
continue // Skip this instance because it's still disabled
|
|
||||||
}
|
}
|
||||||
|
url := fmt.Sprintf("https://%s/search?q=%s&filter=all&safe=%s&lang=%s&page=%d",
|
||||||
url := fmt.Sprintf("https://%s/search?q=%s&filter=all&safe=%s&lang=%s&page=%d", instance, url.QueryEscape(query), safe, lang, page)
|
inst, url.QueryEscape(query), safe, lang, page)
|
||||||
resp, err := http.Get(url)
|
resp, err := http.Get(url)
|
||||||
if err != nil || resp.StatusCode != http.StatusOK {
|
if err != nil || resp.StatusCode != http.StatusOK {
|
||||||
printInfo("Disabling instance %s due to error or status code: %v", instance, err)
|
printInfo("Disabling instance %s due to error or status code: %v", inst, err)
|
||||||
disabledInstances[instance] = true
|
disabledInstances[inst] = true
|
||||||
lastError = fmt.Errorf("error making request to %s: %w", instance, err)
|
lastError = fmt.Errorf("error from %s: %w", inst, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
var apiResp VideoAPIResponse
|
var apiResp VideoAPIResponse
|
||||||
if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
|
if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
|
||||||
lastError = fmt.Errorf("error decoding response from %s: %w", instance, err)
|
lastError = fmt.Errorf("decode error from %s: %w", inst, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Store new preferred instance
|
||||||
|
preferredInstance = inst
|
||||||
return &apiResp, nil
|
return &apiResp, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, fmt.Errorf("all instances failed, last error: %v", lastError)
|
return nil, fmt.Errorf("all instances failed, last error: %v", lastError)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue