Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues 8 Pull requests Projects Releases Packages Wiki Activity Actions

added ProxyRetry to config and fixed ProxyStrict
Some checks failed
Run Integration Tests / test (push) Failing after 50s
Details

Browse source
This commit is contained in:
partisan 2025-02-22 22:36:54 +01:00
parent ab707a91e8
commit 35e657bccd
17 changed files with 224 additions and 186 deletions
Download patch file Download diff file Expand all files Collapse all files

10
config.go
View file

@ -43,10 +43,12 @@ type Config struct {
DriveCacheEnabled bool
MetaProxyEnabled bool
MetaProxyStrict bool
MetaProxyRetry int
MetaProxies []string
CrawlerProxyEnabled bool
CrawlerProxyStrict bool
CrawlerProxies []string
CrawlerProxyRetry int
// Maybye add Proxy support for Image Extraction?
LogLevel int
ConcurrentStandardCrawlers int
@ -76,9 +78,11 @@ var defaultConfig = Config{
MetaProxyEnabled: false,
MetaProxyStrict: true,
MetaProxies: []string{},
MetaProxyRetry: 3,
CrawlerProxyEnabled: false,
CrawlerProxyStrict: true,
CrawlerProxies: []string{},
CrawlerProxyRetry: 1,
ConcurrentStandardCrawlers: 12,
ConcurrentChromeCrawlers: 4,
CrawlingInterval: 24 * time.Hour,
@ -307,6 +311,8 @@ func saveConfig(config Config) {
proxiesSec.Key("MetaProxies").SetValue(strings.Join(config.MetaProxies, ","))
proxiesSec.Key("CrawlerProxyStrict").SetValue(strconv.FormatBool(config.CrawlerProxyStrict))
proxiesSec.Key("CrawlerProxies").SetValue(strings.Join(config.CrawlerProxies, ","))
proxiesSec.Key("MetaProxyRetry").SetValue(strconv.Itoa(config.MetaProxyRetry))
proxiesSec.Key("CrawlerProxyRetry").SetValue(strconv.Itoa(config.CrawlerProxyRetry))
// MetaSearch section
metaSec := cfg.Section("MetaSearches")
@ -372,6 +378,8 @@ func loadConfig() Config {
metaProxies := strings.Split(getConfigValueString(cfg.Section("Proxies").Key("MetaProxies"), ""), ",")
crawlerProxyStrict := getConfigValueBool(cfg.Section("Proxies").Key("CrawlerProxyStrict"), defaultConfig.CrawlerProxyStrict)
crawlerProxies := strings.Split(getConfigValueString(cfg.Section("Proxies").Key("CrawlerProxies"), ""), ",")
metaProxyRetry := getConfigValue(cfg.Section("Proxies").Key("MetaProxyRetry"), defaultConfig.MetaProxyRetry, strconv.Atoi)
crawlerProxyRetry := getConfigValue(cfg.Section("Proxies").Key("CrawlerProxyRetry"), defaultConfig.CrawlerProxyRetry, strconv.Atoi)
// MetaSearch
searchXInstances := strings.Split(getConfigValueString(cfg.Section("MetaSearches").Key("LibreXInstances"), strings.Join(defaultConfig.LibreXInstances, ",")), ",")
@ -415,9 +423,11 @@ func loadConfig() Config {
MetaProxyEnabled: metaProxyEnabled,
MetaProxyStrict: metaProxyStrict,
MetaProxies: metaProxies,
MetaProxyRetry: metaProxyRetry,
CrawlerProxyEnabled: crawlerProxyEnabled,
CrawlerProxyStrict: crawlerProxyStrict,
CrawlerProxies: crawlerProxies,
CrawlerProxyRetry: crawlerProxyRetry,
ConcurrentStandardCrawlers: concurrentStandardCrawlers,
ConcurrentChromeCrawlers: concurrentChromeCrawlers,
CrawlingInterval: crawlingInterval,