README.md

@@ -47,11 +47,11 @@ A self-hosted private search engine designed to be scalable and more resource-ef
 
 ### For Self-Hosting
 
-- **[Easy to Set Up](https://weforge.xyz/Spitfire/Search#running-the-qgato)** - Quick and straightforward setup process for anyone.  
+- **Self-hosted option** - Run on your own server for even more privacy.
 - **Lightweight** - Low memory footprint (15-30MiB) even during searches.
 - **Decentralized** - No single point of failure.
 - **Results caching in RAM** - Faster response times through caching.
-- **[Configurable](https://weforge.xyz/Spitfire/Search/wiki/Configuration)** - Fully customizable via the `config.ini` file.
+- **Configurable** - Tweak features via `config.ini`.
 - **Flexible media support** - Images optionally stored on HDD/SSD for caching and improved response time.
 
 ### Results Sources
@@ -73,20 +73,30 @@ A self-hosted private search engine designed to be scalable and more resource-ef
 
 ### Running the QGato
 
+Linux:
+
 ```bash
 git clone https://weforge.xyz/Spitfire/Search.git
 cd Search
-go run .
+chmod +x ./run.sh
+./run.sh
+```
+
+Windows:
+
+```powershell
+git clone https://weforge.xyz/Spitfire/Search.git
+cd Search
+.\run.bat
 ```
 
 *Its that easy!*
 
 ### Configuring
 
-- Configuration is done via the `config.ini` file.
+Configuration is done via the ``config.ini`` file.
-- On first start, you will be guided through the basic setup.
+On first start, you will be guided through the basic setup.
-- For more advanced configuration options, visit the [Wiki Configuration Page](https://weforge.xyz/Spitfire/Search/wiki/Configuration).
+More advanced setup and all options will be listed here later, as this is still being updated.
-
 
 ## License
 

config.go

@@ -23,25 +23,18 @@ type CacheConfig struct {
 }
 
 type Config struct {
-	Port                int    // Added
+	Port                       int    // Added
-	AuthCode            string // Added
+	AuthCode                   string // Added
-	PeerID              string // Added
+	PeerID                     string // Added
-	Peers               []string
+	Peers                      []string
-	Domain              string // Added
+	Domain                     string // Added
-	NodesEnabled        bool   // Added
+	NodesEnabled               bool   // Added
-	MetaSearchEnabled   bool   // Added
+	CrawlerEnabled             bool   // Added
-	IndexerEnabled      bool   // Added
+	IndexerEnabled             bool   // Added
-	WebsiteEnabled      bool   // Added
+	WebsiteEnabled             bool   // Added
-	RamCacheEnabled     bool
+	RamCacheEnabled            bool
-	DriveCacheEnabled   bool     // Added
+	DriveCacheEnabled          bool // Added
-	MetaProxyEnabled    bool     // Added
+	LogLevel                   int  // Added
-	MetaProxyStrict     bool     // Added
-	MetaProxies         []string // Added
-	CrawlerProxyEnabled bool     // Added
-	CrawlerProxyStrict  bool     // Added
-	CrawlerProxies      []string // Added
-	// Maybye add Proxy support for Image Extraction?
-	LogLevel                   int // Added
 	ConcurrentStandardCrawlers int
 	ConcurrentChromeCrawlers   int
 	CrawlingInterval           time.Duration // Refres crawled results in...
@@ -58,17 +51,11 @@ var defaultConfig = Config{
 	Peers:                      []string{},
 	AuthCode:                   generateStrongRandomString(64),
 	NodesEnabled:               false,
-	MetaSearchEnabled:          true,
+	CrawlerEnabled:             true,
 	IndexerEnabled:             false,
 	WebsiteEnabled:             true,
 	RamCacheEnabled:            true,
 	DriveCacheEnabled:          false,
-	MetaProxyEnabled:           false,
-	MetaProxyStrict:            true,
-	MetaProxies:                []string{},
-	CrawlerProxyEnabled:        false,
-	CrawlerProxyStrict:         true,
-	CrawlerProxies:             []string{},
 	ConcurrentStandardCrawlers: 12,
 	ConcurrentChromeCrawlers:   4,
 	CrawlingInterval:           24 * time.Hour,
@@ -258,23 +245,14 @@ func saveConfig(config Config) {
 	// Features section
 	featuresSec := cfg.Section("Features")
 	featuresSec.Key("Nodes").SetValue(strconv.FormatBool(config.NodesEnabled))
-	featuresSec.Key("Crawler").SetValue(strconv.FormatBool(config.MetaSearchEnabled))
+	featuresSec.Key("Crawler").SetValue(strconv.FormatBool(config.CrawlerEnabled))
 	featuresSec.Key("Indexer").SetValue(strconv.FormatBool(config.IndexerEnabled))
 	featuresSec.Key("Website").SetValue(strconv.FormatBool(config.WebsiteEnabled))
-	featuresSec.Key("MetaProxy").SetValue(strconv.FormatBool(config.MetaProxyEnabled))
-	featuresSec.Key("CrawlerProxy").SetValue(strconv.FormatBool(config.CrawlerProxyEnabled))
-
-	// Proxies section
-	proxiesSec := cfg.Section("Proxies")
-	proxiesSec.Key("MetaProxyStrict").SetValue(strconv.FormatBool(config.MetaProxyStrict))
-	proxiesSec.Key("MetaProxies").SetValue(strings.Join(config.MetaProxies, ","))
-	proxiesSec.Key("CrawlerProxyStrict").SetValue(strconv.FormatBool(config.CrawlerProxyStrict))
-	proxiesSec.Key("CrawlerProxies").SetValue(strings.Join(config.CrawlerProxies, ","))
 
 	// Indexer section
 	indexerSec := cfg.Section("Indexer")
 	indexerSec.Key("ConcurrentStandardCrawlers").SetValue(strconv.Itoa(config.ConcurrentStandardCrawlers))
-	indexerSec.Key("ConcurrentChromeCrawlers").SetValue(strconv.Itoa(config.ConcurrentChromeCrawlers))
+	indexerSec.Key("ConcurrentChromeCrawlers").SetValue(strconv.Itoa(config.ConcurrentStandardCrawlers))
 	indexerSec.Key("CrawlingInterval").SetValue(config.CrawlingInterval.String())
 	indexerSec.Key("MaxPagesPerDomain").SetValue(strconv.Itoa(config.MaxPagesPerDomain))
 	indexerSec.Key("IndexBatchSize").SetValue(strconv.Itoa(config.IndexBatchSize))
@@ -314,19 +292,11 @@ func loadConfig() Config {
 
 	// Features
 	nodesEnabled := getConfigValueBool(cfg.Section("Features").Key("Nodes"), defaultConfig.NodesEnabled)
-	metaSearchEnabled := getConfigValueBool(cfg.Section("Features").Key("Crawler"), defaultConfig.MetaSearchEnabled)
+	crawlerEnabled := getConfigValueBool(cfg.Section("Features").Key("Crawler"), defaultConfig.CrawlerEnabled)
 	indexerEnabled := getConfigValueBool(cfg.Section("Features").Key("Indexer"), defaultConfig.IndexerEnabled)
 	websiteEnabled := getConfigValueBool(cfg.Section("Features").Key("Website"), defaultConfig.WebsiteEnabled)
 	ramCacheEnabled := getConfigValueBool(cfg.Section("Features").Key("RamCache"), defaultConfig.RamCacheEnabled)
 	driveCacheEnabled := getConfigValueBool(cfg.Section("Features").Key("DriveCache"), defaultConfig.DriveCacheEnabled)
-	metaProxyEnabled := getConfigValueBool(cfg.Section("Features").Key("MetaProxy"), defaultConfig.MetaProxyEnabled)
-	crawlerProxyEnabled := getConfigValueBool(cfg.Section("Features").Key("CrawlerProxy"), defaultConfig.CrawlerProxyEnabled)
-
-	// Proxies
-	metaProxyStrict := getConfigValueBool(cfg.Section("Proxies").Key("MetaProxyStrict"), defaultConfig.MetaProxyStrict)
-	metaProxies := strings.Split(getConfigValueString(cfg.Section("Proxies").Key("MetaProxies"), ""), ",")
-	crawlerProxyStrict := getConfigValueBool(cfg.Section("Proxies").Key("CrawlerProxyStrict"), defaultConfig.CrawlerProxyStrict)
-	crawlerProxies := strings.Split(getConfigValueString(cfg.Section("Proxies").Key("CrawlerProxies"), ""), ",")
 
 	// Indexing
 	concurrentStandardCrawlers := getConfigValue(cfg.Section("Indexer").Key("ConcurrentStandardCrawlers"), defaultConfig.ConcurrentStandardCrawlers, strconv.Atoi)
@@ -355,17 +325,11 @@ func loadConfig() Config {
 		AuthCode:                   authCode,
 		Peers:                      peers,
 		NodesEnabled:               nodesEnabled,
-		MetaSearchEnabled:          metaSearchEnabled,
+		CrawlerEnabled:             crawlerEnabled,
 		IndexerEnabled:             indexerEnabled,
 		WebsiteEnabled:             websiteEnabled,
 		RamCacheEnabled:            ramCacheEnabled,
 		DriveCacheEnabled:          driveCacheEnabled,
-		MetaProxyEnabled:           metaProxyEnabled,
-		MetaProxyStrict:            metaProxyStrict,
-		MetaProxies:                metaProxies,
-		CrawlerProxyEnabled:        crawlerProxyEnabled,
-		CrawlerProxyStrict:         crawlerProxyStrict,
-		CrawlerProxies:             crawlerProxies,
 		ConcurrentStandardCrawlers: concurrentStandardCrawlers,
 		ConcurrentChromeCrawlers:   concurrentChromeCrawlers,
 		CrawlingInterval:           crawlingInterval,

crawler-extraction.go

@@ -32,12 +32,8 @@ func fetchPageMetadataStandard(pageURL, userAgent string) (string, string, strin
 
 // fetchPageMetadataChrome uses Chromedp to handle JavaScript-rendered pages.
 func fetchPageMetadataChrome(pageURL, userAgent string) (string, string, string) {
-	// Create a custom allocator context for Chromedp with proxy support if enabled
+	// Create context
-	allocCtx, cancelAlloc := chromedp.NewExecAllocator(context.Background(), configureChromeOptions()...)
+	ctx, cancel := chromedp.NewContext(context.Background())
-	defer cancelAlloc()
-
-	// Create a browser context
-	ctx, cancel := chromedp.NewContext(allocCtx)
 	defer cancel()
 
 	var renderedHTML string
@@ -61,32 +57,6 @@ func fetchPageMetadataChrome(pageURL, userAgent string) (string, string, string)
 	return extractParsedDOM(doc)
 }
 
-// configureChromeOptions sets up Chrome options and proxy if CrawlerProxy is enabled.
-func configureChromeOptions() []chromedp.ExecAllocatorOption {
-	options := chromedp.DefaultExecAllocatorOptions[:]
-
-	if config.CrawlerProxyEnabled && crawlerProxyClient != nil {
-		// Retrieve proxy settings from CrawlerProxy
-		proxy := crawlerProxyClient.GetProxy() // Ensure a `GetProxy` method is implemented for your proxy client
-		if proxy != "" {
-			options = append(options, chromedp.ProxyServer(proxy))
-			printDebug("Using CrawlerProxy for Chromedp: %s", proxy)
-		} else {
-			printWarn("CrawlerProxy is enabled but no valid proxy is available")
-		}
-	}
-
-	// // Add additional Chrome
-	// options = append(options,
-	// 	chromedp.Flag("headless", true),
-	// 	chromedp.Flag("disable-gpu", true),
-	// 	chromedp.Flag("no-sandbox", true),
-	// 	chromedp.Flag("disable-setuid-sandbox", true),
-	// )
-
-	return options
-}
-
 // extractStandard does the normal HTML parse with OG, Twitter, etc.
 func extractStandard(pageURL, userAgent string) (title, desc, keywords string) {
 	client := &http.Client{Timeout: 15 * time.Second}
@@ -98,13 +68,7 @@ func extractStandard(pageURL, userAgent string) (title, desc, keywords string) {
 	req.Header.Set("User-Agent", userAgent)
 	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
 
-	// Use CrawlerProxy if enabled
+	resp, err := client.Do(req)
-	var resp *http.Response
-	if config.CrawlerProxyEnabled && crawlerProxyClient != nil {
-		resp, err = crawlerProxyClient.Do(req)
-	} else {
-		resp, err = client.Do(req)
-	}
 	if err != nil {
 		printDebug("Failed to GET %s: %v", pageURL, err)
 		return
@@ -221,13 +185,7 @@ func fallbackReadability(pageURL, userAgent, title, desc, keywords string) (stri
 	readReq.Header.Set("User-Agent", userAgent)
 	readReq.Header.Set("Accept-Language", "en-US,en;q=0.9")
 
-	// Use CrawlerProxy if enabled
+	readResp, err := client.Do(readReq)
-	var readResp *http.Response
-	if config.CrawlerProxyEnabled && crawlerProxyClient != nil {
-		readResp, err = crawlerProxyClient.Do(readReq)
-	} else {
-		readResp, err = client.Do(readReq)
-	}
 	if err != nil || readResp.StatusCode < 200 || readResp.StatusCode >= 300 {
 		if err != nil {
 			printDebug("go-readability GET error for %s: %v", pageURL, err)

files-thepiratebay.go

@@ -57,40 +57,31 @@ func (t *ThePirateBay) Search(query string, category string) ([]TorrentResult, e
 		return []TorrentResult{}, nil
 	}
 
-	searchURL := fmt.Sprintf("https://%s/q.php?q=%s&cat=%s", PIRATEBAY_DOMAIN, url.QueryEscape(query), categoryCode)
+	url := fmt.Sprintf("https://%s/q.php?q=%s&cat=%s", PIRATEBAY_DOMAIN, url.QueryEscape(query), categoryCode)
 
 	// User Agent generation
 	userAgent, err := GetUserAgent("files-tpb")
 	if err != nil {
-		return nil, fmt.Errorf("error generating User-Agent: %w", err)
+		fmt.Println("Error:", err)
+		return nil, err
 	}
 
-	req, err := http.NewRequest("GET", searchURL, nil)
+	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
-		return nil, fmt.Errorf("error creating request: %w", err)
+		return nil, err
 	}
 	req.Header.Set("User-Agent", userAgent)
 
-	// Perform the request using MetaProxy if enabled
+	client := &http.Client{}
-	var resp *http.Response
+	response, err := client.Do(req)
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{}
-		resp, err = client.Do(req)
-	}
 	if err != nil {
-		return nil, fmt.Errorf("error making request to The Pirate Bay: %w", err)
+		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
+	defer response.Body.Close()
 
 	var torrentData []map[string]interface{}
-	if err := json.NewDecoder(resp.Body).Decode(&torrentData); err != nil {
+	if err := json.NewDecoder(response.Body).Decode(&torrentData); err != nil {
-		return nil, fmt.Errorf("error decoding response JSON: %w", err)
+		return nil, err
 	}
 
 	var results []TorrentResult

files-torrentgalaxy.go

@@ -62,23 +62,18 @@ func (tg *TorrentGalaxy) Search(query string, category string) ([]TorrentResult,
 	// User Agent generation
 	userAgent, err := GetUserAgent("files-torrentgalaxy")
 	if err != nil {
-		return nil, fmt.Errorf("error generating User-Agent: %w", err)
+		fmt.Println("Error:", err)
+		return nil, err
 	}
 
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
-		return nil, fmt.Errorf("error creating request: %w", err)
+		return nil, err
 	}
 	req.Header.Set("User-Agent", userAgent)
 
-	// Perform the request using MetaProxy if enabled
+	client := &http.Client{}
-	var resp *http.Response
+	resp, err := client.Do(req)
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{}
-		resp, err = client.Do(req)
-	}
 	if err != nil {
 		return nil, fmt.Errorf("error making request to TorrentGalaxy: %w", err)
 	}

files.go

@@ -88,7 +88,7 @@ func getFileResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string,
 	case results := <-cacheChan:
 		if results == nil {
 			// Fetch only if the cache miss occurs and Crawler is enabled
-			if config.MetaSearchEnabled {
+			if config.CrawlerEnabled {
 				combinedResults = fetchFileResults(query, safe, lang, page)
 				if len(combinedResults) > 0 {
 					resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
@@ -102,7 +102,7 @@ func getFileResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string,
 		}
 	case <-time.After(2 * time.Second):
 		printDebug("Cache check timeout")
-		if config.MetaSearchEnabled {
+		if config.CrawlerEnabled {
 			combinedResults = fetchFileResults(query, safe, lang, page)
 			if len(combinedResults) > 0 {
 				resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
@@ -117,7 +117,7 @@ func getFileResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string,
 
 func fetchFileResults(query, safe, lang string, page int) []TorrentResult {
 	// If Crawler is disabled, skip fetching from torrent sites
-	if !config.MetaSearchEnabled {
+	if !config.CrawlerEnabled {
 		printInfo("Crawler is disabled; skipping torrent site fetching.")
 		return []TorrentResult{}
 	}

forums.go

@@ -10,7 +10,7 @@ import (
 )
 
 func PerformRedditSearch(query string, safe string, page int) ([]ForumSearchResult, error) {
-	if !config.MetaSearchEnabled {
+	if !config.CrawlerEnabled {
 		printDebug("Crawler is disabled; skipping forum search.")
 		return []ForumSearchResult{}, nil
 	}
@@ -150,7 +150,7 @@ func getForumResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
 	case results := <-cacheChan:
 		if results == nil {
 			// Fetch only if the cache miss occurs and Crawler is enabled
-			if config.MetaSearchEnabled {
+			if config.CrawlerEnabled {
 				combinedResults = fetchForumResults(query, safe, lang, page)
 				if len(combinedResults) > 0 {
 					resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
@@ -164,7 +164,7 @@ func getForumResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
 		}
 	case <-time.After(2 * time.Second):
 		printDebug("Cache check timeout")
-		if config.MetaSearchEnabled {
+		if config.CrawlerEnabled {
 			combinedResults = fetchForumResults(query, safe, lang, page)
 			if len(combinedResults) > 0 {
 				resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))

images-bing.go

@@ -18,27 +18,8 @@ func PerformBingImageSearch(query, safe, lang string, page int) ([]ImageSearchRe
 	// Build the search URL
 	searchURL := buildBingSearchURL(query, page)
 
-	// Create the HTTP request
+	// Make the HTTP request
-	req, err := http.NewRequest("GET", searchURL, nil)
+	resp, err := http.Get(searchURL)
-	if err != nil {
-		return nil, 0, fmt.Errorf("creating request: %v", err)
-	}
-
-	// Set User-Agent
-	ImageUserAgent, err := GetUserAgent("Image-Search-Bing")
-	if err != nil {
-		return nil, 0, fmt.Errorf("generating User-Agent: %v", err)
-	}
-	req.Header.Set("User-Agent", ImageUserAgent)
-
-	// Use MetaProxy if enabled
-	var resp *http.Response
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{Timeout: 10 * time.Second}
-		resp, err = client.Do(req)
-	}
 	if err != nil {
 		return nil, 0, fmt.Errorf("making request: %v", err)
 	}

images-deviantart.go

@@ -87,21 +87,15 @@ func PerformDeviantArtImageSearch(query, safe, lang string, page int) ([]ImageSe
 		return nil, 0, err
 	}
 
-	// Create the HTTP request
+	// Make the HTTP request with User-Agent header
+	client := &http.Client{}
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
 		return nil, 0, fmt.Errorf("creating request: %v", err)
 	}
 	req.Header.Set("User-Agent", DeviantArtImageUserAgent)
 
-	// Perform the request using MetaProxy if enabled
+	resp, err := client.Do(req)
-	var resp *http.Response
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{}
-		resp, err = client.Do(req)
-	}
 	if err != nil {
 		return nil, 0, fmt.Errorf("making request: %v", err)
 	}
@@ -188,7 +182,7 @@ func PerformDeviantArtImageSearch(query, safe, lang string, page int) ([]ImageSe
 
 	duration := time.Since(startTime)
 
-	// Check if the number of results is zero
+	// Check if the number of results is one or less
 	if len(results) == 0 {
 		return nil, duration, fmt.Errorf("no images found")
 	}

images-imgur.go

@@ -18,27 +18,7 @@ func PerformImgurImageSearch(query, safe, lang string, page int) ([]ImageSearchR
 	var results []ImageSearchResult
 	searchURL := buildImgurSearchURL(query, page)
 
-	// Create the HTTP request
+	resp, err := http.Get(searchURL)
-	req, err := http.NewRequest("GET", searchURL, nil)
-	if err != nil {
-		return nil, 0, fmt.Errorf("creating request: %v", err)
-	}
-
-	// Get the User-Agent string
-	imgurUserAgent, err := GetUserAgent("Image-Search-Imgur")
-	if err != nil {
-		return nil, 0, fmt.Errorf("getting user-agent: %v", err)
-	}
-	req.Header.Set("User-Agent", imgurUserAgent)
-
-	// Perform the HTTP request with MetaProxy if enabled
-	var resp *http.Response
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{}
-		resp, err = client.Do(req)
-	}
 	if err != nil {
 		return nil, 0, fmt.Errorf("making request: %v", err)
 	}
@@ -48,7 +28,6 @@ func PerformImgurImageSearch(query, safe, lang string, page int) ([]ImageSearchR
 		return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
 
-	// Parse the HTML document
 	doc, err := goquery.NewDocumentFromReader(resp.Body)
 	if err != nil {
 		return nil, 0, fmt.Errorf("loading HTML document: %v", err)
@@ -97,35 +76,12 @@ func PerformImgurImageSearch(query, safe, lang string, page int) ([]ImageSearchR
 
 	duration := time.Since(startTime) // Calculate the duration
 
-	if len(results) == 0 {
-		return nil, duration, fmt.Errorf("no images found")
-	}
-
 	return results, duration, nil
 }
 
 // scrapeImageFromImgurPage scrapes the image source from the Imgur page
 func scrapeImageFromImgurPage(pageURL string) string {
-	req, err := http.NewRequest("GET", pageURL, nil)
+	resp, err := http.Get(pageURL)
-	if err != nil {
-		fmt.Printf("Error creating request for page: %v\n", err)
-		return ""
-	}
-
-	// Get the User-Agent string
-	imgurUserAgent, err := GetUserAgent("Image-Search-Imgur")
-	if err == nil {
-		req.Header.Set("User-Agent", imgurUserAgent)
-	}
-
-	// Perform the request using MetaProxy if enabled
-	var resp *http.Response
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{}
-		resp, err = client.Do(req)
-	}
 	if err != nil {
 		fmt.Printf("Error fetching page: %v\n", err)
 		return ""

images-quant.go

@@ -97,7 +97,7 @@ func PerformQwantImageSearch(query, safe, lang string, page int) ([]ImageSearchR
 
 	// Ensure count + offset is within acceptable limits
 	if offset+resultsPerPage > 250 {
-		return nil, 0, fmt.Errorf("count + offset must be lower than 250 for Qwant")
+		return nil, 0, fmt.Errorf("count + offset must be lower than 250 for quant")
 	}
 
 	if safe == "" {
@@ -113,27 +113,21 @@ func PerformQwantImageSearch(query, safe, lang string, page int) ([]ImageSearchR
 		offset,
 		safe)
 
-	// Create the HTTP request
+	client := &http.Client{Timeout: 10 * time.Second}
+
 	req, err := http.NewRequest("GET", apiURL, nil)
 	if err != nil {
 		return nil, 0, fmt.Errorf("creating request: %v", err)
 	}
 
-	// Get the User-Agent string
 	ImageUserAgent, err := GetUserAgent("Image-Search-Quant")
 	if err != nil {
-		return nil, 0, fmt.Errorf("getting user-agent: %v", err)
+		return nil, 0, err
 	}
-	req.Header.Set("User-Agent", ImageUserAgent)
 
-	// Perform the request with MetaProxy if enabled
+	req.Header.Set("User-Agent", ImageUserAgent) // Quant seems to not like some specific User-Agent strings
-	var resp *http.Response
+
-	if config.MetaProxyEnabled && metaProxyClient != nil {
+	resp, err := client.Do(req)
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{Timeout: 10 * time.Second}
-		resp, err = client.Do(req)
-	}
 	if err != nil {
 		return nil, 0, fmt.Errorf("making request: %v", err)
 	}
@@ -143,13 +137,11 @@ func PerformQwantImageSearch(query, safe, lang string, page int) ([]ImageSearchR
 		return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
 
-	// Parse the API response
 	var apiResp QwantAPIResponse
 	if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
 		return nil, 0, fmt.Errorf("decoding response: %v", err)
 	}
 
-	// Process the results
 	var wg sync.WaitGroup
 	results := make([]ImageSearchResult, len(apiResp.Data.Result.Items))
 
@@ -182,9 +174,5 @@ func PerformQwantImageSearch(query, safe, lang string, page int) ([]ImageSearchR
 
 	duration := time.Since(startTime) // Calculate the duration
 
-	if len(results) == 0 {
-		return nil, duration, fmt.Errorf("no images found")
-	}
-
 	return results, duration, nil
 }

images.go

@@ -86,7 +86,7 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
 	select {
 	case results := <-cacheChan:
 		if results == nil {
-			if config.MetaSearchEnabled {
+			if config.CrawlerEnabled {
 				combinedResults = fetchImageResults(query, safe, lang, page, synchronous)
 				if len(combinedResults) > 0 {
 					combinedResults = filterValidImages(combinedResults)
@@ -101,7 +101,7 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
 		}
 	case <-time.After(2 * time.Second):
 		printDebug("Cache check timeout")
-		if config.MetaSearchEnabled {
+		if config.CrawlerEnabled {
 			combinedResults = fetchImageResults(query, safe, lang, page, synchronous)
 			if len(combinedResults) > 0 {
 				combinedResults = filterValidImages(combinedResults)
@@ -118,8 +118,8 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
 func fetchImageResults(query, safe, lang string, page int, synchronous bool) []ImageSearchResult {
 	var results []ImageSearchResult
 
-	// Check if MetaSearchEnabled is false
+	// Check if CrawlerEnabled is false
-	if !config.MetaSearchEnabled {
+	if !config.CrawlerEnabled {
 		printDebug("Crawler is disabled; skipping image search engine fetching.")
 		return results
 	}

init.go

@@ -60,12 +60,8 @@ func main() {
 	}
 	config.PeerID = hostID
 
-	if config.CrawlerProxyEnabled || config.MetaProxyEnabled {
-		InitProxies()
-	}
-
 	// Initiate Browser Agent updater
-	if config.MetaSearchEnabled || config.IndexerEnabled {
+	if config.CrawlerEnabled || config.IndexerEnabled {
 		go periodicAgentUpdate()
 	}
 

node.go

@@ -5,7 +5,7 @@ import (
 	"crypto/rand"
 	"encoding/json"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"time"
 )
@@ -65,10 +65,7 @@ func sendMessage(serverAddr string, msg Message) error {
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		body, err := io.ReadAll(resp.Body)
+		body, _ := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return fmt.Errorf("failed to read response body: %v", err)
-		}
 		return fmt.Errorf("server error: %s", body)
 	}
 

proxy.go

@@ -1,169 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
-
-	"golang.org/x/net/proxy"
-)
-
-// ProxyConfig holds configuration for a single proxy.
-type ProxyConfig struct {
-	Address  string
-	Username string
-	Password string
-}
-
-// ProxyClient provides an HTTP client pool for proxies.
-type ProxyClient struct {
-	clients []*http.Client
-	lock    sync.Mutex
-	index   int
-}
-
-// Package-level proxy clients
-var (
-	metaProxyClient    *ProxyClient
-	crawlerProxyClient *ProxyClient
-)
-
-// NewProxyClientPool creates a pool of HTTP clients with proxies.
-func NewProxyClientPool(proxies []ProxyConfig, timeout time.Duration) (*ProxyClient, error) {
-	if len(proxies) == 0 {
-		return nil, fmt.Errorf("no proxies provided")
-	}
-
-	clients := make([]*http.Client, len(proxies))
-
-	for i, proxyConfig := range proxies {
-		var auth *proxy.Auth
-		if proxyConfig.Username != "" || proxyConfig.Password != "" {
-			auth = &proxy.Auth{
-				User:     proxyConfig.Username,
-				Password: proxyConfig.Password,
-			}
-		}
-
-		dialer, err := proxy.SOCKS5("tcp", proxyConfig.Address, auth, proxy.Direct)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create SOCKS5 dialer for %s: %w", proxyConfig.Address, err)
-		}
-
-		transport := &http.Transport{Dial: dialer.Dial}
-		clients[i] = &http.Client{
-			Transport: transport,
-			Timeout:   timeout,
-		}
-	}
-
-	return &ProxyClient{clients: clients}, nil
-}
-
-// Do sends an HTTP request using the next proxy in the pool.
-func (p *ProxyClient) Do(req *http.Request) (*http.Response, error) {
-	p.lock.Lock()
-	client := p.clients[p.index]
-	p.index = (p.index + 1) % len(p.clients)
-	p.lock.Unlock()
-	return client.Do(req)
-}
-
-func (p *ProxyClient) GetProxy() string {
-	p.lock.Lock()
-	defer p.lock.Unlock()
-
-	if len(p.clients) == 0 {
-		return ""
-	}
-
-	// Round-robin proxy retrieval
-	client := p.clients[p.index]
-	p.index = (p.index + 1) % len(p.clients)
-
-	// Assume each client has a proxy string saved
-	// Example implementation depends on how your proxies are configured
-	proxyTransport, ok := client.Transport.(*http.Transport)
-	if ok && proxyTransport.Proxy != nil {
-		proxyURL, _ := proxyTransport.Proxy(nil)
-		if proxyURL != nil {
-			return proxyURL.String()
-		}
-	}
-
-	return ""
-}
-
-// ParseProxies parses the proxy strings in the format ADDRESS:PORT or ADDRESS:PORT:USER:PASSWORD.
-func ParseProxies(proxyStrings []string) []ProxyConfig {
-	var proxies []ProxyConfig
-	for _, proxy := range proxyStrings {
-		parts := strings.Split(proxy, ":")
-		if len(parts) == 2 { // ADDRESS:PORT
-			proxies = append(proxies, ProxyConfig{
-				Address: fmt.Sprintf("%s:%s", parts[0], parts[1]),
-			})
-		} else if len(parts) == 4 { // ADDRESS:PORT:USER:PASSWORD
-			proxies = append(proxies, ProxyConfig{
-				Address:  fmt.Sprintf("%s:%s", parts[0], parts[1]),
-				Username: parts[2],
-				Password: parts[3],
-			})
-		} else {
-			fmt.Printf("Invalid proxy format: %s\n", proxy)
-		}
-	}
-	return proxies
-}
-
-// InitProxies initializes the proxy clients for Meta and Crawler proxies.
-func InitProxies() {
-	// Initialize Meta Proxy Client
-	if config.MetaProxyEnabled {
-		metaProxies := ParseProxies(config.MetaProxies)
-		client, err := NewProxyClientPool(metaProxies, 30*time.Second)
-		if err != nil {
-			if config.MetaProxyStrict {
-				panic(fmt.Sprintf("Failed to initialize Meta proxies: %v", err))
-			}
-			fmt.Printf("Warning: Meta proxy initialization failed: %v\n", err)
-		}
-		metaProxyClient = client
-	}
-
-	// Initialize Crawler Proxy Client
-	if config.CrawlerProxyEnabled {
-		crawlerProxies := ParseProxies(config.CrawlerProxies)
-		client, err := NewProxyClientPool(crawlerProxies, 30*time.Second)
-		if err != nil {
-			if config.CrawlerProxyStrict {
-				panic(fmt.Sprintf("Failed to initialize Crawler proxies: %v", err))
-			}
-			fmt.Printf("Warning: Crawler proxy initialization failed: %v\n", err)
-		}
-		crawlerProxyClient = client
-	}
-}
-
-// func main() {
-// 	config := loadConfig()
-
-// 	// Initialize proxies if enabled
-// 	if config.CrawlerProxyEnabled || config.MetaProxyEnabled {
-// 		InitProxies()
-// 	}
-
-// 	// Example usage
-// 	if metaProxyClient != nil {
-// 		req, _ := http.NewRequest("GET", "https://example.com", nil)
-// 		resp, err := metaProxyClient.Do(req)
-// 		if err != nil {
-// 			fmt.Printf("Error using MetaProxyClient: %v\n", err)
-// 		} else {
-// 			fmt.Printf("Meta Proxy Response Status: %s\n", resp.Status)
-// 			resp.Body.Close()
-// 		}
-// 	}
-// }

run.bat

@@ -5,6 +5,7 @@ rem Initialize variables
 set SKIP_CONFIG=""
 set PORT=""
 set DOMAIN=""
+set BUILD_MODE=false
 set BUILD_OUTPUT=qgato.exe
 
 rem Parse arguments
@@ -27,6 +28,11 @@ if "%~1"=="--skip-config-check" (
     shift
     goto parse_args
 )
+if "%~1"=="--build" (
+    set BUILD_MODE=true
+    shift
+    goto parse_args
+)
 echo Unknown argument: %~1
 exit /b 1
 
@@ -44,28 +50,37 @@ for %%f in (*.go) do (
     )
 )
 
-rem Always delete and rebuild the binary
+if "%BUILD_MODE%"=="true" (
-echo Cleaning previous build...
+    rem Build mode
-if exist "%BUILD_OUTPUT%" del "%BUILD_OUTPUT%"
+    echo Building application...
+    go build -o "%BUILD_OUTPUT%" !GO_FILES!
+    if errorlevel 1 (
+        echo Build failed!
+        exit /b 1
+    )
+    echo Build successful! Output: %CD%\%BUILD_OUTPUT%
+) else (
+    rem Check if the executable exists
+    if not exist "%BUILD_OUTPUT%" (
+        echo Executable not found. Building it first...
+        go build -o "%BUILD_OUTPUT%" !GO_FILES!
+        if errorlevel 1 (
+            echo Build failed! Unable to run the application.
+            exit /b 1
+        )
+    )
 
-echo Building application...
+    rem Construct the command
-go build -o "%BUILD_OUTPUT%" !GO_FILES!
+    set CMD="%BUILD_OUTPUT% !SKIP_CONFIG!"
-if errorlevel 1 (
+    if not "%PORT%"=="" set CMD=!CMD! --port %PORT%
-    echo Build failed!
+    if not "%DOMAIN%"=="" set CMD=!CMD! --domain %DOMAIN%
-    exit /b 1
+
+    rem Informative output
+    echo Starting application with command: !CMD!
+
+    rem Run the application
+    call !CMD!
 )
-echo Build successful! Output: %CD%\%BUILD_OUTPUT%
-
-rem Construct the command
-set CMD=%BUILD_OUTPUT% !SKIP_CONFIG!
-if not "%PORT%"=="" set CMD=!CMD! --port %PORT%
-if not "%DOMAIN%"=="" set CMD=!CMD! --domain %DOMAIN%
-
-rem Informative output
-echo Starting application with command: !CMD!
-
-rem Run the built executable
-call !CMD!
 
 rem Return to the original directory
 popd

run.sh

@@ -4,6 +4,7 @@
 SKIP_CONFIG=""
 PORT=""
 DOMAIN=""
+BUILD_MODE=false
 BUILD_OUTPUT="qgato"
 
 # Parse arguments
@@ -21,6 +22,10 @@ while [ $# -gt 0 ]; do
       SKIP_CONFIG="--skip-config-check"
       shift
       ;;
+    --build)
+      BUILD_MODE=true
+      shift
+      ;;
     *)
       echo "Unknown argument: $1"
       exit 1
@@ -34,25 +39,33 @@ SCRIPT_DIR=$(dirname "$0")
 # List all Go files in the script directory (excluding test files)
 GO_FILES=$(find "$SCRIPT_DIR" -name '*.go' ! -name '*_test.go' -print)
 
-# Always delete and rebuild the binary
+if $BUILD_MODE; then
-echo "Cleaning previous build..."
+  # Build mode
-rm -f "$SCRIPT_DIR/$BUILD_OUTPUT"
+  echo "Building application..."
-
+  go build -o "$SCRIPT_DIR/$BUILD_OUTPUT" $GO_FILES
-echo "Building application..."
+  if [ $? -eq 0 ]; then
-go build -o "$SCRIPT_DIR/$BUILD_OUTPUT" $GO_FILES
+    echo "Build successful! Output: $SCRIPT_DIR/$BUILD_OUTPUT"
-if [ $? -eq 0 ]; then
+  else
-  echo "Build successful! Output: $SCRIPT_DIR/$BUILD_OUTPUT"
+    echo "Build failed!"
+    exit 1
+  fi
 else
-  echo "Build failed!"
+  # Run mode
-  exit 1
+  CMD="./$BUILD_OUTPUT $SKIP_CONFIG"
+  [ -n "$PORT" ] && CMD="$CMD --port $PORT"
+  [ -n "$DOMAIN" ] && CMD="$CMD --domain $DOMAIN"
+
+  if [ ! -f "$SCRIPT_DIR/$BUILD_OUTPUT" ]; then
+    echo "Executable not found. Building it first..."
+    go build -o "$SCRIPT_DIR/$BUILD_OUTPUT" $GO_FILES
+    if [ $? -ne 0 ]; then
+      echo "Build failed! Unable to run the application."
+      exit 1
+    fi
+  fi
+
+  echo "Starting application with command: $CMD"
+
+  # Run the executable
+  eval $CMD
 fi
-
-# Construct the run command
-CMD="$SCRIPT_DIR/$BUILD_OUTPUT $SKIP_CONFIG"
-[ -n "$PORT" ] && CMD="$CMD --port $PORT"
-[ -n "$DOMAIN" ] && CMD="$CMD --domain $DOMAIN"
-
-echo "Starting application with command: $CMD"
-
-# Run the built executable
-eval $CMD

tests/integration_test.go

@@ -2,6 +2,7 @@ package tests
 
 import (
 	"bufio"
+	"context"
 	"crypto/rand"
 	"encoding/json"
 	"fmt"
@@ -9,7 +10,9 @@ import (
 	"math/big"
 	"net/http"
 	"net/url"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"sync"
 	"syscall"
 	"testing"
@@ -53,27 +56,50 @@ func TestApplication(t *testing.T) {
 	// Ensure the test runs from the root directory
 	rootDir := "../" // Path to the root directory of the repository
 
-	// Run the application using `run.sh`
+	// Build the application using `run.sh --build`
-	runCmd := exec.Command("sh", "./run.sh", "--skip-config-check")
+	buildCmd := exec.Command("sh", "./run.sh", "--build")
-	runCmd.Dir = rootDir
+	buildCmd.Dir = rootDir
+
+	buildOutput, err := buildCmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("Failed to build application: %v\nOutput:\n%s", err, string(buildOutput))
+	}
+	t.Log("Application built successfully")
+
+	// Path to the built executable relative to rootDir
+	executablePath := "./qgato" // Since cmd.Dir is rootDir, this path is relative to rootDir
+
+	// Ensure the executable has execute permissions
+	execFullPath := filepath.Join(rootDir, "qgato")
+	if err := os.Chmod(execFullPath, 0755); err != nil {
+		t.Fatalf("Failed to set execute permissions on the executable: %v", err)
+	}
+
+	// Create a context with cancellation
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel() // Ensure resources are cleaned up
+
+	// Start the application using the built executable
+	cmd := exec.CommandContext(ctx, executablePath, "--skip-config-check")
+	cmd.Dir = rootDir // Set the working directory to the root directory
 
 	// Set process group ID so we can kill it and its children
-	runCmd.SysProcAttr = &syscall.SysProcAttr{
+	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setpgid: true,
 	}
 
 	// Capture application output for logging
-	appStdout, err := runCmd.StdoutPipe()
+	appStdout, err := cmd.StdoutPipe()
 	if err != nil {
 		t.Fatalf("Failed to capture stdout: %v", err)
 	}
-	appStderr, err := runCmd.StderrPipe()
+	appStderr, err := cmd.StderrPipe()
 	if err != nil {
 		t.Fatalf("Failed to capture stderr: %v", err)
 	}
 
 	// Start the application
-	if err := runCmd.Start(); err != nil {
+	if err := cmd.Start(); err != nil {
 		t.Fatalf("Failed to start application: %v", err)
 	}
 
@@ -94,28 +120,28 @@ func TestApplication(t *testing.T) {
 	// Defer cleanup to ensure process is killed after the test
 	defer func() {
 		// Kill the process group
-		pgid, err := syscall.Getpgid(runCmd.Process.Pid)
+		pgid, err := syscall.Getpgid(cmd.Process.Pid)
 		if err == nil {
 			syscall.Kill(-pgid, syscall.SIGKILL)
 		} else {
 			t.Logf("Failed to get process group ID: %v", err)
-			runCmd.Process.Kill()
+			cmd.Process.Kill()
 		}
-		runCmd.Wait()
+		cmd.Wait()
 
 		// Print summary
 		printSummary(summary, t)
 	}()
 
 	// Wait for the server to start
-	if !waitForServer("http://localhost:5000", 600*time.Second) {
+	if !waitForServer("http://localhost:5000", 15*time.Second) {
 		t.Fatalf("Server did not start within the expected time")
 	}
 
 	t.Log("Application is running")
 
 	// Create a process instance for the application
-	appProcess, err := process.NewProcess(int32(runCmd.Process.Pid))
+	appProcess, err := process.NewProcess(int32(cmd.Process.Pid))
 	if err != nil {
 		t.Fatalf("Failed to create process instance: %v", err)
 	}

text-brave.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
@@ -21,45 +21,33 @@ func PerformBraveTextSearch(query, safe, lang string, offset int) ([]TextSearchR
 
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
-		printWarn("Error creating request: %v", err)
 		return nil, 0, fmt.Errorf("creating request: %v", err)
 	}
 
 	// Set headers including User-Agent
 	TextUserAgent, err := GetUserAgent("Text-Search")
 	if err != nil {
-		printWarn("Error generating User-Agent: %v", err)
 		return nil, 0, err
 	}
 	req.Header.Set("User-Agent", TextUserAgent)
 
-	var resp *http.Response
+	// Perform the HTTP request
-
+	client := &http.Client{}
-	// Determine whether to use a proxy client or a default client
+	resp, err := client.Do(req)
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{}
-		resp, err = client.Do(req)
-	}
-
 	if err != nil {
-		printWarn("Error performing request: %v", err)
 		return nil, 0, fmt.Errorf("performing request: %v", err)
 	}
 	defer resp.Body.Close()
 
 	// Read the response body
-	body, err := io.ReadAll(resp.Body)
+	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		printWarn("Error reading response body: %v", err)
 		return nil, 0, fmt.Errorf("reading response body: %v", err)
 	}
 
 	// Parse the response body
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(body)))
 	if err != nil {
-		printErr("Error parsing response body: %v", err)
 		return nil, 0, fmt.Errorf("parsing response body: %v", err)
 	}
 
@@ -83,10 +71,8 @@ func PerformBraveTextSearch(query, safe, lang string, offset int) ([]TextSearchR
 
 	// Return an error if no results are found
 	if len(results) == 0 {
-		printDebug("No results found for query: %s", query)
 		return nil, duration, fmt.Errorf("no results found")
 	}
 
-	printDebug("Search completed successfully for query: %s, found %d results", query, len(results))
 	return results, duration, nil
 }

text-duckduckgo.go

@@ -16,38 +16,21 @@ func PerformDuckDuckGoTextSearch(query, safe, lang string, page int) ([]TextSear
 	var results []TextSearchResult
 	searchURL := buildDuckDuckGoSearchURL(query, page)
 
-	// Create a request
+	resp, err := http.Get(searchURL)
-	req, err := http.NewRequest("GET", searchURL, nil)
-	if err != nil {
-		return nil, 0, fmt.Errorf("creating request: %v", err)
-	}
-
-	// Use proxy client if MetaProxy is enabled
-	var resp *http.Response
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{}
-		resp, err = client.Do(req)
-	}
-
 	if err != nil {
 		return nil, 0, fmt.Errorf("making request: %v", err)
 	}
 	defer resp.Body.Close()
 
-	// Check for HTTP status code
 	if resp.StatusCode != http.StatusOK {
 		return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
 
-	// Parse HTML response
 	doc, err := goquery.NewDocumentFromReader(resp.Body)
 	if err != nil {
 		return nil, 0, fmt.Errorf("loading HTML document: %v", err)
 	}
 
-	// Extract results from the page
 	doc.Find(".result__body").Each(func(i int, s *goquery.Selection) {
 		header := s.Find(".result__a").Text()
 		description := s.Find(".result__snippet").Text()

text-google.go

@@ -16,48 +16,37 @@ func PerformGoogleTextSearch(query, safe, lang string, page int) ([]TextSearchRe
 
 	startTime := time.Now() // Start the timer
 
-	// Build the search URL
+	client := &http.Client{}
 	searchURL := buildSearchURL(query, safe, lang, page, resultsPerPage)
 
-	// Create a new request
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
 		return nil, 0, fmt.Errorf("failed to create request: %v", err)
 	}
 
-	// Generate and set User-Agent header
+	// User Agent generation
-	TextUserAgent, err := GetUserAgent("Google-Text-Search")
+	TextUserAgent, err := GetUserAgent("Text-Search")
 	if err != nil {
 		return nil, 0, err
 	}
+
 	req.Header.Set("User-Agent", TextUserAgent)
 
-	// Perform the request using proxy if MetaProxy is enabled
+	resp, err := client.Do(req)
-	var resp *http.Response
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{}
-		resp, err = client.Do(req)
-	}
-
 	if err != nil {
 		return nil, 0, fmt.Errorf("making request: %v", err)
 	}
 	defer resp.Body.Close()
 
-	// Check for HTTP status code
 	if resp.StatusCode != http.StatusOK {
 		return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
 
-	// Parse the HTML response
 	doc, err := goquery.NewDocumentFromReader(resp.Body)
 	if err != nil {
 		return nil, 0, fmt.Errorf("loading HTML document: %v", err)
 	}
 
-	// Extract search results
 	results = parseResults(doc)
 
 	duration := time.Since(startTime) // Calculate the duration

text-librex.go

@@ -28,7 +28,7 @@ func PerformLibreXTextSearch(query, safe, lang string, page int) ([]TextSearchRe
 
 	searchURL := fmt.Sprintf("https://%s/api.php?q=%s&p=%d&t=0", LIBREX_DOMAIN, url.QueryEscape(query), page)
 
-	// Generate User-Agent
+	// User Agent generation
 	userAgent, err := GetUserAgent("librex-text-search")
 	if err != nil {
 		return nil, 0, err
@@ -40,15 +40,8 @@ func PerformLibreXTextSearch(query, safe, lang string, page int) ([]TextSearchRe
 	}
 	req.Header.Set("User-Agent", userAgent)
 
-	// Perform the request using the appropriate client
+	client := &http.Client{}
-	var resp *http.Response
+	resp, err := client.Do(req)
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{}
-		resp, err = client.Do(req)
-	}
-
 	if err != nil {
 		return nil, 0, logError("error making request to LibreX", err)
 	}
@@ -84,7 +77,7 @@ func PerformLibreXTextSearch(query, safe, lang string, page int) ([]TextSearchRe
 	return results, duration, nil
 }
 
-// This is just stupid it will probably lead to printing error twice
+// This is just stupid it will probbably lead to printing error twice
 func logError(message string, err error) error {
 	log.Printf("%s: %v", message, err)
 	return fmt.Errorf("%s: %w", message, err)

text-quant.go

@@ -46,6 +46,8 @@ func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error
 		lang,
 		offset)
 
+	client := &http.Client{Timeout: 10 * time.Second}
+
 	req, err := http.NewRequest("GET", apiURL, nil)
 	if err != nil {
 		return nil, fmt.Errorf("creating request: %v", err)
@@ -53,15 +55,7 @@ func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error
 
 	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
 
-	// Perform the request using the appropriate client
+	resp, err := client.Do(req)
-	var resp *http.Response
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{Timeout: 10 * time.Second}
-		resp, err = client.Do(req)
-	}
-
 	if err != nil {
 		return nil, fmt.Errorf("making request: %v", err)
 	}

text-searchxng.go

@@ -3,63 +3,31 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io"
+	"io/ioutil"
+	"log"
 	"net/http"
-	"net/url"
-	"strings"
 	"time"
 )
 
 type Instance struct {
-	URL             string                 `json:"-"` // Populated from map key
+	URL      string `json:"url"`
-	Analytics       bool                   `json:"analytics"`
+	Status   int    `json:"status"`
-	Comments        []string               `json:"comments"`
+	SSLGrade string `json:"ssl_grade"`
-	AlternativeUrls map[string]interface{} `json:"alternativeUrls"`
-	Main            bool                   `json:"main"`
-	NetworkType     string                 `json:"network_type"`
-	HTTP            struct {
-		StatusCode int    `json:"status_code"`
-		Error      string `json:"error"`
-	} `json:"http"`
-	Version    string       `json:"version"`
-	Grade      string       `json:"grade"`
-	GradeURL   string       `json:"gradeUrl"`
-	Generator  string       `json:"generator"`
-	ContactURL FlexibleType `json:"contact_url"` // Custom type
-	DocsURL    string       `json:"docs_url"`
-}
-
-type FlexibleType struct {
-	StringValue string
-	BoolValue   bool
-	IsString    bool
 }
 
 const searxInstancesURL = "https://searx.space/data/instances.json"
 
-// FetchInstances fetches available SearX instances from the registry.
 func fetchInstances() ([]Instance, error) {
+	client := &http.Client{Timeout: 10 * time.Second}
 	req, err := http.NewRequest("GET", searxInstancesURL, nil)
 	if err != nil {
-		return nil, fmt.Errorf("creating request: %v", err)
+		return nil, err
 	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
 
-	XNGUserAgent, err := GetUserAgent("Text-Search-XNG")
+	resp, err := client.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("generating User-Agent: %v", err)
+		return nil, err
-	}
-	req.Header.Set("User-Agent", XNGUserAgent)
-
-	var resp *http.Response
-	if config.MetaProxyEnabled && config.MetaProxyStrict && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{Timeout: 10 * time.Second}
-		resp, err = client.Do(req)
-	}
-
-	if err != nil {
-		return nil, fmt.Errorf("performing request: %v", err)
 	}
 	defer resp.Body.Close()
 
@@ -67,105 +35,44 @@ func fetchInstances() ([]Instance, error) {
 		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
 
-	body, err := io.ReadAll(resp.Body)
+	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return nil, fmt.Errorf("reading response body: %v", err)
+		return nil, err
 	}
 
-	// Root structure of the JSON response
-	var root struct {
-		Instances map[string]Instance `json:"instances"`
-	}
-
-	// Unmarshal JSON into the root structure
-	err = json.Unmarshal(body, &root)
-	if err != nil {
-		return nil, fmt.Errorf("parsing response JSON: %v", err)
-	}
-
-	// Collect instances into a slice
 	var instances []Instance
-	for url, instance := range root.Instances {
+	err = json.Unmarshal(body, &instances)
-		instance.URL = url // Assign the URL from the map key
+	if err != nil {
-		instances = append(instances, instance)
+		return nil, err
 	}
 
 	return instances, nil
 }
 
-// UnmarshalJSON implements custom unmarshalling for FlexibleType.
-func (f *FlexibleType) UnmarshalJSON(data []byte) error {
-	// Try to unmarshal as a string
-	var str string
-	if err := json.Unmarshal(data, &str); err == nil {
-		f.StringValue = str
-		f.IsString = true
-		return nil
-	}
-
-	// Try to unmarshal as a bool
-	var b bool
-	if err := json.Unmarshal(data, &b); err == nil {
-		f.BoolValue = b
-		f.IsString = false
-		return nil
-	}
-
-	// Return an error if neither works
-	return fmt.Errorf("invalid FlexibleType: %s", string(data))
-}
-
-// String returns the string representation of FlexibleType.
-func (f FlexibleType) String() string {
-	if f.IsString {
-		return f.StringValue
-	}
-	return fmt.Sprintf("%v", f.BoolValue)
-}
-
-// ValidateInstance checks if a SearX instance is valid by performing a test query.
 func validateInstance(instance Instance) bool {
-	// Skip .onion instances
+	client := &http.Client{Timeout: 10 * time.Second}
-	if strings.Contains(instance.URL, ".onion") {
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/search?q=test&categories=general&language=en&safe_search=1&page=1&format=json", instance.URL), nil)
-		printDebug("Skipping .onion instance: %s", instance.URL)
-		return false
-	}
-
-	client := &http.Client{
-		Timeout: 10 * time.Second,
-	}
-
-	testURL := fmt.Sprintf("%s/search?q=test&categories=general&language=en&safe_search=1&page=1&format=json", instance.URL)
-	req, err := http.NewRequest("GET", testURL, nil)
 	if err != nil {
-		printDebug("Error creating SearchXNG request for instance validation: %v", err)
+		log.Printf("Error creating request for URL: %s, Error: %v", instance.URL, err)
 		return false
 	}
-
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
-	XNGUserAgent, err := GetUserAgent("Text-Search-XNG")
-	if err != nil {
-		printWarn("Error generating User-Agent: %v", err)
-		return false
-	}
-	req.Header.Set("User-Agent", XNGUserAgent)
 
 	resp, err := client.Do(req)
 	if err != nil {
-		printDebug("Error performing request for SearchXNG instance validation: %v", err)
+		log.Printf("Error performing request for URL: %s, Error: %v", instance.URL, err)
 		return false
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		printDebug("SearchXNG Instance validation failed. StatusCode: %d", resp.StatusCode)
+		log.Printf("Instance validation failed for URL: %s, StatusCode: %d", instance.URL, resp.StatusCode)
 		return false
 	}
 
-	// Successful validation
 	return true
 }
 
-// GetValidInstance fetches and validates SearX instances, returning a valid one.
 func getValidInstance() (*Instance, error) {
 	instances, err := fetchInstances()
 	if err != nil {
@@ -181,81 +88,11 @@ func getValidInstance() (*Instance, error) {
 	return nil, fmt.Errorf("no valid SearX instances found")
 }
 
-// PerformSearXTextSearch performs a text search using a SearX instance.
+// func main() {
-func PerformSearXTextSearch(query, categories, language string, page int) ([]TextSearchResult, time.Duration, error) {
+// 	instance, err := getValidInstance()
-	// Default value for "safe" search
+// 	if err != nil {
-	safe := "1"
+// 		log.Fatalf("Failed to get a valid SearX instance: %v", err)
+// 	}
 
-	startTime := time.Now() // Start the timer
+// 	log.Printf("Selected SearX instance: %s", instance.URL)
-	var results []TextSearchResult
+// }
-
-	instance, err := getValidInstance()
-	if err != nil {
-		return nil, 0, fmt.Errorf("failed to get a valid SearX instance: %w", err)
-	}
-
-	searchURL := fmt.Sprintf("%s/search?q=%s&categories=%s&language=%s&safe_search=%s&page=%d&format=json",
-		instance.URL, url.QueryEscape(query), categories, language, safe, page)
-
-	req, err := http.NewRequest("GET", searchURL, nil)
-	if err != nil {
-		return nil, 0, fmt.Errorf("creating request: %v", err)
-	}
-
-	XNGUserAgent, err := GetUserAgent("Text-Search-XNG")
-	if err != nil {
-		return nil, 0, fmt.Errorf("generating User-Agent: %v", err)
-	}
-	req.Header.Set("User-Agent", XNGUserAgent)
-
-	var resp *http.Response
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{Timeout: 10 * time.Second}
-		resp, err = client.Do(req)
-	}
-	if err != nil {
-		return nil, 0, fmt.Errorf("performing request: %v", err)
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, 0, fmt.Errorf("reading response body: %v", err)
-	}
-
-	// Parse the JSON response to extract search results
-	var response map[string]interface{}
-	err = json.Unmarshal(body, &response)
-	if err != nil {
-		return nil, 0, fmt.Errorf("parsing response JSON: %v", err)
-	}
-
-	// Extract search results
-	if items, ok := response["results"].([]interface{}); ok {
-		for _, item := range items {
-			if result, ok := item.(map[string]interface{}); ok {
-				title := strings.TrimSpace(fmt.Sprintf("%v", result["title"]))
-				url := strings.TrimSpace(fmt.Sprintf("%v", result["url"]))
-				description := strings.TrimSpace(fmt.Sprintf("%v", result["content"]))
-
-				results = append(results, TextSearchResult{
-					Header:      title,
-					URL:         url,
-					Description: description,
-				})
-			}
-		}
-	}
-
-	duration := time.Since(startTime) // Calculate the duration
-
-	if len(results) == 0 {
-		printDebug("No results found for query: %s", query)
-		return nil, duration, fmt.Errorf("no results found")
-	}
-
-	printDebug("Search completed successfully for query: %s, found %d results", query, len(results))
-	return results, duration, nil
-}

text.go

@@ -14,7 +14,7 @@ func init() {
 		{Name: "LibreX", Func: wrapTextSearchFunc(PerformLibreXTextSearch)},
 		{Name: "Brave", Func: wrapTextSearchFunc(PerformBraveTextSearch)},
 		{Name: "DuckDuckGo", Func: wrapTextSearchFunc(PerformDuckDuckGoTextSearch)},
-		//  {Name: "SearXNG", Func: wrapTextSearchFunc(PerformSearXTextSearch)}, // Always says StatusCode: 429
+		// {Name: "SearXNG", Func: wrapTextSearchFunc(PerformSearXNGTextSearch), Weight: 2}, // Uncomment when implemented
 	}
 }
 
@@ -98,7 +98,7 @@ func prefetchPage(query, safe, lang string, page int) {
 	cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "active", Lang: lang, Type: "text"}
 	if _, exists := resultsCache.Get(cacheKey); !exists {
 		printInfo("Page %d not cached, caching now...", page)
-		if config.MetaSearchEnabled {
+		if config.CrawlerEnabled {
 			pageResults := fetchTextResults(query, safe, lang, page)
 			if len(pageResults) > 0 {
 				resultsCache.Set(cacheKey, convertToSearchResults(pageResults))
@@ -114,7 +114,7 @@ func prefetchPage(query, safe, lang string, page int) {
 func fetchTextResults(query, safe, lang string, page int) []TextSearchResult {
 	var results []TextSearchResult
 
-	if !config.MetaSearchEnabled {
+	if !config.CrawlerEnabled {
 		printDebug("Crawler is disabled; fetching from local index.")
 
 		// Calculate the starting position based on the page number

user-settings.go

@@ -72,9 +72,8 @@ func saveUserSettings(w http.ResponseWriter, settings UserSettings) {
 			Value:    cd.GetValue(settings),
 			Path:     "/",
 			Expires:  expiration,
-			Secure:   true, // Ensure HTTPS is required
+			Secure:   true,
-			HttpOnly: true,
+			SameSite: http.SameSiteStrictMode,
-			SameSite: http.SameSiteStrictMode, // Restrict cross-site usage
 		})
 	}
 

video.go

@@ -151,7 +151,7 @@ func handleVideoSearch(w http.ResponseWriter, settings UserSettings, query strin
 	start := time.Now()
 
 	var results []VideoResult
-	if config.MetaSearchEnabled {
+	if config.CrawlerEnabled {
 		results = fetchVideoResults(query, settings.SafeSearch, settings.SearchLanguage, page)
 	}
 
@@ -184,12 +184,12 @@ func handleVideoSearch(w http.ResponseWriter, settings UserSettings, query strin
 
 func fetchVideoResults(query, safe, lang string, page int) []VideoResult {
 	// Check if the crawler is enabled
-	if !config.MetaSearchEnabled {
+	if !config.CrawlerEnabled {
 		printDebug("Crawler is disabled; skipping video search.")
 		return []VideoResult{}
 	}
 
-	// Proceed with Piped API request if MetaSearchEnabled
+	// Proceed with Piped API request if CrawlerEnabled
 	apiResp, err := makeHTMLRequest(query, safe, lang, page)
 	if err != nil {
 		printWarn("Error fetching video results: %v", err)