added ProxyRetry to config and fixed ProxyStrict

text-quant.go

@@ -26,9 +26,11 @@ type QwantTextAPIResponse struct {
 }
 
 // PerformQwantTextSearch contacts the Qwant API and returns a slice of TextSearchResult
-func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
+func PerformQwantTextSearch(query, safe, lang string, page int) ([]TextSearchResult, time.Duration, error) {
+	startTime := time.Now()
+
 	const resultsPerPage = 10
-	const offset = 0
+	offset := (page - 1) * resultsPerPage
 
 	// Ensure safe search is disabled by default if not specified
 	if safe == "" {
@@ -40,45 +42,44 @@ func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error
 		lang = "en_CA"
 	}
 
-	apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?q=%s&count=%d&locale=%s&offset=%d&device=desktop",
+	apiURL := fmt.Sprintf(
+		"https://api.qwant.com/v3/search/web?q=%s&count=%d&locale=%s&offset=%d&device=desktop",
 		url.QueryEscape(query),
 		resultsPerPage,
 		lang,
-		offset)
+		offset,
+	)
 
 	req, err := http.NewRequest("GET", apiURL, nil)
 	if err != nil {
-		return nil, fmt.Errorf("creating request: %v", err)
-	}
-
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
-
-	// Perform the request using the appropriate client
-	var resp *http.Response
-	if config.MetaProxyEnabled && metaProxyClient != nil {
-		resp, err = metaProxyClient.Do(req)
-	} else {
-		client := &http.Client{Timeout: 10 * time.Second}
-		resp, err = client.Do(req)
+		// Return three values: nil for the slice, 0 for duration, error for the third.
+		return nil, 0, fmt.Errorf("creating request: %v", err)
 	}
 
+	userAgent, err := GetUserAgent("Quant-Text-Search")
 	if err != nil {
-		return nil, fmt.Errorf("making request: %v", err)
+		return nil, 0, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	resp, err := DoMetaProxyRequest(req)
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to do meta-request: %v", err)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+		return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
 
 	var apiResp QwantTextAPIResponse
 	if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
-		return nil, fmt.Errorf("decoding response: %v", err)
+		return nil, 0, fmt.Errorf("decoding response: %v", err)
 	}
 
 	// Extracting results from the nested JSON structure
 	if len(apiResp.Data.Result.Items.Mainline) == 0 {
-		return nil, fmt.Errorf("no search results found")
+		return nil, 0, fmt.Errorf("no search results found")
 	}
 
 	var results []TextSearchResult
@@ -92,7 +93,8 @@ func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error
 		})
 	}
 
-	return results, nil
+	duration := time.Since(startTime)
+	return results, duration, nil
 }
 
 // cleanQwantURL extracts the main part of the URL, removing tracking information