added ProxyRetry to config and fixed ProxyStrict

forums.go

@@ -3,7 +3,6 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"math"
 	"net/http"
 	"net/url"
 	"time"
@@ -16,41 +15,45 @@ func PerformRedditSearch(query string, safe string, page int) ([]ForumSearchResu
 	}
 
 	const (
-		pageSize       = 25
-		baseURL        = "https://www.reddit.com"
-		maxRetries     = 5
-		initialBackoff = 2 * time.Second
+		pageSize = 25
+		baseURL  = "https://www.reddit.com"
 	)
+
 	var results []ForumSearchResult
+	offset := page * pageSize
+	searchURL := fmt.Sprintf("%s/search.json?q=%s&limit=%d&start=%d",
+		baseURL,
+		url.QueryEscape(query),
+		pageSize,
+		offset,
+	)
 
-	searchURL := fmt.Sprintf("%s/search.json?q=%s&limit=%d&start=%d", baseURL, url.QueryEscape(query), pageSize, page*pageSize)
-	var resp *http.Response
-	var err error
-
-	// Retry logic with exponential backoff
-	for i := 0; i <= maxRetries; i++ {
-		resp, err = http.Get(searchURL)
-		if err != nil {
-			return nil, fmt.Errorf("making request: %v", err)
-		}
-		if resp.StatusCode != http.StatusTooManyRequests {
-			break
-		}
-
-		// Wait for some time before retrying
-		backoff := time.Duration(math.Pow(2, float64(i))) * initialBackoff
-		time.Sleep(backoff)
+	// Create request
+	req, err := http.NewRequest("GET", searchURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("creating request: %v", err)
 	}
 
+	// Set User-Agent
+	userAgent, uaErr := GetUserAgent("Reddit-Forum-Search")
+	if uaErr != nil {
+		return nil, fmt.Errorf("getting user agent: %v", uaErr)
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	// Make request using MetaProxy logic
+	resp, err := DoMetaProxyRequest(req)
 	if err != nil {
 		return nil, fmt.Errorf("making request: %v", err)
 	}
 	defer resp.Body.Close()
 
+	// Validate response status
 	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
 
+	// Parse JSON response
 	var searchResults map[string]interface{}
 	if err := json.NewDecoder(resp.Body).Decode(&searchResults); err != nil {
 		return nil, fmt.Errorf("decoding response: %v", err)
@@ -66,9 +69,9 @@ func PerformRedditSearch(query string, safe string, page int) ([]ForumSearchResu
 		return nil, fmt.Errorf("no children field in data")
 	}
 
+	// Extract search results
 	for _, post := range posts {
 		postData := post.(map[string]interface{})["data"].(map[string]interface{})
-
 		if safe == "active" && postData["over_18"].(bool) {
 			continue
 		}
@@ -78,6 +81,7 @@ func PerformRedditSearch(query string, safe string, page int) ([]ForumSearchResu
 		if len(description) > 500 {
 			description = description[:500] + "..."
 		}
+
 		publishedDate := time.Unix(int64(postData["created_utc"].(float64)), 0)
 		permalink := postData["permalink"].(string)
 		resultURL := fmt.Sprintf("%s%s", baseURL, permalink)