added ProxyRetry to config and fixed ProxyStrict

proxy.go

@@ -30,7 +30,7 @@ var (
 	crawlerProxyClient *ProxyClient
 )
 
-// NewProxyClientPool creates a pool of HTTP clients with proxies.
+// NewProxyClientPool creates a pool of HTTP clients with SOCKS5 proxies.
 func NewProxyClientPool(proxies []ProxyConfig, timeout time.Duration) (*ProxyClient, error) {
 	if len(proxies) == 0 {
 		return nil, fmt.Errorf("no proxies provided")
@@ -38,18 +38,17 @@ func NewProxyClientPool(proxies []ProxyConfig, timeout time.Duration) (*ProxyCli
 
 	clients := make([]*http.Client, len(proxies))
 
-	for i, proxyConfig := range proxies {
+	for i, pc := range proxies {
 		var auth *proxy.Auth
-		if proxyConfig.Username != "" || proxyConfig.Password != "" {
+		if pc.Username != "" || pc.Password != "" {
 			auth = &proxy.Auth{
-				User:     proxyConfig.Username,
-				Password: proxyConfig.Password,
+				User:     pc.Username,
+				Password: pc.Password,
 			}
 		}
-
-		dialer, err := proxy.SOCKS5("tcp", proxyConfig.Address, auth, proxy.Direct)
+		dialer, err := proxy.SOCKS5("tcp", pc.Address, auth, proxy.Direct)
 		if err != nil {
-			return nil, fmt.Errorf("failed to create SOCKS5 dialer for %s: %w", proxyConfig.Address, err)
+			return nil, fmt.Errorf("failed to create SOCKS5 dialer for %s: %w", pc.Address, err)
 		}
 
 		transport := &http.Transport{Dial: dialer.Dial}
@@ -99,20 +98,21 @@ func (p *ProxyClient) GetProxy() string {
 // ParseProxies parses the proxy strings in the format ADDRESS:PORT or ADDRESS:PORT:USER:PASSWORD.
 func ParseProxies(proxyStrings []string) []ProxyConfig {
 	var proxies []ProxyConfig
-	for _, proxy := range proxyStrings {
-		parts := strings.Split(proxy, ":")
-		if len(parts) == 2 { // ADDRESS:PORT
+	for _, proxyStr := range proxyStrings {
+		parts := strings.Split(proxyStr, ":")
+		switch len(parts) {
+		case 2: // ADDRESS:PORT
 			proxies = append(proxies, ProxyConfig{
 				Address: fmt.Sprintf("%s:%s", parts[0], parts[1]),
 			})
-		} else if len(parts) == 4 { // ADDRESS:PORT:USER:PASSWORD
+		case 4: // ADDRESS:PORT:USER:PASSWORD
 			proxies = append(proxies, ProxyConfig{
 				Address:  fmt.Sprintf("%s:%s", parts[0], parts[1]),
 				Username: parts[2],
 				Password: parts[3],
 			})
-		} else {
-			fmt.Printf("Invalid proxy format: %s\n", proxy)
+		default:
+			fmt.Printf("Invalid proxy format: %s\n", proxyStr)
 		}
 	}
 	return proxies
@@ -147,6 +147,107 @@ func InitProxies() {
 	}
 }
 
+// Doer is an interface so we can accept *http.Client or *ProxyClient for requests.
+type Doer interface {
+	Do(*http.Request) (*http.Response, error)
+}
+
+// DoProxyRequest handles “try direct, then proxy if needed,” with retries if proxy is used.
+//
+// - strict: if true, always try proxy first if enabled; if not available, do one direct attempt
+// - enabled: whether this type of proxy is turned on
+// - retryCount: how many times to retry with the proxy
+// - proxyClient: the pool of proxy connections
+func DoProxyRequest(req *http.Request, strict bool, enabled bool, retryCount int, proxyClient *ProxyClient) (*http.Response, error) {
+	// 1) If !strict => try direct once first
+	if !strict {
+		resp, err := tryRequestOnce(req, http.DefaultClient)
+		if isSuccessful(resp, err) {
+			return resp, nil
+		}
+		// If direct fails => if proxy is enabled, retry
+		if enabled && proxyClient != nil {
+			resp, err = tryRequestWithRetry(req, proxyClient, retryCount)
+			if isSuccessful(resp, err) {
+				return resp, nil
+			}
+			return nil, fmt.Errorf("failed after direct & proxy attempts: %v", err)
+		}
+		return nil, fmt.Errorf("request failed direct, no valid proxy: %v", err)
+	}
+
+	// 2) If strict => if proxy is enabled, try it up to “retryCount”
+	if enabled && proxyClient != nil {
+		resp, err := tryRequestWithRetry(req, proxyClient, retryCount)
+		if isSuccessful(resp, err) {
+			return resp, nil
+		}
+		return nil, fmt.Errorf("failed after %d proxy attempts: %v", retryCount, err)
+	}
+
+	// If strict but no proxy => direct once
+	resp, err := tryRequestOnce(req, http.DefaultClient)
+	if isSuccessful(resp, err) {
+		return resp, nil
+	}
+	return nil, fmt.Errorf("direct request failed in strict mode, no proxy: %v", err)
+}
+
+// Helper Wrapper functions for DoProxyRequest()
+func DoMetaProxyRequest(req *http.Request) (*http.Response, error) {
+	return DoProxyRequest(
+		req,
+		config.MetaProxyStrict,
+		config.MetaProxyEnabled,
+		config.MetaProxyRetry,
+		metaProxyClient,
+	)
+}
+func DoCrawlerProxyRequest(req *http.Request) (*http.Response, error) {
+	return DoProxyRequest(
+		req,
+		config.CrawlerProxyStrict,
+		config.CrawlerProxyEnabled,
+		config.CrawlerProxyRetry,
+		metaProxyClient,
+	)
+}
+
+// tryRequestWithRetry tries the request up to "retries" times, waiting 200ms between attempts.
+func tryRequestWithRetry(req *http.Request, client Doer, retries int) (*http.Response, error) {
+	var resp *http.Response
+	var err error
+	for i := 1; i <= retries; i++ {
+		if resp != nil {
+			resp.Body.Close()
+		}
+		printDebug("Attempt %d of %d with proxy/client...", i, retries)
+		resp, err = tryRequestOnce(req, client)
+		if isSuccessful(resp, err) {
+			return resp, nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return resp, err
+}
+
+// tryRequestOnce sends a single request with the given client. If client is nil, uses default client.
+func tryRequestOnce(req *http.Request, client Doer) (*http.Response, error) {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	resp, err := client.Do(req)
+	return resp, err
+}
+
+// isSuccessful checks if err==nil & resp != nil & resp.StatusCode in [200..299].
+func isSuccessful(resp *http.Response, err error) bool {
+	if err != nil || resp == nil {
+		return false
+	}
+	return resp.StatusCode >= 200 && resp.StatusCode < 300
+}
+
 // func main() {
 // 	config := loadConfig()