added SOCKS5 proxy support

proxy.go

@@ -0,0 +1,169 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/net/proxy"
+)
+
+// ProxyConfig holds configuration for a single proxy.
+type ProxyConfig struct {
+	Address  string
+	Username string
+	Password string
+}
+
+// ProxyClient provides an HTTP client pool for proxies.
+type ProxyClient struct {
+	clients []*http.Client
+	lock    sync.Mutex
+	index   int
+}
+
+// Package-level proxy clients
+var (
+	metaProxyClient    *ProxyClient
+	crawlerProxyClient *ProxyClient
+)
+
+// NewProxyClientPool creates a pool of HTTP clients with proxies.
+func NewProxyClientPool(proxies []ProxyConfig, timeout time.Duration) (*ProxyClient, error) {
+	if len(proxies) == 0 {
+		return nil, fmt.Errorf("no proxies provided")
+	}
+
+	clients := make([]*http.Client, len(proxies))
+
+	for i, proxyConfig := range proxies {
+		var auth *proxy.Auth
+		if proxyConfig.Username != "" || proxyConfig.Password != "" {
+			auth = &proxy.Auth{
+				User:     proxyConfig.Username,
+				Password: proxyConfig.Password,
+			}
+		}
+
+		dialer, err := proxy.SOCKS5("tcp", proxyConfig.Address, auth, proxy.Direct)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create SOCKS5 dialer for %s: %w", proxyConfig.Address, err)
+		}
+
+		transport := &http.Transport{Dial: dialer.Dial}
+		clients[i] = &http.Client{
+			Transport: transport,
+			Timeout:   timeout,
+		}
+	}
+
+	return &ProxyClient{clients: clients}, nil
+}
+
+// Do sends an HTTP request using the next proxy in the pool.
+func (p *ProxyClient) Do(req *http.Request) (*http.Response, error) {
+	p.lock.Lock()
+	client := p.clients[p.index]
+	p.index = (p.index + 1) % len(p.clients)
+	p.lock.Unlock()
+	return client.Do(req)
+}
+
+func (p *ProxyClient) GetProxy() string {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	if len(p.clients) == 0 {
+		return ""
+	}
+
+	// Round-robin proxy retrieval
+	client := p.clients[p.index]
+	p.index = (p.index + 1) % len(p.clients)
+
+	// Assume each client has a proxy string saved
+	// Example implementation depends on how your proxies are configured
+	proxyTransport, ok := client.Transport.(*http.Transport)
+	if ok && proxyTransport.Proxy != nil {
+		proxyURL, _ := proxyTransport.Proxy(nil)
+		if proxyURL != nil {
+			return proxyURL.String()
+		}
+	}
+
+	return ""
+}
+
+// ParseProxies parses the proxy strings in the format ADDRESS:PORT or ADDRESS:PORT:USER:PASSWORD.
+func ParseProxies(proxyStrings []string) []ProxyConfig {
+	var proxies []ProxyConfig
+	for _, proxy := range proxyStrings {
+		parts := strings.Split(proxy, ":")
+		if len(parts) == 2 { // ADDRESS:PORT
+			proxies = append(proxies, ProxyConfig{
+				Address: fmt.Sprintf("%s:%s", parts[0], parts[1]),
+			})
+		} else if len(parts) == 4 { // ADDRESS:PORT:USER:PASSWORD
+			proxies = append(proxies, ProxyConfig{
+				Address:  fmt.Sprintf("%s:%s", parts[0], parts[1]),
+				Username: parts[2],
+				Password: parts[3],
+			})
+		} else {
+			fmt.Printf("Invalid proxy format: %s\n", proxy)
+		}
+	}
+	return proxies
+}
+
+// InitProxies initializes the proxy clients for Meta and Crawler proxies.
+func InitProxies() {
+	// Initialize Meta Proxy Client
+	if config.MetaProxyEnabled {
+		metaProxies := ParseProxies(config.MetaProxies)
+		client, err := NewProxyClientPool(metaProxies, 30*time.Second)
+		if err != nil {
+			if config.MetaProxyStrict {
+				panic(fmt.Sprintf("Failed to initialize Meta proxies: %v", err))
+			}
+			fmt.Printf("Warning: Meta proxy initialization failed: %v\n", err)
+		}
+		metaProxyClient = client
+	}
+
+	// Initialize Crawler Proxy Client
+	if config.CrawlerProxyEnabled {
+		crawlerProxies := ParseProxies(config.CrawlerProxies)
+		client, err := NewProxyClientPool(crawlerProxies, 30*time.Second)
+		if err != nil {
+			if config.CrawlerProxyStrict {
+				panic(fmt.Sprintf("Failed to initialize Crawler proxies: %v", err))
+			}
+			fmt.Printf("Warning: Crawler proxy initialization failed: %v\n", err)
+		}
+		crawlerProxyClient = client
+	}
+}
+
+// func main() {
+// 	config := loadConfig()
+
+// 	// Initialize proxies if enabled
+// 	if config.CrawlerProxyEnabled || config.MetaProxyEnabled {
+// 		InitProxies()
+// 	}
+
+// 	// Example usage
+// 	if metaProxyClient != nil {
+// 		req, _ := http.NewRequest("GET", "https://example.com", nil)
+// 		resp, err := metaProxyClient.Do(req)
+// 		if err != nil {
+// 			fmt.Printf("Error using MetaProxyClient: %v\n", err)
+// 		} else {
+// 			fmt.Printf("Meta Proxy Response Status: %s\n", resp.Status)
+// 			resp.Body.Close()
+// 		}
+// 	}
+// }