added SOCKS5 proxy support
All checks were successful
Run Integration Tests / test (push) Successful in 33s
All checks were successful
Run Integration Tests / test (push) Successful in 33s
This commit is contained in:
parent
234f1dd3be
commit
614ce8903e
22 changed files with 501 additions and 106 deletions
|
@ -16,37 +16,48 @@ func PerformGoogleTextSearch(query, safe, lang string, page int) ([]TextSearchRe
|
|||
|
||||
startTime := time.Now() // Start the timer
|
||||
|
||||
client := &http.Client{}
|
||||
// Build the search URL
|
||||
searchURL := buildSearchURL(query, safe, lang, page, resultsPerPage)
|
||||
|
||||
// Create a new request
|
||||
req, err := http.NewRequest("GET", searchURL, nil)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to create request: %v", err)
|
||||
}
|
||||
|
||||
// User Agent generation
|
||||
TextUserAgent, err := GetUserAgent("Text-Search")
|
||||
// Generate and set User-Agent header
|
||||
TextUserAgent, err := GetUserAgent("Google-Text-Search")
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", TextUserAgent)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
// Perform the request using proxy if MetaProxy is enabled
|
||||
var resp *http.Response
|
||||
if config.MetaProxyEnabled && metaProxyClient != nil {
|
||||
resp, err = metaProxyClient.Do(req)
|
||||
} else {
|
||||
client := &http.Client{}
|
||||
resp, err = client.Do(req)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("making request: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Check for HTTP status code
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Parse the HTML response
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("loading HTML document: %v", err)
|
||||
}
|
||||
|
||||
// Extract search results
|
||||
results = parseResults(doc)
|
||||
|
||||
duration := time.Since(startTime) // Calculate the duration
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue