83 lines
2.1 KiB
Go
83 lines
2.1 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
)
|
|
|
|
func PerformDuckDuckGoTextSearch(query, safe, lang string, page int) ([]TextSearchResult, time.Duration, error) {
|
|
startTime := time.Now() // Start the timer
|
|
|
|
var results []TextSearchResult
|
|
searchURL := buildDuckDuckGoSearchURL(query, page)
|
|
|
|
// Create a request
|
|
req, err := http.NewRequest("GET", searchURL, nil)
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("creating request: %v", err)
|
|
}
|
|
|
|
// Use proxy client if MetaProxy is enabled
|
|
var resp *http.Response
|
|
if config.MetaProxyEnabled && metaProxyClient != nil {
|
|
resp, err = metaProxyClient.Do(req)
|
|
} else {
|
|
client := &http.Client{}
|
|
resp, err = client.Do(req)
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("making request: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Check for HTTP status code
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
|
|
}
|
|
|
|
// Parse HTML response
|
|
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("loading HTML document: %v", err)
|
|
}
|
|
|
|
// Extract results from the page
|
|
doc.Find(".result__body").Each(func(i int, s *goquery.Selection) {
|
|
header := s.Find(".result__a").Text()
|
|
description := s.Find(".result__snippet").Text()
|
|
rawURL, exists := s.Find(".result__a").Attr("href")
|
|
if exists {
|
|
parsedURL, err := url.Parse(rawURL)
|
|
if err == nil {
|
|
queryParams := parsedURL.Query()
|
|
uddg := queryParams.Get("uddg")
|
|
if uddg != "" {
|
|
result := TextSearchResult{
|
|
URL: uddg,
|
|
Header: strings.TrimSpace(header),
|
|
Description: strings.TrimSpace(description),
|
|
}
|
|
results = append(results, result)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
duration := time.Since(startTime) // Calculate the duration
|
|
|
|
return results, duration, nil
|
|
}
|
|
|
|
func buildDuckDuckGoSearchURL(query string, page int) string {
|
|
startParam := ""
|
|
if page > 1 {
|
|
startParam = fmt.Sprintf("&s=%d", (page-1)*10)
|
|
}
|
|
return fmt.Sprintf("https://duckduckgo.com/html/?q=%s%s", url.QueryEscape(query), startParam)
|
|
}
|