quant+google+duckdcuckgo results

main.go

@@ -104,7 +104,7 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
 
 	switch searchType {
 	case "text":
-		handleTextSearch(w, query, safe, lang)
+		HandleTextSearch(w, query, safe, lang)
 	case "image":
 		handleImageSearch(w, query, safe, lang, page)
 	case "video":

run.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go
+go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go --debug

text-duckduckgo.go

@@ -0,0 +1,58 @@
+// text-duckduckgo.go
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+func PerformDuckDuckGoTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
+	var results []TextSearchResult
+	searchURL := fmt.Sprintf("https://duckduckgo.com/html/?q=%s", url.QueryEscape(query))
+
+	resp, err := http.Get(searchURL)
+	if err != nil {
+		return nil, fmt.Errorf("making request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("loading HTML document: %v", err)
+	}
+
+	doc.Find(".result__body").Each(func(i int, s *goquery.Selection) {
+		header := s.Find(".result__a").Text()
+		description := s.Find(".result__snippet").Text()
+		rawURL, exists := s.Find(".result__a").Attr("href")
+		if exists {
+			parsedURL, err := url.Parse(rawURL)
+			if err == nil {
+				queryParams := parsedURL.Query()
+				uddg := queryParams.Get("uddg")
+				if uddg != "" {
+					result := TextSearchResult{
+						URL:         uddg,
+						Header:      strings.TrimSpace(header),
+						Description: strings.TrimSpace(description),
+					}
+					results = append(results, result)
+					if debugMode {
+						log.Printf("Processed DuckDuckGo result: %+v\n", result)
+					}
+				}
+			}
+		}
+	})
+
+	return results, nil
+}

text-google.go

@@ -24,7 +24,7 @@ func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, erro
 		langParam = "&lr=" + lang
 	}
 
-	searchURL := "https://www.google.com/search?q=" + url.QueryEscape(query) + safeParam + langParam
+	searchURL := "https://www.google.com/search?q=" + url.QueryEscape(query) + safeParam + langParam + "&udm=14"
 
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
@@ -56,11 +56,15 @@ func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, erro
 			description = descSelection.Text()
 		}
 
-		results = append(results, TextSearchResult{
+		result := TextSearchResult{
 			URL:         href,
 			Header:      header,
 			Description: description,
-		})
+		}
+		results = append(results, result)
+		if debugMode {
+			log.Printf("Google result: %+v\n", result)
+		}
 	})
 
 	return results, nil

text-quant.go

@@ -1,4 +1,3 @@
-// text-quant.go
 package main
 
 import (
@@ -13,22 +12,39 @@ import (
 type QwantTextAPIResponse struct {
 	Data struct {
 		Result struct {
-			Items []struct {
+			Items struct {
-				Title   string `json:"title"`
+				Mainline []struct {
-				Url     string `json:"url"`
+					Items []struct {
-				Snippet string `json:"desc"`
+						URL         string `json:"url"`
+						Title       string `json:"title"`
+						Description string `json:"desc"`
+					} `json:"items"`
+				} `json:"mainline"`
 			} `json:"items"`
 		} `json:"result"`
 	} `json:"data"`
 }
 
+// PerformQwantTextSearch contacts the Qwant API and returns a slice of TextSearchResult
 func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
 	const resultsPerPage = 10
-	apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?t=web&q=%s&count=%d&locale=%s&safesearch=%s",
+	const offset = 0
+
+	// Ensure safe search is disabled by default if not specified
+	if safe == "" {
+		safe = "0"
+	}
+
+	// Default to English Canada locale if not specified
+	if lang == "" {
+		lang = "en_CA"
+	}
+
+	apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?q=%s&count=%d&locale=%s&offset=%d&device=desktop",
 		url.QueryEscape(query),
 		resultsPerPage,
 		lang,
-		safe)
+		offset)
 
 	client := &http.Client{Timeout: 10 * time.Second}
 
@@ -54,14 +70,30 @@ func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error
 		return nil, fmt.Errorf("decoding response: %v", err)
 	}
 
+	// Extracting results from the nested JSON structure
+	if len(apiResp.Data.Result.Items.Mainline) == 0 {
+		return nil, fmt.Errorf("no search results found")
+	}
+
 	var results []TextSearchResult
-	for _, item := range apiResp.Data.Result.Items {
+	for _, item := range apiResp.Data.Result.Items.Mainline[0].Items {
+		cleanURL := cleanQwantURL(item.URL)
 		results = append(results, TextSearchResult{
-			URL:         item.Url,
+			URL:         cleanURL,
 			Header:      item.Title,
-			Description: item.Snippet,
+			Description: item.Description,
+			Source:      "Qwant",
 		})
 	}
 
 	return results, nil
 }
+
+// cleanQwantURL extracts the main part of the URL, removing tracking information
+func cleanQwantURL(rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+	return u.Scheme + "://" + u.Host + u.Path
+}

text.go

@@ -1,12 +1,13 @@
-// text.go
 package main
 
 import (
+	"flag"
 	"fmt"
 	"html/template"
 	"log"
 	"net/http"
 	"sort"
+	"sync"
 	"time"
 )
 
@@ -14,49 +15,101 @@ type TextSearchResult struct {
 	URL         string
 	Header      string
 	Description string
+	Source      string
 }
 
-func handleTextSearch(w http.ResponseWriter, query, safe, lang string) {
+var debugMode bool
-	googleResults, googleErr := PerformGoogleTextSearch(query, safe, lang)
+
-	if googleErr != nil {
+func init() {
-		log.Printf("Error performing Google text search: %v", googleErr)
+	flag.BoolVar(&debugMode, "debug", false, "enable debug mode")
+	flag.Parse()
+}
+
+func HandleTextSearch(w http.ResponseWriter, query, safe, lang string) {
+	startTime := time.Now()
+	var combinedResults []TextSearchResult
+	var resultMap = make(map[string]TextSearchResult)
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+
+	resultsChan := make(chan []TextSearchResult)
+
+	searchFuncs := []struct {
+		Func   func(string, string, string) ([]TextSearchResult, error)
+		Source string
+	}{
+		{PerformGoogleTextSearch, "Google"},
+		{PerformDuckDuckGoTextSearch, "DuckDuckGo"},
+		{PerformQwantTextSearch, "Qwant"},
 	}
 
-	qwantResults, qwantErr := PerformQwantTextSearch(query, safe, lang)
+	wg.Add(len(searchFuncs))
-	if qwantErr != nil {
+
-		log.Printf("Error performing Qwant text search: %v", qwantErr)
+	for _, searchFunc := range searchFuncs {
+		go func(searchFunc func(string, string, string) ([]TextSearchResult, error), source string) {
+			defer wg.Done()
+			results, err := searchFunc(query, safe, lang)
+			if err == nil {
+				for i := range results {
+					results[i].Source = source
+				}
+				resultsChan <- results
+			} else {
+				log.Printf("Error performing search from %s: %v", source, err)
+			}
+		}(searchFunc.Func, searchFunc.Source)
 	}
 
-	// Use a map to track URLs and prioritize Qwant results
+	go func() {
-	resultMap := make(map[string]TextSearchResult)
+		wg.Wait()
+		close(resultsChan)
+	}()
 
-	// Add Qwant results to the map first
+	for results := range resultsChan {
-	for _, result := range qwantResults {
+		mu.Lock()
-		resultMap[result.URL] = result
+		for _, result := range results {
-	}
+			existingResult, exists := resultMap[result.URL]
-
+			if !exists || shouldReplace(existingResult.Source, result.Source) {
-	// Add Google results to the map if the URL is not already present
+				resultMap[result.URL] = result
-	for _, result := range googleResults {
+			}
-		if _, exists := resultMap[result.URL]; !exists {
+			if debugMode {
-			resultMap[result.URL] = result
+				log.Printf("Result from %s: %+v\n", result.Source, result)
+			}
 		}
+		mu.Unlock()
 	}
 
 	// Convert the map back to a slice
-	var combinedResults []TextSearchResult
 	for _, result := range resultMap {
 		combinedResults = append(combinedResults, result)
 	}
 
-	// Sort results (optional, based on some criteria)
+	// Custom sorting: Google first, DuckDuckGo second, Qwant third
 	sort.SliceStable(combinedResults, func(i, j int) bool {
-		return combinedResults[i].Header < combinedResults[j].Header
+		return sourceOrder(combinedResults[i].Source) < sourceOrder(combinedResults[j].Source)
 	})
 
-	displayResults(w, combinedResults, query, lang)
+	displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds())
 }
 
-func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string) {
+func shouldReplace(existingSource, newSource string) bool {
+	return sourceOrder(newSource) < sourceOrder(existingSource)
+}
+
+func sourceOrder(source string) int {
+	switch source {
+	case "Qwant":
+		return 1
+	case "DuckDuckGo":
+		return 2
+	case "Google":
+		return 3
+	default:
+		return 4
+	}
+}
+
+func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string, elapsed float64) {
 	tmpl, err := template.ParseFiles("templates/text.html")
 	if err != nil {
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@@ -72,7 +125,7 @@ func displayResults(w http.ResponseWriter, results []TextSearchResult, query, la
 	}{
 		Results:         results,
 		Query:           query,
-		Fetched:         fmt.Sprintf("%.2f seconds", time.Since(time.Now()).Seconds()),
+		Fetched:         fmt.Sprintf("%.2f seconds", elapsed),
 		LanguageOptions: languageOptions,
 		CurrentLang:     lang,
 	}