Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues 7 Pull requests Projects Releases Packages Wiki Activity Actions

quant+google+duckdcuckgo results

Browse source
This commit is contained in:
partisan 2024-05-18 01:59:29 +02:00
parent c8cf762222
commit a3a7c69679
6 changed files with 187 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
main.go
View file

@ -104,7 +104,7 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
switch searchType {
case "text":
handleTextSearch(w, query, safe, lang)
HandleTextSearch(w, query, safe, lang)
case "image":
handleImageSearch(w, query, safe, lang, page)
case "video":

2
run.sh
View file

@ -1,3 +1,3 @@
#!/bin/bash
go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go
go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go --debug

58
text-duckduckgo.go Normal file
View file

@ -0,0 +1,58 @@
// text-duckduckgo.go
package main
import (
"fmt"
"log"
"net/http"
"net/url"
"strings"
"github.com/PuerkitoBio/goquery"
)
func PerformDuckDuckGoTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
var results []TextSearchResult
searchURL := fmt.Sprintf("https://duckduckgo.com/html/?q=%s", url.QueryEscape(query))
resp, err := http.Get(searchURL)
if err != nil {
return nil, fmt.Errorf("making request: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return nil, fmt.Errorf("loading HTML document: %v", err)
}
doc.Find(".result__body").Each(func(i int, s *goquery.Selection) {
header := s.Find(".result__a").Text()
description := s.Find(".result__snippet").Text()
rawURL, exists := s.Find(".result__a").Attr("href")
if exists {
parsedURL, err := url.Parse(rawURL)
if err == nil {
queryParams := parsedURL.Query()
uddg := queryParams.Get("uddg")
if uddg != "" {
result := TextSearchResult{
URL: uddg,
Header: strings.TrimSpace(header),
Description: strings.TrimSpace(description),
}
results = append(results, result)
if debugMode {
log.Printf("Processed DuckDuckGo result: %+v\n", result)
}
}
}
}
})
return results, nil
}

10
text-google.go
View file

@ -24,7 +24,7 @@ func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, erro
langParam = "&lr=" + lang
}
searchURL := "https://www.google.com/search?q=" + url.QueryEscape(query) + safeParam + langParam
searchURL := "https://www.google.com/search?q=" + url.QueryEscape(query) + safeParam + langParam + "&udm=14"
req, err := http.NewRequest("GET", searchURL, nil)
if err != nil {
@ -56,11 +56,15 @@ func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, erro
description = descSelection.Text()
}
results = append(results, TextSearchResult{
result := TextSearchResult{
URL: href,
Header: header,
Description: description,
})
}
results = append(results, result)
if debugMode {
log.Printf("Google result: %+v\n", result)
}
})
return results, nil

48
text-quant.go
View file

@ -1,4 +1,3 @@
// text-quant.go
package main
import (
@ -13,22 +12,39 @@ import (
type QwantTextAPIResponse struct {
Data struct {
Result struct {
Items struct {
Mainline []struct {
Items []struct {
URL string `json:"url"`
Title string `json:"title"`
Url string `json:"url"`
Snippet string `json:"desc"`
Description string `json:"desc"`
} `json:"items"`
} `json:"mainline"`
} `json:"items"`
} `json:"result"`
} `json:"data"`
}
// PerformQwantTextSearch contacts the Qwant API and returns a slice of TextSearchResult
func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
const resultsPerPage = 10
apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?t=web&q=%s&count=%d&locale=%s&safesearch=%s",
const offset = 0
// Ensure safe search is disabled by default if not specified
if safe == "" {
safe = "0"
}
// Default to English Canada locale if not specified
if lang == "" {
lang = "en_CA"
}
apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?q=%s&count=%d&locale=%s&offset=%d&device=desktop",
url.QueryEscape(query),
resultsPerPage,
lang,
safe)
offset)
client := &http.Client{Timeout: 10 * time.Second}
@ -54,14 +70,30 @@ func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error
return nil, fmt.Errorf("decoding response: %v", err)
}
// Extracting results from the nested JSON structure
if len(apiResp.Data.Result.Items.Mainline) == 0 {
return nil, fmt.Errorf("no search results found")
}
var results []TextSearchResult
for _, item := range apiResp.Data.Result.Items {
for _, item := range apiResp.Data.Result.Items.Mainline[0].Items {
cleanURL := cleanQwantURL(item.URL)
results = append(results, TextSearchResult{
URL: item.Url,
URL: cleanURL,
Header: item.Title,
Description: item.Snippet,
Description: item.Description,
Source: "Qwant",
})
}
return results, nil
}
// cleanQwantURL extracts the main part of the URL, removing tracking information
func cleanQwantURL(rawURL string) string {
u, err := url.Parse(rawURL)
if err != nil {
return rawURL
}
return u.Scheme + "://" + u.Host + u.Path
}

99
text.go
View file

@ -1,12 +1,13 @@
// text.go
package main
import (
"flag"
"fmt"
"html/template"
"log"
"net/http"
"sort"
"sync"
"time"
)
@ -14,49 +15,101 @@ type TextSearchResult struct {
URL string
Header string
Description string
Source string
}
func handleTextSearch(w http.ResponseWriter, query, safe, lang string) {
googleResults, googleErr := PerformGoogleTextSearch(query, safe, lang)
if googleErr != nil {
log.Printf("Error performing Google text search: %v", googleErr)
var debugMode bool
func init() {
flag.BoolVar(&debugMode, "debug", false, "enable debug mode")
flag.Parse()
}
func HandleTextSearch(w http.ResponseWriter, query, safe, lang string) {
startTime := time.Now()
var combinedResults []TextSearchResult
var resultMap = make(map[string]TextSearchResult)
var wg sync.WaitGroup
var mu sync.Mutex
resultsChan := make(chan []TextSearchResult)
searchFuncs := []struct {
Func func(string, string, string) ([]TextSearchResult, error)
Source string
}{
{PerformGoogleTextSearch, "Google"},
{PerformDuckDuckGoTextSearch, "DuckDuckGo"},
{PerformQwantTextSearch, "Qwant"},
}
qwantResults, qwantErr := PerformQwantTextSearch(query, safe, lang)
if qwantErr != nil {
log.Printf("Error performing Qwant text search: %v", qwantErr)
wg.Add(len(searchFuncs))
for _, searchFunc := range searchFuncs {
go func(searchFunc func(string, string, string) ([]TextSearchResult, error), source string) {
defer wg.Done()
results, err := searchFunc(query, safe, lang)
if err == nil {
for i := range results {
results[i].Source = source
}
resultsChan <- results
} else {
log.Printf("Error performing search from %s: %v", source, err)
}
}(searchFunc.Func, searchFunc.Source)
}
// Use a map to track URLs and prioritize Qwant results
resultMap := make(map[string]TextSearchResult)
go func() {
wg.Wait()
close(resultsChan)
}()
// Add Qwant results to the map first
for _, result := range qwantResults {
for results := range resultsChan {
mu.Lock()
for _, result := range results {
existingResult, exists := resultMap[result.URL]
if !exists || shouldReplace(existingResult.Source, result.Source) {
resultMap[result.URL] = result
}
// Add Google results to the map if the URL is not already present
for _, result := range googleResults {
if _, exists := resultMap[result.URL]; !exists {
resultMap[result.URL] = result
if debugMode {
log.Printf("Result from %s: %+v\n", result.Source, result)
}
}
mu.Unlock()
}
// Convert the map back to a slice
var combinedResults []TextSearchResult
for _, result := range resultMap {
combinedResults = append(combinedResults, result)
}
// Sort results (optional, based on some criteria)
// Custom sorting: Google first, DuckDuckGo second, Qwant third
sort.SliceStable(combinedResults, func(i, j int) bool {
return combinedResults[i].Header < combinedResults[j].Header
return sourceOrder(combinedResults[i].Source) < sourceOrder(combinedResults[j].Source)
})
displayResults(w, combinedResults, query, lang)
displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds())
}
func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string) {
func shouldReplace(existingSource, newSource string) bool {
return sourceOrder(newSource) < sourceOrder(existingSource)
}
func sourceOrder(source string) int {
switch source {
case "Qwant":
return 1
case "DuckDuckGo":
return 2
case "Google":
return 3
default:
return 4
}
}
func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string, elapsed float64) {
tmpl, err := template.ParseFiles("templates/text.html")
if err != nil {
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@ -72,7 +125,7 @@ func displayResults(w http.ResponseWriter, results []TextSearchResult, query, la
}{
Results: results,
Query: query,
Fetched: fmt.Sprintf("%.2f seconds", time.Since(time.Now()).Seconds()),
Fetched: fmt.Sprintf("%.2f seconds", elapsed),
LanguageOptions: languageOptions,
CurrentLang: lang,
}