Search/suggestions.go

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sort"
	"sync"
	"time"
)

// SuggestionSource represents a search suggestion source along with its latency.
type SuggestionSource struct {
	Name      string
	FetchFunc func(string) []string
	Latency   time.Duration
	mu        sync.Mutex
}

// Initialize suggestion sources with default latency values.
var suggestionSources = []SuggestionSource{
	{
		Name:      "DuckDuckGo",
		FetchFunc: fetchDuckDuckGoSuggestions,
		Latency:   50 * time.Millisecond,
	},
	{
		Name:      "Edge",
		FetchFunc: fetchEdgeSuggestions,
		Latency:   50 * time.Millisecond,
	},
	{
		Name:      "Brave",
		FetchFunc: fetchBraveSuggestions,
		Latency:   50 * time.Millisecond,
	},
	{
		Name:      "Ecosia",
		FetchFunc: fetchEcosiaSuggestions,
		Latency:   50 * time.Millisecond,
	},
	// { // Not working with fetchSuggestionsFromURL func
	// 	Name:      "Qwant",
	// 	FetchFunc: fetchQwantSuggestions,
	// 	Latency:   50 * time.Millisecond,
	// },
	{
		Name:      "Startpage",
		FetchFunc: fetchStartpageSuggestions,
		Latency:   50 * time.Millisecond,
	},
	{
		Name:      "Yahoo",
		FetchFunc: fetchYahooSuggestions,
		Latency:   50 * time.Millisecond,
	},
	// I advise against it, but you can use it if you want to
	// {
	// 	Name:      "Google",
	// 	FetchFunc: fetchGoogleSuggestions,
	// 	Latency:   500 * time.Millisecond,
	// },
}

// Mutex to protect the suggestionSources during sorting.
var suggestionsMU sync.Mutex

func handleSuggestions(w http.ResponseWriter, r *http.Request) {
	query := r.URL.Query().Get("q")
	if query == "" {
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, `["",[]]`)
		return
	}

	// Sort the suggestion sources based on their latency.
	suggestionsMU.Lock()
	sort.Slice(suggestionSources, func(i, j int) bool {
		return suggestionSources[i].Latency < suggestionSources[j].Latency
	})
	suggestionsMU.Unlock()

	var suggestions []string
	for i := range suggestionSources {
		source := &suggestionSources[i]
		start := time.Now()
		suggestions = source.FetchFunc(query)
		elapsed := time.Since(start)

		updateLatency(source, elapsed)

		if len(suggestions) > 0 {
			printDebug("Suggestions found using %s", source.Name)
			break
		} else {
			printWarn("%s did not return any suggestions or failed.", source.Name)
		}
	}

	// Trim the suggestions to a maximum of 8 items
	suggestions = trimSuggestions(suggestions)

	if len(suggestions) == 0 {
		printErr("All suggestion services failed. Returning empty response.")
	}

	// Return the final suggestions as JSON.
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, `["",%s]`, toJSONStringArray(suggestions))
}

// trimSuggestions trims the suggestion list to a maximum of 8 suggestions.
func trimSuggestions(suggestions []string) []string {
	if len(suggestions) > 8 {
		return suggestions[:8]
	}
	return suggestions
}

// updateLatency updates the latency of a suggestion source using an exponential moving average.
func updateLatency(source *SuggestionSource, newLatency time.Duration) {
	source.mu.Lock()
	defer source.mu.Unlock()
	const alpha = 0.5 // Smoothing factor.
	source.Latency = time.Duration(float64(source.Latency)*(1-alpha) + float64(newLatency)*alpha)
}

func fetchGoogleSuggestions(query string) []string {
	encodedQuery := url.QueryEscape(query)
	url := fmt.Sprintf("http://suggestqueries.google.com/complete/search?client=firefox&q=%s", encodedQuery)
	printDebug("Fetching suggestions from Google: %s", url)
	return fetchSuggestionsFromURL(url)
}

func fetchDuckDuckGoSuggestions(query string) []string {
	encodedQuery := url.QueryEscape(query)
	url := fmt.Sprintf("https://duckduckgo.com/ac/?q=%s&type=list", encodedQuery)
	printDebug("Fetching suggestions from DuckDuckGo: %s", url)
	return fetchSuggestionsFromURL(url)
}

func fetchEdgeSuggestions(query string) []string {
	encodedQuery := url.QueryEscape(query)
	url := fmt.Sprintf("https://api.bing.com/osjson.aspx?query=%s", encodedQuery)
	printDebug("Fetching suggestions from Edge (Bing): %s", url)
	return fetchSuggestionsFromURL(url)
}

func fetchBraveSuggestions(query string) []string {
	encodedQuery := url.QueryEscape(query)
	url := fmt.Sprintf("https://search.brave.com/api/suggest?q=%s", encodedQuery)
	printDebug("Fetching suggestions from Brave: %s", url)
	return fetchSuggestionsFromURL(url)
}

func fetchEcosiaSuggestions(query string) []string {
	encodedQuery := url.QueryEscape(query)
	url := fmt.Sprintf("https://ac.ecosia.org/?q=%s&type=list", encodedQuery)
	printDebug("Fetching suggestions from Ecosia: %s", url)
	return fetchSuggestionsFromURL(url)
}

// Is this working?
func fetchQwantSuggestions(query string) []string {
	encodedQuery := url.QueryEscape(query)
	url := fmt.Sprintf("https://api.qwant.com/v3/suggest?q=%s", encodedQuery)
	printDebug("Fetching suggestions from Qwant: %s", url)
	return fetchSuggestionsFromURL(url)
}

func fetchStartpageSuggestions(query string) []string {
	encodedQuery := url.QueryEscape(query)
	url := fmt.Sprintf("https://startpage.com/suggestions?q=%s", encodedQuery)
	printDebug("Fetching suggestions from Startpage: %s", url)
	return fetchSuggestionsFromURL(url)
}

func fetchYahooSuggestions(query string) []string {
	encodedQuery := url.QueryEscape(query)
	url := fmt.Sprintf("https://search.yahoo.com/sugg/gossip/gossip-us-ura/?output=fxjson&command=%s", encodedQuery)
	printDebug("Fetching suggestions from Yahoo: %s", url)
	return fetchSuggestionsFromURL(url)
}

// func fetchBaiduSuggestions(query string) []string {
// 	encodedQuery := url.QueryEscape(query)
// 	url := fmt.Sprintf("https://suggestion.baidu.com/su?wd=%s", encodedQuery)
// 	printDebug("Fetching suggestions from Baidu: %s", url)
// 	return fetchSuggestionsFromURL(url)
// }

func fetchSuggestionsFromURL(url string) []string {
	resp, err := http.Get(url)
	if err != nil {
		printWarn("Error fetching suggestions from %s: %v", url, err)
		return []string{}
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		printWarn("Error reading response body from %s: %v", url, err)
		return []string{}
	}

	// Log the Content-Type for debugging.
	contentType := resp.Header.Get("Content-Type")
	printDebug("Response Content-Type from %s: %s", url, contentType)

	// Check if the body is non-empty.
	if len(body) == 0 {
		printWarn("Received empty response body from %s", url)
		return []string{}
	}

	// Attempt to parse the response as JSON regardless of Content-Type.
	var parsedResponse []interface{}
	if err := json.Unmarshal(body, &parsedResponse); err != nil {
		printErr("Error parsing JSON from %s: %v", url, err)
		printDebug("Response body: %s", string(body))
		return []string{}
	}

	// Ensure the response structure is as expected.
	if len(parsedResponse) < 2 {
		printWarn("Unexpected response format from %v: %v", url, string(body))
		return []string{}
	}

	suggestions := []string{}
	if items, ok := parsedResponse[1].([]interface{}); ok {
		for _, item := range items {
			if suggestion, ok := item.(string); ok {
				suggestions = append(suggestions, suggestion)
			}
		}
	} else {
		printErr("Unexpected suggestions format in response from: %v", url)
	}

	return suggestions
}

// toJSONStringArray converts a slice of strings to a JSON array string.
func toJSONStringArray(strings []string) string {
	result := ""
	for i, str := range strings {
		result += fmt.Sprintf(`"%s"`, str)
		if i < len(strings)-1 {
			result += ","
		}
	}
	return "[" + result + "]"
}

// func testSuggestionSources(query string) {
// 	for _, source := range suggestionSources {
// 		fmt.Printf("Testing %s...\n", source.Name)

// 		// Fetch suggestions
// 		suggestions := source.FetchFunc(query)

// 		// If we get results, print them
// 		if len(suggestions) > 0 {
// 			fmt.Printf("Suggestions from %s:\n", source.Name)
// 			for i, suggestion := range suggestions {
// 				fmt.Printf("%d: %s\n", i+1, suggestion)
// 			}
// 		} else {
// 			fmt.Printf("No suggestions from %s.\n", source.Name)
// 		}

// 		// Small separator for clarity
// 		fmt.Println("--------------------------")
// 	}
// }

// func main() {
// 	query := "test query"
// 	testSuggestionSources(query)
// }