Search/text-qwant.go

package main

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// QwantTextAPIResponse represents the JSON response structure from Qwant API
type QwantTextAPIResponse struct {
	Data struct {
		Result struct {
			Items struct {
				Mainline []struct {
					Items []struct {
						URL         string `json:"url"`
						Title       string `json:"title"`
						Description string `json:"desc"`
					} `json:"items"`
				} `json:"mainline"`
			} `json:"items"`
		} `json:"result"`
	} `json:"data"`
}

// PerformQwantTextSearch contacts the Qwant API and returns a slice of TextSearchResult
func PerformQwantTextSearch(query, safe, lang string, page int) ([]TextSearchResult, time.Duration, error) {
	startTime := time.Now()
	const resultsPerPage = 10
	offset := (page - 1) * resultsPerPage

	if safe == "" {
		safe = "0"
	}
	if lang == "" {
		lang = "en_CA"
	}

	// Create URL
	apiURL := "https://api.qwant.com/v3/search/web"
	params := url.Values{}
	params.Set("q", query)
	params.Set("count", fmt.Sprint(resultsPerPage))
	params.Set("locale", lang)
	params.Set("offset", fmt.Sprint(offset))
	params.Set("device", "desktop")
	params.Set("safesearch", safe)
	fullURL := apiURL + "?" + params.Encode()

	// Create HTTP request
	req, err := http.NewRequest("GET", fullURL, nil)
	if err != nil {
		return nil, 0, fmt.Errorf("creating request: %w", err)
	}

	ua, err := GetUserAgent("Qwant-Text-Search")
	if err != nil {
		return nil, 0, fmt.Errorf("user-agent error: %w", err)
	}

	// Set headers
	req.Header.Set("User-Agent", ua)
	req.Header.Set("Accept", "application/json, text/plain, */*")
	req.Header.Set("Accept-Language", "en-US,en;q=0.5")
	req.Header.Set("Accept-Encoding", "gzip")
	req.Header.Set("DNT", "1")
	req.Header.Set("Connection", "keep-alive")
	req.Header.Set("Origin", "https://www.qwant.com")
	req.Header.Set("Referer", "https://www.qwant.com/")
	req.Header.Set("Sec-Fetch-Dest", "empty")
	req.Header.Set("Sec-Fetch-Mode", "cors")
	req.Header.Set("Sec-Fetch-Site", "same-site")
	req.Header.Set("TE", "trailers")

	// Send Request
	resp, err := DoMetaProxyRequest(req)
	if err != nil {
		return nil, 0, fmt.Errorf("failed to do meta-request: %w", err)
	}
	defer resp.Body.Close()

	// Wait for HTTP 200
	if resp.StatusCode != http.StatusOK {
		return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
	}

	var body []byte
	if strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
		gr, err := gzip.NewReader(resp.Body)
		if err != nil {
			return nil, 0, fmt.Errorf("gzip reader: %w", err)
		}
		gr.Close()
	} else {
		body, err = io.ReadAll(resp.Body)
	}
	if err != nil {
		return nil, 0, fmt.Errorf("reading body: %w", err)
	}

	// Captcha block error
	if strings.Contains(string(body), `"error_code":27`) {
		return nil, 0, fmt.Errorf("Qwant CAPTCHA triggered, request blocked")
	}

	var apiResp QwantTextAPIResponse
	if err := json.Unmarshal(body, &apiResp); err != nil {
		return nil, 0, fmt.Errorf("parsing JSON: %w", err)
	}

	if len(apiResp.Data.Result.Items.Mainline) == 0 {
		return nil, 0, fmt.Errorf("no results in mainline")
	}

	var results []TextSearchResult
	for _, item := range apiResp.Data.Result.Items.Mainline[0].Items {
		results = append(results, TextSearchResult{
			URL:         cleanQwantURL(item.URL),
			Header:      item.Title,
			Description: item.Description,
			Source:      "Qwant",
		})
	}

	return results, time.Since(startTime), nil
}

// cleanQwantURL extracts the main part of the URL, removing tracking information
func cleanQwantURL(rawURL string) string {
	u, err := url.Parse(rawURL)
	if err != nil {
		return rawURL
	}
	return u.Scheme + "://" + u.Host + u.Path
}