261 lines
7.1 KiB
Go
261 lines
7.1 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type Instance struct {
|
|
URL string `json:"-"` // Populated from map key
|
|
Analytics bool `json:"analytics"`
|
|
Comments []string `json:"comments"`
|
|
AlternativeUrls map[string]interface{} `json:"alternativeUrls"`
|
|
Main bool `json:"main"`
|
|
NetworkType string `json:"network_type"`
|
|
HTTP struct {
|
|
StatusCode int `json:"status_code"`
|
|
Error string `json:"error"`
|
|
} `json:"http"`
|
|
Version string `json:"version"`
|
|
Grade string `json:"grade"`
|
|
GradeURL string `json:"gradeUrl"`
|
|
Generator string `json:"generator"`
|
|
ContactURL FlexibleType `json:"contact_url"` // Custom type
|
|
DocsURL string `json:"docs_url"`
|
|
}
|
|
|
|
type FlexibleType struct {
|
|
StringValue string
|
|
BoolValue bool
|
|
IsString bool
|
|
}
|
|
|
|
const searxInstancesURL = "https://searx.space/data/instances.json"
|
|
|
|
// FetchInstances fetches available SearX instances from the registry.
|
|
func fetchInstances() ([]Instance, error) {
|
|
req, err := http.NewRequest("GET", searxInstancesURL, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("creating request: %v", err)
|
|
}
|
|
|
|
XNGUserAgent, err := GetUserAgent("Text-Search-XNG")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("generating User-Agent: %v", err)
|
|
}
|
|
req.Header.Set("User-Agent", XNGUserAgent)
|
|
|
|
var resp *http.Response
|
|
if config.MetaProxyEnabled && config.MetaProxyStrict && metaProxyClient != nil {
|
|
resp, err = metaProxyClient.Do(req)
|
|
} else {
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
resp, err = client.Do(req)
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("performing request: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
|
|
}
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reading response body: %v", err)
|
|
}
|
|
|
|
// Root structure of the JSON response
|
|
var root struct {
|
|
Instances map[string]Instance `json:"instances"`
|
|
}
|
|
|
|
// Unmarshal JSON into the root structure
|
|
err = json.Unmarshal(body, &root)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parsing response JSON: %v", err)
|
|
}
|
|
|
|
// Collect instances into a slice
|
|
var instances []Instance
|
|
for url, instance := range root.Instances {
|
|
instance.URL = url // Assign the URL from the map key
|
|
instances = append(instances, instance)
|
|
}
|
|
|
|
return instances, nil
|
|
}
|
|
|
|
// UnmarshalJSON implements custom unmarshalling for FlexibleType.
|
|
func (f *FlexibleType) UnmarshalJSON(data []byte) error {
|
|
// Try to unmarshal as a string
|
|
var str string
|
|
if err := json.Unmarshal(data, &str); err == nil {
|
|
f.StringValue = str
|
|
f.IsString = true
|
|
return nil
|
|
}
|
|
|
|
// Try to unmarshal as a bool
|
|
var b bool
|
|
if err := json.Unmarshal(data, &b); err == nil {
|
|
f.BoolValue = b
|
|
f.IsString = false
|
|
return nil
|
|
}
|
|
|
|
// Return an error if neither works
|
|
return fmt.Errorf("invalid FlexibleType: %s", string(data))
|
|
}
|
|
|
|
// String returns the string representation of FlexibleType.
|
|
func (f FlexibleType) String() string {
|
|
if f.IsString {
|
|
return f.StringValue
|
|
}
|
|
return fmt.Sprintf("%v", f.BoolValue)
|
|
}
|
|
|
|
// ValidateInstance checks if a SearX instance is valid by performing a test query.
|
|
func validateInstance(instance Instance) bool {
|
|
// Skip .onion instances
|
|
if strings.Contains(instance.URL, ".onion") {
|
|
printDebug("Skipping .onion instance: %s", instance.URL)
|
|
return false
|
|
}
|
|
|
|
client := &http.Client{
|
|
Timeout: 10 * time.Second,
|
|
}
|
|
|
|
testURL := fmt.Sprintf("%s/search?q=test&categories=general&language=en&safe_search=1&page=1&format=json", instance.URL)
|
|
req, err := http.NewRequest("GET", testURL, nil)
|
|
if err != nil {
|
|
printDebug("Error creating SearchXNG request for instance validation: %v", err)
|
|
return false
|
|
}
|
|
|
|
XNGUserAgent, err := GetUserAgent("Text-Search-XNG")
|
|
if err != nil {
|
|
printWarn("Error generating User-Agent: %v", err)
|
|
return false
|
|
}
|
|
req.Header.Set("User-Agent", XNGUserAgent)
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
printDebug("Error performing request for SearchXNG instance validation: %v", err)
|
|
return false
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
printDebug("SearchXNG Instance validation failed. StatusCode: %d", resp.StatusCode)
|
|
return false
|
|
}
|
|
|
|
// Successful validation
|
|
return true
|
|
}
|
|
|
|
// GetValidInstance fetches and validates SearX instances, returning a valid one.
|
|
func getValidInstance() (*Instance, error) {
|
|
instances, err := fetchInstances()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to fetch instances: %w", err)
|
|
}
|
|
|
|
for _, instance := range instances {
|
|
if validateInstance(instance) {
|
|
return &instance, nil
|
|
}
|
|
}
|
|
|
|
return nil, fmt.Errorf("no valid SearX instances found")
|
|
}
|
|
|
|
// PerformSearXTextSearch performs a text search using a SearX instance.
|
|
func PerformSearXTextSearch(query, categories, language string, page int) ([]TextSearchResult, time.Duration, error) {
|
|
// Default value for "safe" search
|
|
safe := "1"
|
|
|
|
startTime := time.Now() // Start the timer
|
|
var results []TextSearchResult
|
|
|
|
instance, err := getValidInstance()
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("failed to get a valid SearX instance: %w", err)
|
|
}
|
|
|
|
searchURL := fmt.Sprintf("%s/search?q=%s&categories=%s&language=%s&safe_search=%s&page=%d&format=json",
|
|
instance.URL, url.QueryEscape(query), categories, language, safe, page)
|
|
|
|
req, err := http.NewRequest("GET", searchURL, nil)
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("creating request: %v", err)
|
|
}
|
|
|
|
XNGUserAgent, err := GetUserAgent("Text-Search-XNG")
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("generating User-Agent: %v", err)
|
|
}
|
|
req.Header.Set("User-Agent", XNGUserAgent)
|
|
|
|
var resp *http.Response
|
|
if config.MetaProxyEnabled && metaProxyClient != nil {
|
|
resp, err = metaProxyClient.Do(req)
|
|
} else {
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
resp, err = client.Do(req)
|
|
}
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("performing request: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("reading response body: %v", err)
|
|
}
|
|
|
|
// Parse the JSON response to extract search results
|
|
var response map[string]interface{}
|
|
err = json.Unmarshal(body, &response)
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("parsing response JSON: %v", err)
|
|
}
|
|
|
|
// Extract search results
|
|
if items, ok := response["results"].([]interface{}); ok {
|
|
for _, item := range items {
|
|
if result, ok := item.(map[string]interface{}); ok {
|
|
title := strings.TrimSpace(fmt.Sprintf("%v", result["title"]))
|
|
url := strings.TrimSpace(fmt.Sprintf("%v", result["url"]))
|
|
description := strings.TrimSpace(fmt.Sprintf("%v", result["content"]))
|
|
|
|
results = append(results, TextSearchResult{
|
|
Header: title,
|
|
URL: url,
|
|
Description: description,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
duration := time.Since(startTime) // Calculate the duration
|
|
|
|
if len(results) == 0 {
|
|
printDebug("No results found for query: %s", query)
|
|
return nil, duration, fmt.Errorf("no results found")
|
|
}
|
|
|
|
printDebug("Search completed successfully for query: %s, found %d results", query, len(results))
|
|
return results, duration, nil
|
|
}
|