package main import ( "encoding/json" "fmt" "io" "net/http" "net/url" "strings" "time" ) type Instance struct { URL string `json:"-"` // Populated from map key Analytics bool `json:"analytics"` Comments []string `json:"comments"` AlternativeUrls map[string]interface{} `json:"alternativeUrls"` Main bool `json:"main"` NetworkType string `json:"network_type"` HTTP struct { StatusCode int `json:"status_code"` Error string `json:"error"` } `json:"http"` Version string `json:"version"` Grade string `json:"grade"` GradeURL string `json:"gradeUrl"` Generator string `json:"generator"` ContactURL FlexibleType `json:"contact_url"` // Custom type DocsURL string `json:"docs_url"` } type FlexibleType struct { StringValue string BoolValue bool IsString bool } const searxInstancesURL = "https://searx.space/data/instances.json" // FetchInstances fetches available SearX instances from the registry. func fetchInstances() ([]Instance, error) { req, err := http.NewRequest("GET", searxInstancesURL, nil) if err != nil { return nil, fmt.Errorf("creating request: %v", err) } XNGUserAgent, err := GetUserAgent("Text-Search-XNG") if err != nil { return nil, fmt.Errorf("generating User-Agent: %v", err) } req.Header.Set("User-Agent", XNGUserAgent) var resp *http.Response if config.MetaProxyEnabled && config.MetaProxyStrict && metaProxyClient != nil { resp, err = metaProxyClient.Do(req) } else { client := &http.Client{Timeout: 10 * time.Second} resp, err = client.Do(req) } if err != nil { return nil, fmt.Errorf("performing request: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("reading response body: %v", err) } // Root structure of the JSON response var root struct { Instances map[string]Instance `json:"instances"` } // Unmarshal JSON into the root structure err = json.Unmarshal(body, &root) if err != nil { return nil, fmt.Errorf("parsing response JSON: %v", err) } // Collect instances into a slice var instances []Instance for url, instance := range root.Instances { instance.URL = url // Assign the URL from the map key instances = append(instances, instance) } return instances, nil } // UnmarshalJSON implements custom unmarshalling for FlexibleType. func (f *FlexibleType) UnmarshalJSON(data []byte) error { // Try to unmarshal as a string var str string if err := json.Unmarshal(data, &str); err == nil { f.StringValue = str f.IsString = true return nil } // Try to unmarshal as a bool var b bool if err := json.Unmarshal(data, &b); err == nil { f.BoolValue = b f.IsString = false return nil } // Return an error if neither works return fmt.Errorf("invalid FlexibleType: %s", string(data)) } // String returns the string representation of FlexibleType. func (f FlexibleType) String() string { if f.IsString { return f.StringValue } return fmt.Sprintf("%v", f.BoolValue) } // ValidateInstance checks if a SearX instance is valid by performing a test query. func validateInstance(instance Instance) bool { // Skip .onion instances if strings.Contains(instance.URL, ".onion") { printDebug("Skipping .onion instance: %s", instance.URL) return false } client := &http.Client{ Timeout: 10 * time.Second, } testURL := fmt.Sprintf("%s/search?q=test&categories=general&language=en&safe_search=1&page=1&format=json", instance.URL) req, err := http.NewRequest("GET", testURL, nil) if err != nil { printDebug("Error creating SearchXNG request for instance validation: %v", err) return false } XNGUserAgent, err := GetUserAgent("Text-Search-XNG") if err != nil { printWarn("Error generating User-Agent: %v", err) return false } req.Header.Set("User-Agent", XNGUserAgent) resp, err := client.Do(req) if err != nil { printDebug("Error performing request for SearchXNG instance validation: %v", err) return false } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { printDebug("SearchXNG Instance validation failed. StatusCode: %d", resp.StatusCode) return false } // Successful validation return true } // GetValidInstance fetches and validates SearX instances, returning a valid one. func getValidInstance() (*Instance, error) { instances, err := fetchInstances() if err != nil { return nil, fmt.Errorf("failed to fetch instances: %w", err) } for _, instance := range instances { if validateInstance(instance) { return &instance, nil } } return nil, fmt.Errorf("no valid SearX instances found") } // PerformSearXTextSearch performs a text search using a SearX instance. func PerformSearXTextSearch(query, categories, language string, page int) ([]TextSearchResult, time.Duration, error) { // Default value for "safe" search safe := "1" startTime := time.Now() // Start the timer var results []TextSearchResult instance, err := getValidInstance() if err != nil { return nil, 0, fmt.Errorf("failed to get a valid SearX instance: %w", err) } searchURL := fmt.Sprintf("%s/search?q=%s&categories=%s&language=%s&safe_search=%s&page=%d&format=json", instance.URL, url.QueryEscape(query), categories, language, safe, page) req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, 0, fmt.Errorf("creating request: %v", err) } XNGUserAgent, err := GetUserAgent("Text-Search-XNG") if err != nil { return nil, 0, fmt.Errorf("generating User-Agent: %v", err) } req.Header.Set("User-Agent", XNGUserAgent) var resp *http.Response if config.MetaProxyEnabled && metaProxyClient != nil { resp, err = metaProxyClient.Do(req) } else { client := &http.Client{Timeout: 10 * time.Second} resp, err = client.Do(req) } if err != nil { return nil, 0, fmt.Errorf("performing request: %v", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return nil, 0, fmt.Errorf("reading response body: %v", err) } // Parse the JSON response to extract search results var response map[string]interface{} err = json.Unmarshal(body, &response) if err != nil { return nil, 0, fmt.Errorf("parsing response JSON: %v", err) } // Extract search results if items, ok := response["results"].([]interface{}); ok { for _, item := range items { if result, ok := item.(map[string]interface{}); ok { title := strings.TrimSpace(fmt.Sprintf("%v", result["title"])) url := strings.TrimSpace(fmt.Sprintf("%v", result["url"])) description := strings.TrimSpace(fmt.Sprintf("%v", result["content"])) results = append(results, TextSearchResult{ Header: title, URL: url, Description: description, }) } } } duration := time.Since(startTime) // Calculate the duration if len(results) == 0 { printDebug("No results found for query: %s", query) return nil, duration, fmt.Errorf("no results found") } printDebug("Search completed successfully for query: %s, found %d results", query, len(results)) return results, duration, nil }