- title := strings.TrimSpace(anchor.Find(".title").Text())
- if title == "" {
- // fallback if the .title is slightly off in the DOM
- title = strings.TrimSpace(s.Find(".title").Text())
- }
-
- // Description is inside
- desc := strings.TrimSpace(s.Find(".snippet-description").Text())
-
- // Add only if everything is non-empty
- if title != "" && desc != "" {
+ // Add to results only if all components are present
+ if title != "" && description != "" && exists && url != "" {
results = append(results, TextSearchResult{
Header: title,
- URL: link,
- Description: desc,
+ URL: url,
+ Description: description,
})
}
})
- duration := time.Since(startTime)
+ duration := time.Since(startTime) // Calculate the duration
// Return an error if no results are found
if len(results) == 0 {
- printDebug("No results found for query")
return nil, duration, fmt.Errorf("no results found")
}
- printDebug("Search completed successfully found %d results", len(results))
return results, duration, nil
}
diff --git a/text-duckduckgo.go b/text-duckduckgo.go
index 6fc836a..b4033c1 100644
--- a/text-duckduckgo.go
+++ b/text-duckduckgo.go
@@ -16,36 +16,21 @@ func PerformDuckDuckGoTextSearch(query, safe, lang string, page int) ([]TextSear
var results []TextSearchResult
searchURL := buildDuckDuckGoSearchURL(query, page)
- // Create a request
- req, err := http.NewRequest("GET", searchURL, nil)
+ resp, err := http.Get(searchURL)
if err != nil {
- return nil, 0, fmt.Errorf("creating request: %v", err)
- }
-
- userAgent, err := GetUserAgent("duckduck-text-search")
- if err != nil {
- return nil, 0, err
- }
- req.Header.Set("User-Agent", userAgent)
-
- resp, err := DoMetaProxyRequest(req)
- if err != nil {
- return nil, 0, fmt.Errorf("failed to do meta-request: %v", err)
+ return nil, 0, fmt.Errorf("making request: %v", err)
}
defer resp.Body.Close()
- // Check for HTTP status code
if resp.StatusCode != http.StatusOK {
return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
- // Parse HTML response
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return nil, 0, fmt.Errorf("loading HTML document: %v", err)
}
- // Extract results from the page
doc.Find(".result__body").Each(func(i int, s *goquery.Selection) {
header := s.Find(".result__a").Text()
description := s.Find(".result__snippet").Text()
diff --git a/text-google.go b/text-google.go
index 346c76c..a706aff 100644
--- a/text-google.go
+++ b/text-google.go
@@ -11,46 +11,45 @@ import (
)
func PerformGoogleTextSearch(query, safe, lang string, page int) ([]TextSearchResult, time.Duration, error) {
- startTime := time.Now()
const resultsPerPage = 10
+ var results []TextSearchResult
- // 1) Build the search URL
+ startTime := time.Now() // Start the timer
+
+ client := &http.Client{}
searchURL := buildSearchURL(query, safe, lang, page, resultsPerPage)
- // 2) Create a new request
req, err := http.NewRequest("GET", searchURL, nil)
if err != nil {
return nil, 0, fmt.Errorf("failed to create request: %v", err)
}
- // 3) Generate and set a User-Agent header
- userAgent, err := GetUserAgent("Google-Text-Search")
+ // User Agent generation
+ TextUserAgent, err := GetUserAgent("Text-Search")
if err != nil {
return nil, 0, err
}
- req.Header.Set("User-Agent", userAgent)
- // 4) Use the meta-proxy wrapper
- resp, err := DoMetaProxyRequest(req)
+ req.Header.Set("User-Agent", TextUserAgent)
+
+ resp, err := client.Do(req)
if err != nil {
- return nil, 0, fmt.Errorf("failed to do meta-request: %v", err)
+ return nil, 0, fmt.Errorf("making request: %v", err)
}
defer resp.Body.Close()
- // 5) Check HTTP status
if resp.StatusCode != http.StatusOK {
return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
- // 6) Parse the HTML response
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return nil, 0, fmt.Errorf("loading HTML document: %v", err)
}
- results := parseResults(doc)
- // 7) Calculate duration
- duration := time.Since(startTime)
+ results = parseResults(doc)
+
+ duration := time.Since(startTime) // Calculate the duration
if len(results) == 0 {
printDebug("No results found from Google Search")
@@ -66,7 +65,7 @@ func buildSearchURL(query, safe, lang string, page, resultsPerPage int) string {
}
langParam := ""
- glParam := ""
+ var glParam string
if lang != "" {
// Use lang as the geolocation
@@ -78,16 +77,12 @@ func buildSearchURL(query, safe, lang string, page, resultsPerPage int) string {
}
startIndex := (page - 1) * resultsPerPage
- udmParam := "&udm=14" // Add the required parameter
- // Build the URL string
- baseURL := "https://www.google.com/search?q=%s%s%s%s&start=%d%s"
- fullURL := fmt.Sprintf(baseURL, url.QueryEscape(query), safeParam, langParam, glParam, startIndex, udmParam)
+ printDebug(fmt.Sprintf("https://www.google.com/search?q=%s%s%s%s&start=%d",
+ url.QueryEscape(query), safeParam, langParam, glParam, startIndex))
- // Debug print
- printDebug("%s", fullURL)
-
- return fullURL
+ return fmt.Sprintf("https://www.google.com/search?q=%s%s%s%s&start=%d",
+ url.QueryEscape(query), safeParam, langParam, glParam, startIndex)
}
// func getRandomGeoLocation() (string, string) {
diff --git a/text-librex.go b/text-librex.go
index 622d343..55eeb91 100644
--- a/text-librex.go
+++ b/text-librex.go
@@ -3,11 +3,14 @@ package main
import (
"encoding/json"
"fmt"
+ "log"
"net/http"
"net/url"
"time"
)
+const LIBREX_DOMAIN = "librex.antopie.org"
+
type LibreXResult struct {
Title string `json:"title"`
URL string `json:"url"`
@@ -17,65 +20,65 @@ type LibreXResult struct {
type LibreXResponse []LibreXResult
func PerformLibreXTextSearch(query, safe, lang string, page int) ([]TextSearchResult, time.Duration, error) {
- startTime := time.Now()
+ startTime := time.Now() // Start the timer
- // LibreX uses offset instead of page (starting at 0)
- pageOffset := (page - 1) * 10
+ // LibreX/Y uses offset instead of page that starts at 0
+ page--
+ page = page * 10
- // Generate User-Agent
+ searchURL := fmt.Sprintf("https://%s/api.php?q=%s&p=%d&t=0", LIBREX_DOMAIN, url.QueryEscape(query), page)
+
+ // User Agent generation
userAgent, err := GetUserAgent("librex-text-search")
if err != nil {
return nil, 0, err
}
- var allResults []TextSearchResult
+ req, err := http.NewRequest("GET", searchURL, nil)
+ if err != nil {
+ return nil, 0, err
+ }
+ req.Header.Set("User-Agent", userAgent)
- for _, domain := range config.LibreXInstances {
- searchURL := fmt.Sprintf("https://%s/api.php?q=%s&p=%d&t=0",
- domain,
- url.QueryEscape(query),
- pageOffset,
- )
+ client := &http.Client{}
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, 0, logError("error making request to LibreX", err)
+ }
+ defer resp.Body.Close()
- req, err := http.NewRequest("GET", searchURL, nil)
- if err != nil {
- printWarn("failed to create request for domain %s: %v", domain, err)
- continue
- }
- req.Header.Set("User-Agent", userAgent)
-
- resp, err := DoMetaProxyRequest(req)
- if err != nil {
- return nil, 0, fmt.Errorf("failed to do meta-request: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- printWarn("unexpected status code from %s: %d", domain, resp.StatusCode)
- continue
- }
-
- var librexResp LibreXResponse
- if err := json.NewDecoder(resp.Body).Decode(&librexResp); err != nil {
- printWarn("error decoding response from %s: %v", domain, err)
- continue
- }
-
- // Accumulate results from this instance
- for _, item := range librexResp {
- allResults = append(allResults, TextSearchResult{
- URL: item.URL,
- Header: item.Title,
- Description: item.Description,
- Source: "LibreX",
- })
- }
+ if resp.StatusCode != http.StatusOK {
+ return nil, 0, logError("unexpected status code", fmt.Errorf("%d", resp.StatusCode))
}
- duration := time.Since(startTime)
- if len(allResults) == 0 {
- return nil, duration, fmt.Errorf("no results found from any LibreX instance")
+ var librexResp LibreXResponse
+ if err := json.NewDecoder(resp.Body).Decode(&librexResp); err != nil {
+ return nil, 0, logError("error decoding LibreX response", err)
}
- return allResults, duration, nil
+ var results []TextSearchResult
+ for _, item := range librexResp {
+ result := TextSearchResult{
+ URL: item.URL,
+ Header: item.Title,
+ Description: item.Description,
+ Source: "LibreX",
+ }
+
+ results = append(results, result)
+ }
+
+ duration := time.Since(startTime) // Calculate the duration
+
+ if len(results) == 0 {
+ return nil, duration, fmt.Errorf("no results found")
+ }
+
+ return results, duration, nil
+}
+
+// This is just stupid it will probbably lead to printing error twice
+func logError(message string, err error) error {
+ log.Printf("%s: %v", message, err)
+ return fmt.Errorf("%s: %w", message, err)
}
diff --git a/text-quant.go b/text-quant.go
index 959879d..de8b03a 100644
--- a/text-quant.go
+++ b/text-quant.go
@@ -26,11 +26,9 @@ type QwantTextAPIResponse struct {
}
// PerformQwantTextSearch contacts the Qwant API and returns a slice of TextSearchResult
-func PerformQwantTextSearch(query, safe, lang string, page int) ([]TextSearchResult, time.Duration, error) {
- startTime := time.Now()
-
+func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
const resultsPerPage = 10
- offset := (page - 1) * resultsPerPage
+ const offset = 0
// Ensure safe search is disabled by default if not specified
if safe == "" {
@@ -42,44 +40,39 @@ func PerformQwantTextSearch(query, safe, lang string, page int) ([]TextSearchRes
lang = "en_CA"
}
- apiURL := fmt.Sprintf(
- "https://api.qwant.com/v3/search/web?q=%s&count=%d&locale=%s&offset=%d&device=desktop",
+ apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?q=%s&count=%d&locale=%s&offset=%d&device=desktop",
url.QueryEscape(query),
resultsPerPage,
lang,
- offset,
- )
+ offset)
+
+ client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest("GET", apiURL, nil)
if err != nil {
- // Return three values: nil for the slice, 0 for duration, error for the third.
- return nil, 0, fmt.Errorf("creating request: %v", err)
+ return nil, fmt.Errorf("creating request: %v", err)
}
- userAgent, err := GetUserAgent("Quant-Text-Search")
- if err != nil {
- return nil, 0, err
- }
- req.Header.Set("User-Agent", userAgent)
+ req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
- resp, err := DoMetaProxyRequest(req)
+ resp, err := client.Do(req)
if err != nil {
- return nil, 0, fmt.Errorf("failed to do meta-request: %v", err)
+ return nil, fmt.Errorf("making request: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+ return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
var apiResp QwantTextAPIResponse
if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
- return nil, 0, fmt.Errorf("decoding response: %v", err)
+ return nil, fmt.Errorf("decoding response: %v", err)
}
// Extracting results from the nested JSON structure
if len(apiResp.Data.Result.Items.Mainline) == 0 {
- return nil, 0, fmt.Errorf("no search results found")
+ return nil, fmt.Errorf("no search results found")
}
var results []TextSearchResult
@@ -93,8 +86,7 @@ func PerformQwantTextSearch(query, safe, lang string, page int) ([]TextSearchRes
})
}
- duration := time.Since(startTime)
- return results, duration, nil
+ return results, nil
}
// cleanQwantURL extracts the main part of the URL, removing tracking information
diff --git a/text-searchxng.go b/text-searchxng.go
index 68cd315..3ed118f 100644
--- a/text-searchxng.go
+++ b/text-searchxng.go
@@ -3,56 +3,31 @@ package main
import (
"encoding/json"
"fmt"
- "io"
+ "io/ioutil"
+ "log"
"net/http"
- "net/url"
- "strings"
"time"
)
type Instance struct {
- URL string `json:"-"` // Populated from map key
- Analytics bool `json:"analytics"`
- Comments []string `json:"comments"`
- AlternativeUrls map[string]interface{} `json:"alternativeUrls"`
- Main bool `json:"main"`
- NetworkType string `json:"network_type"`
- HTTP struct {
- StatusCode int `json:"status_code"`
- Error string `json:"error"`
- } `json:"http"`
- Version string `json:"version"`
- Grade string `json:"grade"`
- GradeURL string `json:"gradeUrl"`
- Generator string `json:"generator"`
- ContactURL FlexibleType `json:"contact_url"` // Custom type
- DocsURL string `json:"docs_url"`
-}
-
-type FlexibleType struct {
- StringValue string
- BoolValue bool
- IsString bool
+ URL string `json:"url"`
+ Status int `json:"status"`
+ SSLGrade string `json:"ssl_grade"`
}
const searxInstancesURL = "https://searx.space/data/instances.json"
-// FetchInstances fetches available SearX instances from the registry.
func fetchInstances() ([]Instance, error) {
+ client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest("GET", searxInstancesURL, nil)
if err != nil {
- return nil, fmt.Errorf("creating request: %v", err)
+ return nil, err
}
+ req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
- XNGUserAgent, err := GetUserAgent("Text-Search-XNG")
+ resp, err := client.Do(req)
if err != nil {
- return nil, fmt.Errorf("generating User-Agent: %v", err)
- }
- req.Header.Set("User-Agent", XNGUserAgent)
-
- resp, err := DoMetaProxyRequest(req)
- if err != nil {
- return nil, fmt.Errorf("failed to do meta-request: %v", err)
+ return nil, err
}
defer resp.Body.Close()
@@ -60,105 +35,44 @@ func fetchInstances() ([]Instance, error) {
return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
- body, err := io.ReadAll(resp.Body)
+ body, err := ioutil.ReadAll(resp.Body)
if err != nil {
- return nil, fmt.Errorf("reading response body: %v", err)
+ return nil, err
}
- // Root structure of the JSON response
- var root struct {
- Instances map[string]Instance `json:"instances"`
- }
-
- // Unmarshal JSON into the root structure
- err = json.Unmarshal(body, &root)
- if err != nil {
- return nil, fmt.Errorf("parsing response JSON: %v", err)
- }
-
- // Collect instances into a slice
var instances []Instance
- for url, instance := range root.Instances {
- instance.URL = url // Assign the URL from the map key
- instances = append(instances, instance)
+ err = json.Unmarshal(body, &instances)
+ if err != nil {
+ return nil, err
}
return instances, nil
}
-// UnmarshalJSON implements custom unmarshalling for FlexibleType.
-func (f *FlexibleType) UnmarshalJSON(data []byte) error {
- // Try to unmarshal as a string
- var str string
- if err := json.Unmarshal(data, &str); err == nil {
- f.StringValue = str
- f.IsString = true
- return nil
- }
-
- // Try to unmarshal as a bool
- var b bool
- if err := json.Unmarshal(data, &b); err == nil {
- f.BoolValue = b
- f.IsString = false
- return nil
- }
-
- // Return an error if neither works
- return fmt.Errorf("invalid FlexibleType: %s", string(data))
-}
-
-// String returns the string representation of FlexibleType.
-func (f FlexibleType) String() string {
- if f.IsString {
- return f.StringValue
- }
- return fmt.Sprintf("%v", f.BoolValue)
-}
-
-// ValidateInstance checks if a SearX instance is valid by performing a test query.
func validateInstance(instance Instance) bool {
- // Skip .onion instances
- if strings.Contains(instance.URL, ".onion") {
- printDebug("Skipping .onion instance: %s", instance.URL)
- return false
- }
-
- client := &http.Client{
- Timeout: 10 * time.Second,
- }
-
- testURL := fmt.Sprintf("%s/search?q=test&categories=general&language=en&safe_search=1&page=1&format=json", instance.URL)
- req, err := http.NewRequest("GET", testURL, nil)
+ client := &http.Client{Timeout: 10 * time.Second}
+ req, err := http.NewRequest("GET", fmt.Sprintf("%s/search?q=test&categories=general&language=en&safe_search=1&page=1&format=json", instance.URL), nil)
if err != nil {
- printDebug("Error creating SearchXNG request for instance validation: %v", err)
+ log.Printf("Error creating request for URL: %s, Error: %v", instance.URL, err)
return false
}
-
- XNGUserAgent, err := GetUserAgent("Text-Search-XNG")
- if err != nil {
- printWarn("Error generating User-Agent: %v", err)
- return false
- }
- req.Header.Set("User-Agent", XNGUserAgent)
+ req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
resp, err := client.Do(req)
if err != nil {
- printDebug("Error performing request for SearchXNG instance validation: %v", err)
+ log.Printf("Error performing request for URL: %s, Error: %v", instance.URL, err)
return false
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- printDebug("SearchXNG Instance validation failed. StatusCode: %d", resp.StatusCode)
+ log.Printf("Instance validation failed for URL: %s, StatusCode: %d", instance.URL, resp.StatusCode)
return false
}
- // Successful validation
return true
}
-// GetValidInstance fetches and validates SearX instances, returning a valid one.
func getValidInstance() (*Instance, error) {
instances, err := fetchInstances()
if err != nil {
@@ -174,81 +88,11 @@ func getValidInstance() (*Instance, error) {
return nil, fmt.Errorf("no valid SearX instances found")
}
-// PerformSearXTextSearch performs a text search using a SearX instance.
-func PerformSearXTextSearch(query, categories, language string, page int) ([]TextSearchResult, time.Duration, error) {
- // Default value for "safe" search
- safe := "1"
+// func main() {
+// instance, err := getValidInstance()
+// if err != nil {
+// log.Fatalf("Failed to get a valid SearX instance: %v", err)
+// }
- startTime := time.Now() // Start the timer
- var results []TextSearchResult
-
- instance, err := getValidInstance()
- if err != nil {
- return nil, 0, fmt.Errorf("failed to get a valid SearX instance: %w", err)
- }
-
- searchURL := fmt.Sprintf("%s/search?q=%s&categories=%s&language=%s&safe_search=%s&page=%d&format=json",
- instance.URL, url.QueryEscape(query), categories, language, safe, page)
-
- req, err := http.NewRequest("GET", searchURL, nil)
- if err != nil {
- return nil, 0, fmt.Errorf("creating request: %v", err)
- }
-
- XNGUserAgent, err := GetUserAgent("Text-Search-XNG")
- if err != nil {
- return nil, 0, fmt.Errorf("generating User-Agent: %v", err)
- }
- req.Header.Set("User-Agent", XNGUserAgent)
-
- var resp *http.Response
- if config.MetaProxyEnabled && metaProxyClient != nil {
- resp, err = metaProxyClient.Do(req)
- } else {
- client := &http.Client{Timeout: 10 * time.Second}
- resp, err = client.Do(req)
- }
- if err != nil {
- return nil, 0, fmt.Errorf("performing request: %v", err)
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, 0, fmt.Errorf("reading response body: %v", err)
- }
-
- // Parse the JSON response to extract search results
- var response map[string]interface{}
- err = json.Unmarshal(body, &response)
- if err != nil {
- return nil, 0, fmt.Errorf("parsing response JSON: %v", err)
- }
-
- // Extract search results
- if items, ok := response["results"].([]interface{}); ok {
- for _, item := range items {
- if result, ok := item.(map[string]interface{}); ok {
- title := strings.TrimSpace(fmt.Sprintf("%v", result["title"]))
- url := strings.TrimSpace(fmt.Sprintf("%v", result["url"]))
- description := strings.TrimSpace(fmt.Sprintf("%v", result["content"]))
-
- results = append(results, TextSearchResult{
- Header: title,
- URL: url,
- Description: description,
- })
- }
- }
- }
-
- duration := time.Since(startTime) // Calculate the duration
-
- if len(results) == 0 {
- printDebug("No results found for query: %s", query)
- return nil, duration, fmt.Errorf("no results found")
- }
-
- printDebug("Search completed successfully for query: %s, found %d results", query, len(results))
- return results, duration, nil
-}
+// log.Printf("Selected SearX instance: %s", instance.URL)
+// }
diff --git a/text.go b/text.go
index 3422f84..d6e3212 100755
--- a/text.go
+++ b/text.go
@@ -8,26 +8,13 @@ import (
var textSearchEngines []SearchEngine
-var allTextSearchEngines = []SearchEngine{
- {Name: "Google", Func: wrapTextSearchFunc(PerformGoogleTextSearch)},
- {Name: "LibreX", Func: wrapTextSearchFunc(PerformLibreXTextSearch)},
- {Name: "Brave", Func: wrapTextSearchFunc(PerformBraveTextSearch)},
- {Name: "DuckDuckGo", Func: wrapTextSearchFunc(PerformDuckDuckGoTextSearch)},
- {Name: "Quant", Func: wrapTextSearchFunc(PerformQwantTextSearch)}, // Broken !
- //{Name: "SearXNG", Func: wrapTextSearchFunc(PerformSearXTextSearch)}, // bruh
-}
-
-func initTextEngines() {
- // textSearchEngines is your final slice (already declared globally)
- textSearchEngines = nil // or make([]SearchEngine, 0)
-
- for _, engineName := range config.MetaSearch.Text {
- for _, candidate := range allTextSearchEngines {
- if candidate.Name == engineName {
- textSearchEngines = append(textSearchEngines, candidate)
- break
- }
- }
+func init() {
+ textSearchEngines = []SearchEngine{
+ {Name: "Google", Func: wrapTextSearchFunc(PerformGoogleTextSearch)},
+ {Name: "LibreX", Func: wrapTextSearchFunc(PerformLibreXTextSearch)},
+ {Name: "Brave", Func: wrapTextSearchFunc(PerformBraveTextSearch)},
+ {Name: "DuckDuckGo", Func: wrapTextSearchFunc(PerformDuckDuckGoTextSearch)},
+ // {Name: "SearXNG", Func: wrapTextSearchFunc(PerformSearXNGTextSearch), Weight: 2}, // Uncomment when implemented
}
}
@@ -92,7 +79,7 @@ func getTextResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string,
resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
}
} else {
- textResults, _, _, _, _ := convertToSpecificResults(results)
+ textResults, _, _, _ := convertToSpecificResults(results)
combinedResults = textResults
}
case <-time.After(2 * time.Second):
@@ -111,7 +98,7 @@ func prefetchPage(query, safe, lang string, page int) {
cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "active", Lang: lang, Type: "text"}
if _, exists := resultsCache.Get(cacheKey); !exists {
printInfo("Page %d not cached, caching now...", page)
- if config.MetaSearchEnabled {
+ if config.CrawlerEnabled {
pageResults := fetchTextResults(query, safe, lang, page)
if len(pageResults) > 0 {
resultsCache.Set(cacheKey, convertToSearchResults(pageResults))
@@ -124,11 +111,10 @@ func prefetchPage(query, safe, lang string, page int) {
}
}
-// The logic in this function is rotating search engines instead of running them in order as noted in the wiki
func fetchTextResults(query, safe, lang string, page int) []TextSearchResult {
var results []TextSearchResult
- if !config.MetaSearchEnabled {
+ if !config.CrawlerEnabled {
printDebug("Crawler is disabled; fetching from local index.")
// Calculate the starting position based on the page number
diff --git a/user-settings.go b/user-settings.go
index 68c3a39..a872f11 100755
--- a/user-settings.go
+++ b/user-settings.go
@@ -72,9 +72,8 @@ func saveUserSettings(w http.ResponseWriter, settings UserSettings) {
Value: cd.GetValue(settings),
Path: "/",
Expires: expiration,
- Secure: true, // Ensure HTTPS is required
- HttpOnly: true,
- SameSite: http.SameSiteStrictMode, // Restrict cross-site usage
+ Secure: true,
+ SameSite: http.SameSiteStrictMode,
})
}
diff --git a/video.go b/video.go
index 193773c..3120367 100644
--- a/video.go
+++ b/video.go
@@ -12,16 +12,25 @@ import (
const retryDuration = 12 * time.Hour // Retry duration for unresponding piped instances
var (
- pipedInstances = []string{}
+ pipedInstances = []string{
+ "api.piped.yt",
+ "pipedapi.moomoo.me",
+ "pipedapi.darkness.services",
+ "pipedapi.kavin.rocks",
+ "piped-api.hostux.net",
+ "pipedapi.syncpundit.io",
+ "piped-api.cfe.re",
+ "pipedapi.in.projectsegfau.lt",
+ "piapi.ggtyler.dev",
+ "piped-api.codespace.cz",
+ "pipedapi.coldforge.xyz",
+ "pipedapi.osphost.fi",
+ }
disabledInstances = make(map[string]bool)
mu sync.Mutex
videoResultsChan = make(chan []VideoResult) // Channel to receive video results from other nodes
)
-func initPipedInstances() {
- pipedInstances = config.MetaSearch.Video
-}
-
// VideoAPIResponse matches the structure of the JSON response from the Piped API
type VideoAPIResponse struct {
Items []struct {
@@ -142,7 +151,7 @@ func handleVideoSearch(w http.ResponseWriter, settings UserSettings, query strin
start := time.Now()
var results []VideoResult
- if config.MetaSearchEnabled {
+ if config.CrawlerEnabled {
results = fetchVideoResults(query, settings.SafeSearch, settings.SearchLanguage, page)
}
@@ -175,12 +184,12 @@ func handleVideoSearch(w http.ResponseWriter, settings UserSettings, query strin
func fetchVideoResults(query, safe, lang string, page int) []VideoResult {
// Check if the crawler is enabled
- if !config.MetaSearchEnabled {
+ if !config.CrawlerEnabled {
printDebug("Crawler is disabled; skipping video search.")
return []VideoResult{}
}
- // Proceed with Piped API request if MetaSearchEnabled
+ // Proceed with Piped API request if CrawlerEnabled
apiResp, err := makeHTMLRequest(query, safe, lang, page)
if err != nil {
printWarn("Error fetching video results: %v", err)
- 
- 
-