Changed self-crawling as experimental, cleanup unused features

agent.go

@@ -135,10 +135,10 @@ func randomUserAgent() (string, error) {
 
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	// Overall usage: 80% chance for Chromium, 20% for Firefox
+	// Overall usage: 85% chance for Chromium, 15% for Firefox
 	usageStats := map[string]float64{
-		"Firefox":  20.0,
+		"Firefox":  15.0,
-		"Chromium": 80.0,
+		"Chromium": 85.0,
 	}
 
 	// Weighted random selection of the browser type

cache.go

@@ -11,7 +11,6 @@ import (
 // SearchResult is a generic interface for all types of search results.
 type SearchResult interface{}
 
-// Define various search result types implementing SearchResult interface
 type TextSearchResult struct {
 	URL         string
 	Header      string

config.go

@@ -29,9 +29,8 @@ type MetaSearchConfig struct {
 
 type Config struct {
 	Port                int
-	AuthCode            string
+	NodeID              string
-	PeerID              string
+	Nodes               []string
-	Peers               []string
 	Domain              string
 	NodesEnabled        bool
 	MetaSearchEnabled   bool
@@ -65,8 +64,7 @@ type Config struct {
 var defaultConfig = Config{
 	Port:                       5000,
 	Domain:                     "localhost",
-	Peers:                      []string{},
+	Nodes:                      []string{},
-	AuthCode:                   generateStrongRandomString(64),
 	NodesEnabled:               false,
 	MetaSearchEnabled:          true,
 	IndexerEnabled:             false,
@@ -268,12 +266,6 @@ func createConfig() error {
 		config = defaultConfig
 	}
 
-	// Generate AuthCode if missing
-	if config.AuthCode == "" {
-		config.AuthCode = generateStrongRandomString(64)
-		printMessage("Generated connection code: %s\n", config.AuthCode)
-	}
-
 	saveConfig(config)
 	printInfo("Configuration saved successfully.")
 	return nil
@@ -288,29 +280,36 @@ func saveConfig(config Config) {
 	sec.Key("Domain").SetValue(config.Domain)
 	sec.Key("LogLevel").SetValue(strconv.Itoa(config.LogLevel))
 
-	// Peers section
+	// Nodes section
-	peersSec := cfg.Section("Peers")
+	nodesSec := cfg.Section("Nodes")
-	peersSec.Key("AuthCode").SetValue(config.AuthCode)
+	nodesSec.Key("NodeID").SetValue(config.NodeID)
-	peersSec.Key("PeerID").SetValue(config.PeerID)
+	nodesSec.Key("Nodes").SetValue(strings.Join(config.Nodes, ","))
-	peersSec.Key("Peers").SetValue(strings.Join(config.Peers, ","))
 
 	// Features section
-	featuresSec := cfg.Section("Features")
+	if config.NodesEnabled != defaultConfig.NodesEnabled ||
-	featuresSec.Key("Nodes").SetValue(strconv.FormatBool(config.NodesEnabled))
+		config.MetaSearchEnabled != defaultConfig.MetaSearchEnabled ||
-	featuresSec.Key("Crawler").SetValue(strconv.FormatBool(config.MetaSearchEnabled))
+		config.IndexerEnabled != defaultConfig.IndexerEnabled ||
-	featuresSec.Key("Indexer").SetValue(strconv.FormatBool(config.IndexerEnabled))
+		config.WebsiteEnabled != defaultConfig.WebsiteEnabled ||
-	featuresSec.Key("Website").SetValue(strconv.FormatBool(config.WebsiteEnabled))
+		config.MetaProxyEnabled != defaultConfig.MetaProxyEnabled ||
-	featuresSec.Key("MetaProxy").SetValue(strconv.FormatBool(config.MetaProxyEnabled))
+		config.CrawlerProxyEnabled != defaultConfig.CrawlerProxyEnabled {
-	featuresSec.Key("CrawlerProxy").SetValue(strconv.FormatBool(config.CrawlerProxyEnabled))
+
+		featuresSec := cfg.Section("Features")
+		setBoolIfChanged(featuresSec, "Nodes", config.NodesEnabled, defaultConfig.NodesEnabled)
+		setBoolIfChanged(featuresSec, "Crawler", config.MetaSearchEnabled, defaultConfig.MetaSearchEnabled)
+		setBoolIfChanged(featuresSec, "Indexer", config.IndexerEnabled, defaultConfig.IndexerEnabled)
+		setBoolIfChanged(featuresSec, "Website", config.WebsiteEnabled, defaultConfig.WebsiteEnabled)
+		setBoolIfChanged(featuresSec, "MetaProxy", config.MetaProxyEnabled, defaultConfig.MetaProxyEnabled)
+		setBoolIfChanged(featuresSec, "CrawlerProxy", config.CrawlerProxyEnabled, defaultConfig.CrawlerProxyEnabled)
+	}
 
 	// Proxies section
 	proxiesSec := cfg.Section("Proxies")
 	proxiesSec.Key("MetaProxyStrict").SetValue(strconv.FormatBool(config.MetaProxyStrict))
 	proxiesSec.Key("MetaProxies").SetValue(strings.Join(config.MetaProxies, ","))
-	proxiesSec.Key("CrawlerProxyStrict").SetValue(strconv.FormatBool(config.CrawlerProxyStrict))
+	setBoolIfChanged(proxiesSec, "CrawlerProxyStrict", config.CrawlerProxyStrict, defaultConfig.CrawlerProxyStrict)
-	proxiesSec.Key("CrawlerProxies").SetValue(strings.Join(config.CrawlerProxies, ","))
+	setSliceIfChanged(proxiesSec, "CrawlerProxies", config.CrawlerProxies, defaultConfig.CrawlerProxies)
 	proxiesSec.Key("MetaProxyRetry").SetValue(strconv.Itoa(config.MetaProxyRetry))
-	proxiesSec.Key("CrawlerProxyRetry").SetValue(strconv.Itoa(config.CrawlerProxyRetry))
+	setIntIfChanged(proxiesSec, "CrawlerProxyRetry", config.CrawlerProxyRetry, defaultConfig.CrawlerProxyRetry)
 
 	// MetaSearch section
 	metaSec := cfg.Section("MetaSearches")
@@ -321,12 +320,19 @@ func saveConfig(config Config) {
 	metaSec.Key("Video").SetValue(strings.Join(config.MetaSearch.Video, ","))
 
 	// Indexer section
-	indexerSec := cfg.Section("Indexer")
+	if config.ConcurrentStandardCrawlers != defaultConfig.ConcurrentStandardCrawlers ||
-	indexerSec.Key("ConcurrentStandardCrawlers").SetValue(strconv.Itoa(config.ConcurrentStandardCrawlers))
+		config.ConcurrentChromeCrawlers != defaultConfig.ConcurrentChromeCrawlers ||
-	indexerSec.Key("ConcurrentChromeCrawlers").SetValue(strconv.Itoa(config.ConcurrentChromeCrawlers))
+		config.CrawlingInterval != defaultConfig.CrawlingInterval ||
-	indexerSec.Key("CrawlingInterval").SetValue(config.CrawlingInterval.String())
+		config.MaxPagesPerDomain != defaultConfig.MaxPagesPerDomain ||
-	indexerSec.Key("MaxPagesPerDomain").SetValue(strconv.Itoa(config.MaxPagesPerDomain))
+		config.IndexBatchSize != defaultConfig.IndexBatchSize {
-	indexerSec.Key("IndexBatchSize").SetValue(strconv.Itoa(config.IndexBatchSize))
+
+		indexerSec := cfg.Section("Indexer")
+		setIntIfChanged(indexerSec, "ConcurrentStandardCrawlers", config.ConcurrentStandardCrawlers, defaultConfig.ConcurrentStandardCrawlers)
+		setIntIfChanged(indexerSec, "ConcurrentChromeCrawlers", config.ConcurrentChromeCrawlers, defaultConfig.ConcurrentChromeCrawlers)
+		setIfChanged(indexerSec, "CrawlingInterval", config.CrawlingInterval.String(), defaultConfig.CrawlingInterval.String())
+		setIntIfChanged(indexerSec, "MaxPagesPerDomain", config.MaxPagesPerDomain, defaultConfig.MaxPagesPerDomain)
+		setIntIfChanged(indexerSec, "IndexBatchSize", config.IndexBatchSize, defaultConfig.IndexBatchSize)
+	}
 
 	// DriveCache section
 	driveSec := cfg.Section("DriveCache")
@@ -357,9 +363,9 @@ func loadConfig() Config {
 	domain := getConfigValueString(cfg.Section("Server").Key("Domain"), defaultConfig.Domain)
 	logLevel := getConfigValue(cfg.Section("Server").Key("LogLevel"), defaultConfig.LogLevel, strconv.Atoi)
 
-	// Peers
+	// Nodes
-	authCode := getConfigValueString(cfg.Section("Peers").Key("AuthCode"), defaultConfig.AuthCode)
+	nodeID := getConfigValueString(cfg.Section("Nodes").Key("NodeID"), defaultConfig.NodeID)
-	peers := strings.Split(getConfigValueString(cfg.Section("Peers").Key("Peers"), ""), ",")
+	nodes := strings.Split(getConfigValueString(cfg.Section("Nodes").Key("Nodes"), ""), ",")
 
 	// Features
 	nodesEnabled := getConfigValueBool(cfg.Section("Features").Key("Nodes"), defaultConfig.NodesEnabled)
@@ -371,7 +377,7 @@ func loadConfig() Config {
 	metaProxyEnabled := getConfigValueBool(cfg.Section("Features").Key("MetaProxy"), defaultConfig.MetaProxyEnabled)
 	crawlerProxyEnabled := getConfigValueBool(cfg.Section("Features").Key("CrawlerProxy"), defaultConfig.CrawlerProxyEnabled)
 
-	// Proxies
+	// Nodes
 	metaProxyStrict := getConfigValueBool(cfg.Section("Proxies").Key("MetaProxyStrict"), defaultConfig.MetaProxyStrict)
 	metaProxies := strings.Split(getConfigValueString(cfg.Section("Proxies").Key("MetaProxies"), ""), ",")
 	crawlerProxyStrict := getConfigValueBool(cfg.Section("Proxies").Key("CrawlerProxyStrict"), defaultConfig.CrawlerProxyStrict)
@@ -410,8 +416,8 @@ func loadConfig() Config {
 		Port:                       port,
 		Domain:                     domain,
 		LogLevel:                   logLevel,
-		AuthCode:                   authCode,
+		NodeID:                     nodeID,
-		Peers:                      peers,
+		Nodes:                      nodes,
 		NodesEnabled:               nodesEnabled,
 		MetaSearchEnabled:          metaSearchEnabled,
 		IndexerEnabled:             indexerEnabled,
@@ -450,6 +456,30 @@ func loadConfig() Config {
 	}
 }
 
+func setIfChanged(sec *ini.Section, key string, value string, defaultValue string) {
+	if value != defaultValue {
+		sec.Key(key).SetValue(value)
+	}
+}
+
+func setBoolIfChanged(sec *ini.Section, key string, value bool, defaultValue bool) {
+	if value != defaultValue {
+		sec.Key(key).SetValue(strconv.FormatBool(value))
+	}
+}
+
+func setIntIfChanged(sec *ini.Section, key string, value int, defaultValue int) {
+	if value != defaultValue {
+		sec.Key(key).SetValue(strconv.Itoa(value))
+	}
+}
+
+func setSliceIfChanged(sec *ini.Section, key string, value, defaultValue []string) {
+	if strings.Join(value, ",") != strings.Join(defaultValue, ",") {
+		sec.Key(key).SetValue(strings.Join(value, ","))
+	}
+}
+
 // getConfigValue retrieves a configuration value or returns a default value from defaultConfig.
 func getConfigValue[T any](key *ini.Key, defaultValue T, parseFunc func(string) (T, error)) T {
 	if key == nil || key.String() == "" {

crawler-extraction.go

@@ -1,3 +1,6 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (

crawler-visited.go

@@ -1,3 +1,6 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (

crawler.go

@@ -1,3 +1,6 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (

files.go

@@ -28,8 +28,6 @@ var (
 	rutor         TorrentSite
 )
 
-var fileResultsChan = make(chan []TorrentResult)
-
 func initFileEngines() {
 
 	torrentGalaxy = nil
@@ -154,14 +152,6 @@ func fetchFileResults(query, safe, lang string, page int) []TorrentResult {
 		}
 	}
 
-	// If no results, try from other nodes
-	if len(results) == 0 {
-		if config.NodesEnabled {
-			printWarn("No file results found for query: %s, trying other nodes", query)
-			results = tryOtherNodesForFileSearch(query, safe, lang, page, []string{hostID})
-		}
-	}
-
 	return results
 }
 

forums.go

@@ -135,6 +135,15 @@ func handleForumsSearch(w http.ResponseWriter, settings UserSettings, query stri
 	renderTemplate(w, "forums.html", data)
 }
 
+func fetchForumResults(query, safe, lang string, page int) []ForumSearchResult {
+	results, err := PerformRedditSearch(query, safe, page)
+	if err != nil {
+		printWarn("Failed to fetch forum results: %v", err)
+		return nil
+	}
+	return results
+}
+
 func getForumResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page int) []ForumSearchResult {
 	cacheChan := make(chan []SearchResult)
 	var combinedResults []ForumSearchResult

images.go

@@ -98,7 +98,7 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
 	case results := <-cacheChan:
 		if results == nil {
 			if config.MetaSearchEnabled {
-				combinedResults = fetchImageResults(query, safe, lang, page, synchronous)
+				combinedResults = fetchImageResults(query, safe, lang, page, synchronous, true)
 				if len(combinedResults) > 0 {
 					combinedResults = filterValidImages(combinedResults)
 					resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
@@ -113,7 +113,7 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
 	case <-time.After(2 * time.Second):
 		printDebug("Cache check timeout")
 		if config.MetaSearchEnabled {
-			combinedResults = fetchImageResults(query, safe, lang, page, synchronous)
+			combinedResults = fetchImageResults(query, safe, lang, page, synchronous, true)
 			if len(combinedResults) > 0 {
 				combinedResults = filterValidImages(combinedResults)
 				resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
@@ -126,7 +126,7 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
 	return combinedResults
 }
 
-func fetchImageResults(query, safe, lang string, page int, synchronous bool) []ImageSearchResult {
+func fetchImageResults(query, safe, lang string, page int, synchronous bool, thumbsNeeded bool) []ImageSearchResult {
 	var results []ImageSearchResult
 
 	// Check if MetaSearchEnabled is false
@@ -231,23 +231,25 @@ func fetchImageResults(query, safe, lang string, page int, synchronous bool) []I
 				imageURLMapMu.Unlock()
 
 				if config.DriveCacheEnabled {
-					// Cache the thumbnail image asynchronously
+					if thumbsNeeded {
-					go func(imgResult ImageSearchResult) {
+						go func(imgResult ImageSearchResult) {
-						_, success, err := cacheImage(imgResult.Thumb, imgResult.ID, "thumb")
+							_, success, err := cacheImage(imgResult.Thumb, imgResult.ID, "thumb")
-						if err != nil || !success {
+							if err != nil || !success {
-							printWarn("Failed to cache thumbnail image %s: %v", imgResult.Thumb, err)
+								printWarn("Failed to cache thumbnail image %s: %v", imgResult.Thumb, err)
-							removeImageResultFromCache(query, page, safe == "active", lang, imgResult.ID)
+								removeImageResultFromCache(query, page, safe == "active", lang, imgResult.ID)
-						}
+							}
-					}(imageResult)
+						}(imageResult)
-
+						imageResult.ProxyThumb = fmt.Sprintf("/image/%s_thumb.webp", hash)
-					// Set ProxyThumb to the proxy URL (initially placeholder)
+					} else {
-					imageResult.ProxyThumb = fmt.Sprintf("/image/%s_thumb.webp", hash)
+						imageResult.ProxyThumb = "" // fallback ?
-
+					}
-					// Set ProxyFull to the proxy URL
 					imageResult.ProxyFull = fmt.Sprintf("/image/%s_full", hash)
 				} else {
-					// Hard cache disabled, proxy both thumb and full images
+					if thumbsNeeded {
-					imageResult.ProxyThumb = fmt.Sprintf("/image/%s_thumb", hash)
+						imageResult.ProxyThumb = fmt.Sprintf("/image/%s_thumb", hash)
+					} else {
+						imageResult.ProxyThumb = ""
+					}
 					imageResult.ProxyFull = fmt.Sprintf("/image/%s_full", hash)
 				}
 

indexer.go

@@ -1,3 +1,6 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (

init-extra.go

@@ -0,0 +1,142 @@
+//go:build experimental
+// +build experimental
+
+package main
+
+import (
+	"flag"
+	"os"
+	"path/filepath"
+)
+
+var config Config
+
+func main() {
+	// Command-line flags
+	portFlag := flag.Int("port", 0, "Port number to run the application (overrides config)")
+	domainFlag := flag.String("domain", "", "Domain address for the application (overrides config)")
+	skipConfigFlag := flag.Bool("skip-config-check", false, "Skip interactive prompts and load config.ini")
+	configFlag := flag.String("config", "", "Path to configuration file (overrides default)")
+
+	// Parse command-line flags
+	flag.Parse()
+
+	// Override global configFilePath if --config flag is provided
+	if *configFlag != "" {
+		configFilePath = *configFlag
+	}
+
+	if *skipConfigFlag {
+		// Skip interactive configuration
+		if _, err := os.Stat(configFilePath); err == nil {
+			// Load from config file if it exists
+			config = loadConfig()
+		} else {
+			// Use defaults if config file does not exist
+			config = defaultConfig
+			saveConfig(config) // Save the defaults to config.ini
+			printInfo("Configuration saved to %s", configFilePath)
+		}
+	} else {
+		// Initialize configuration interactively or from config file
+		err := initConfig()
+		if err != nil {
+			printErr("Error during initialization: %v", err)
+			return
+		}
+	}
+
+	// Override with command-line arguments if provided
+	if *portFlag != 0 {
+		config.Port = *portFlag
+	}
+	if *domainFlag != "" {
+		config.Domain = *domainFlag
+	}
+
+	loadNodeConfig()
+
+	if config.CrawlerProxyEnabled || config.MetaProxyEnabled {
+		InitProxies()
+	}
+
+	// Initiate Browser Agent updater
+	if config.MetaSearchEnabled || config.IndexerEnabled {
+		go periodicAgentUpdate()
+	}
+
+	// Load List of Meta Search Engines
+	if config.MetaSearchEnabled {
+		initTextEngines()
+		initImageEngines()
+		initFileEngines()
+		initPipedInstances()
+		initMusicEngines()
+	}
+
+	InitializeLanguage("en") // Initialize language before generating OpenSearch
+	generateOpenSearchXML(config)
+
+	// Start the node client only if NodesEnabled is true
+	if config.NodesEnabled {
+		go startUnixSocketServer(config.NodeID)
+		printInfo("Node client started.")
+	} else {
+		printInfo("Node client is disabled.")
+	}
+
+	// Check if the cache directory exists when caching is enabled
+	if config.DriveCacheEnabled {
+		cacheDir := config.DriveCache.Path
+		imagesDir := filepath.Join(cacheDir, "images")
+
+		// Check if the directory already exists
+		if _, err := os.Stat(imagesDir); os.IsNotExist(err) {
+			// Try to create the directory since it doesn't exist
+			if err := os.MkdirAll(imagesDir, os.ModePerm); err != nil {
+				printErr("Error: Failed to create cache or images directory '%s': %v", imagesDir, err)
+				os.Exit(1) // Exit with a non-zero status to indicate an error
+			}
+			// Print a warning if the directory had to be created
+			printWarn("Warning: Created missing directory '%s'.", imagesDir)
+		}
+	}
+
+	// Start periodic cleanup of expired cache files
+	if config.DriveCacheEnabled {
+		go cleanExpiredCachedImages()
+		printInfo("Drive cache started.")
+	} else {
+		printInfo("Drive cache is disabled.")
+	}
+
+	// Start periodic cleanup of expired cache files
+	if config.RamCacheEnabled {
+		resultsCache = NewResultsCache()
+		geocodeCache = NewGeocodeCache()
+		printInfo("RAM cache started.")
+	} else {
+		printInfo("RAM cache is disabled.")
+	}
+
+	// Init indexer
+	if config.IndexerEnabled {
+		if err := downloadAndSetupDomainsCSV(); err != nil {
+			printErr("Failed to set up domains.csv: %v", err)
+			return
+		}
+
+		err := InitIndex()
+		if err != nil {
+			printErr("Failed to initialize index: %v", err)
+		}
+
+		webCrawlerInit()
+
+		printInfo("Indexer is enabled.")
+	} else {
+		printInfo("Indexer is disabled.")
+	}
+
+	runServer()
+}

init.go

@@ -1,3 +1,6 @@
+//go:build !experimental
+// +build !experimental
+
 package main
 
 import (
@@ -51,22 +54,7 @@ func main() {
 		config.Domain = *domainFlag
 	}
 
-	loadNodeConfig()
+	if config.MetaProxyEnabled {
-
-	if config.AuthCode == "" {
-		config.AuthCode = generateStrongRandomString(64)
-		printInfo("Generated connection code: %s\n", config.AuthCode)
-		saveConfig(config)
-	}
-
-	// Generate Host ID
-	hostID, nodeErr := generateHostID()
-	if nodeErr != nil {
-		printErr("Failed to generate host ID: %v", nodeErr)
-	}
-	config.PeerID = hostID
-
-	if config.CrawlerProxyEnabled || config.MetaProxyEnabled {
 		InitProxies()
 	}
 
@@ -87,14 +75,6 @@ func main() {
 	InitializeLanguage("en") // Initialize language before generating OpenSearch
 	generateOpenSearchXML(config)
 
-	// Start the node client only if NodesEnabled is true
-	if config.NodesEnabled {
-		go startNodeClient()
-		printInfo("Node client started.")
-	} else {
-		printInfo("Node client is disabled.")
-	}
-
 	// Check if the cache directory exists when caching is enabled
 	if config.DriveCacheEnabled {
 		cacheDir := config.DriveCache.Path
@@ -129,24 +109,5 @@ func main() {
 		printInfo("RAM cache is disabled.")
 	}
 
-	// Init indexer
-	if config.IndexerEnabled {
-		if err := downloadAndSetupDomainsCSV(); err != nil {
-			printErr("Failed to set up domains.csv: %v", err)
-			return
-		}
-
-		err := InitIndex()
-		if err != nil {
-			printErr("Failed to initialize index: %v", err)
-		}
-
-		webCrawlerInit()
-
-		printInfo("Indexer is enabled.")
-	} else {
-		printInfo("Indexer is disabled.")
-	}
-
 	runServer()
 }

main.go

@@ -243,10 +243,6 @@ func runServer() {
 		printInfo("Website is disabled.")
 	}
 
-	if config.NodesEnabled {
-		http.HandleFunc("/node", handleNodeRequest)
-	}
-
 	printMessage("Server is listening on http://localhost:%d", config.Port)
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", config.Port), nil))
 }

node-handle-search.go

@@ -1,218 +1,203 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (
 	"encoding/json"
-	"log"
 )
 
-func handleSearchTextMessage(msg Message) {
+type searchParams struct {
-	var searchParams struct {
+	Query        string `json:"query"`
-		Query        string `json:"query"`
+	Safe         string `json:"safe"`
-		Safe         string `json:"safe"`
+	Lang         string `json:"lang"`
-		Lang         string `json:"lang"`
+	Page         int    `json:"page"`
-		Page         int    `json:"page"`
+	ResponseAddr string `json:"responseAddr"`
-		ResponseAddr string `json:"responseAddr"`
+}
+
+func extractTargetFromAddress(addr string) string {
+	if len(addr) > 5 && addr[len(addr)-5:] == ".sock" {
+		return addr[:len(addr)-5]
 	}
-	err := json.Unmarshal([]byte(msg.Content), &searchParams)
+	return addr
-	if err != nil {
+}
-		printWarn("Error parsing search parameters: %v", err)
+
+// Utility to respond to any search
+func respondToSearch(req searchParams, msgType uint8, results any) {
+	if req.ResponseAddr == "" {
+		printErr("ResponseAddr is empty")
 		return
 	}
 
-	printDebug("Received search-text request. ResponseAddr: %s", searchParams.ResponseAddr)
+	respBytes, err := json.Marshal(results)
-
-	results := fetchTextResults(searchParams.Query, searchParams.Safe, searchParams.Lang, searchParams.Page)
-	resultsJSON, err := json.Marshal(results)
 	if err != nil {
-		printWarn("Error marshalling search results: %v", err)
+		printWarn("Failed to marshal results for msg type %d: %v", msgType, err)
 		return
 	}
 
-	responseMsg := Message{
+	resp := Message{
-		ID:      hostID,
+		ID:      generateMessageID(),
-		Type:    "text-results",
+		Type:    msgType,
-		Content: string(resultsJSON),
+		Content: respBytes,
+		Target:  req.ResponseAddr,
 	}
 
-	// Log the address to be used for sending the response
+	err = sendMessage(resp)
-	printDebug("Sending text search results to %s", searchParams.ResponseAddr)
-
-	if searchParams.ResponseAddr == "" {
-		printErr("Error: Response address is empty")
-		return
-	}
-
-	err = sendMessage(searchParams.ResponseAddr, responseMsg)
 	if err != nil {
-		printWarn("Error sending text search results to %s: %v", searchParams.ResponseAddr, err)
+		printWarn("Failed to send search results to %s: %v", req.ResponseAddr, err)
 	}
 }
 
-func handleSearchImageMessage(msg Message) {
+func sendBinaryResponse(req searchParams, msgType uint8, payload []byte, msgID uint32) {
-	var searchParams struct {
+	if req.ResponseAddr == "" {
-		Query        string `json:"query"`
+		printErr("ResponseAddr is empty")
-		Safe         string `json:"safe"`
-		Lang         string `json:"lang"`
-		Page         int    `json:"page"`
-		ResponseAddr string `json:"responseAddr"`
-	}
-	err := json.Unmarshal([]byte(msg.Content), &searchParams)
-	if err != nil {
-		log.Printf("Error parsing search parameters: %v", err)
 		return
 	}
 
-	log.Printf("Received search-image request. ResponseAddr: %s", searchParams.ResponseAddr)
+	resp := Message{
-	results := fetchImageResults(searchParams.Query, searchParams.Safe, searchParams.Lang, searchParams.Page, true)
+		ID:      msgID,
-	resultsJSON, err := json.Marshal(results)
+		Type:    msgType,
+		Content: payload,
+		Target:  req.ResponseAddr,
+	}
+
+	if err := sendMessage(resp); err != nil {
+		printWarn("Failed to send binary search results: %v", err)
+	}
+}
+
+func handleSearchTextMessage(msg Message) {
+	var req searchParams
+	if err := json.Unmarshal([]byte(msg.Content), &req); err != nil {
+		printWarn("Invalid JSON: %v", err)
+		return
+	}
+	printDebug("Received search-text from %s", req.ResponseAddr)
+
+	results := fetchTextResults(req.Query, req.Safe, req.Lang, req.Page)
+	data, err := encodeTextResults(results)
 	if err != nil {
-		log.Printf("Error marshalling search results: %v", err)
+		printWarn("Failed to encode text results: %v", err)
 		return
 	}
 
-	responseMsg := Message{
+	sendBinaryResponse(req, MsgTypeSearchTextResponse, data, msg.ID)
-		ID:      hostID,
-		Type:    "image-results",
-		Content: string(resultsJSON),
-	}
-
-	// Log the address to be used for sending the response
-	log.Printf("Sending image search results to %s", searchParams.ResponseAddr)
-
-	if searchParams.ResponseAddr == "" {
-		log.Printf("Error: Response address is empty")
-		return
-	}
-
-	err = sendMessage(searchParams.ResponseAddr, responseMsg)
-	if err != nil {
-		log.Printf("Error sending image search results to %s: %v", searchParams.ResponseAddr, err)
-	}
 }
 
 func handleSearchVideoMessage(msg Message) {
-	var searchParams struct {
+	var req searchParams
-		Query        string `json:"query"`
+	if err := json.Unmarshal([]byte(msg.Content), &req); err != nil {
-		Safe         string `json:"safe"`
+		printWarn("Invalid JSON: %v", err)
-		Lang         string `json:"lang"`
+		return
-		Page         int    `json:"page"`
-		ResponseAddr string `json:"responseAddr"`
 	}
-	err := json.Unmarshal([]byte(msg.Content), &searchParams)
+	printDebug("Received search-video from %s", req.ResponseAddr)
+
+	results := fetchVideoResults(req.Query, req.Safe, req.Lang, req.Page)
+	data, err := encodeVideoResults(results)
 	if err != nil {
-		log.Printf("Error parsing search parameters: %v", err)
+		printWarn("Failed to encode video results: %v", err)
 		return
 	}
 
-	log.Printf("Received search-video request. ResponseAddr: %s", searchParams.ResponseAddr)
+	sendBinaryResponse(req, MsgTypeSearchVideoResponse, data, msg.ID)
+}
 
-	results := fetchVideoResults(searchParams.Query, searchParams.Safe, searchParams.Lang, searchParams.Page)
+func handleSearchMusicMessage(msg Message) {
-	resultsJSON, err := json.Marshal(results)
+	var req searchParams
+	if err := json.Unmarshal([]byte(msg.Content), &req); err != nil {
+		printWarn("Invalid JSON: %v", err)
+		return
+	}
+	printDebug("Received search-music from %s", req.ResponseAddr)
+
+	results := fetchMusicResults(req.Query, req.Page)
+	data, err := encodeMusicResults(results)
 	if err != nil {
-		log.Printf("Error marshalling search results: %v", err)
+		printWarn("Failed to encode music results: %v", err)
 		return
 	}
 
-	responseMsg := Message{
+	sendBinaryResponse(req, MsgTypeSearchMusicResponse, data, msg.ID)
-		ID:      hostID,
-		Type:    "video-results",
-		Content: string(resultsJSON),
-	}
-
-	log.Printf("Sending video search results to %s", searchParams.ResponseAddr)
-
-	if searchParams.ResponseAddr == "" {
-		log.Printf("Error: Response address is empty")
-		return
-	}
-
-	err = sendMessage(searchParams.ResponseAddr, responseMsg)
-	if err != nil {
-		log.Printf("Error sending video search results to %s: %v", searchParams.ResponseAddr, err)
-	}
 }
 
 func handleSearchFileMessage(msg Message) {
-	var searchParams struct {
+	var req searchParams
-		Query        string `json:"query"`
+	if err := json.Unmarshal([]byte(msg.Content), &req); err != nil {
-		Safe         string `json:"safe"`
+		printWarn("Invalid JSON: %v", err)
-		Lang         string `json:"lang"`
+		return
-		Page         int    `json:"page"`
-		ResponseAddr string `json:"responseAddr"`
 	}
-	err := json.Unmarshal([]byte(msg.Content), &searchParams)
+	printDebug("Received search-file from %s", req.ResponseAddr)
+
+	results := fetchFileResults(req.Query, req.Safe, req.Lang, req.Page)
+	data, err := encodeFileResults(results)
 	if err != nil {
-		log.Printf("Error parsing search parameters: %v", err)
+		printWarn("Failed to encode file results: %v", err)
 		return
 	}
 
-	log.Printf("Received search-file request. ResponseAddr: %s", searchParams.ResponseAddr)
+	sendBinaryResponse(req, MsgTypeSearchFileResponse, data, msg.ID)
-
-	results := fetchFileResults(searchParams.Query, searchParams.Safe, searchParams.Lang, searchParams.Page)
-	resultsJSON, err := json.Marshal(results)
-	if err != nil {
-		log.Printf("Error marshalling search results: %v", err)
-		return
-	}
-
-	responseMsg := Message{
-		ID:      hostID,
-		Type:    "file-results",
-		Content: string(resultsJSON),
-	}
-
-	log.Printf("Sending file search results to %s", searchParams.ResponseAddr)
-
-	if searchParams.ResponseAddr == "" {
-		log.Printf("Error: Response address is empty")
-		return
-	}
-
-	err = sendMessage(searchParams.ResponseAddr, responseMsg)
-	if err != nil {
-		log.Printf("Error sending file search results to %s: %v", searchParams.ResponseAddr, err)
-	}
 }
 
 func handleSearchForumMessage(msg Message) {
-	var searchParams struct {
+	var req searchParams
-		Query        string `json:"query"`
+	if err := json.Unmarshal([]byte(msg.Content), &req); err != nil {
-		Safe         string `json:"safe"`
+		printWarn("Invalid JSON: %v", err)
-		Lang         string `json:"lang"`
+		return
-		Page         int    `json:"page"`
-		ResponseAddr string `json:"responseAddr"`
 	}
-	err := json.Unmarshal([]byte(msg.Content), &searchParams)
+	printDebug("Received search-forum from %s", req.ResponseAddr)
+
+	results := fetchForumResults(req.Query, req.Safe, req.Lang, req.Page)
+	data, err := encodeForumResults(results)
 	if err != nil {
-		log.Printf("Error parsing search parameters: %v", err)
+		printWarn("Failed to encode forum results: %v", err)
 		return
 	}
 
-	log.Printf("Received search-forum request. ResponseAddr: %s", searchParams.ResponseAddr)
+	sendBinaryResponse(req, MsgTypeSearchForumResponse, data, msg.ID)
-
+}
-	results := fetchForumResults(searchParams.Query, searchParams.Safe, searchParams.Lang, searchParams.Page)
+
-	resultsJSON, err := json.Marshal(results)
+func handleSearchImageMessage(msg Message) {
-	if err != nil {
+	var req searchParams
-		log.Printf("Error marshalling search results: %v", err)
+	if err := json.Unmarshal([]byte(msg.Content), &req); err != nil {
-		return
+		printWarn("Invalid JSON: %v", err)
-	}
+		return
-
+	}
-	responseMsg := Message{
+	printDebug("Received image search type %d from %s", msg.Type, req.ResponseAddr)
-		ID:      hostID,
+
-		Type:    "forum-results",
+	var (
-		Content: string(resultsJSON),
+		thumbsNeeded bool
-	}
+		fullNeeded   bool
-
+	)
-	// Log the address to be used for sending the response
+
-	log.Printf("Sending forum search results to %s", searchParams.ResponseAddr)
+	switch msg.Type {
-
+	case MsgTypeSearchImageRawRequest:
-	if searchParams.ResponseAddr == "" {
+		thumbsNeeded = false
-		log.Printf("Error: Response address is empty")
+		fullNeeded = false
-		return
+	case MsgTypeSearchImageThumbRequest:
-	}
+		thumbsNeeded = true
-
+		fullNeeded = false
-	err = sendMessage(searchParams.ResponseAddr, responseMsg)
+	case MsgTypeSearchImageFullRequest:
-	if err != nil {
+		thumbsNeeded = false
-		log.Printf("Error sending forum search results to %s: %v", searchParams.ResponseAddr, err)
+		fullNeeded = true
-	}
+	case MsgTypeSearchImageAllRequest:
+		thumbsNeeded = true
+		fullNeeded = true
+	default:
+		printWarn("Unknown image search type: %d", msg.Type)
+		return
+	}
+
+	results := fetchImageResults(req.Query, req.Safe, req.Lang, req.Page, true, thumbsNeeded)
+
+	if fullNeeded || thumbsNeeded {
+		results = prepareProxiedImages(results, msg.Type)
+	}
+
+	data, err := encodeImageResults(results)
+	if err != nil {
+		printWarn("Failed to encode image results: %v", err)
+		return
+	}
+
+	sendBinaryResponse(req, MsgTypeSearchImageResponse, data, msg.ID)
 }

node-master.go

@@ -1,91 +0,0 @@
-package main
-
-import (
-	"log"
-	"sync"
-	"time"
-)
-
-var (
-	isMaster      bool
-	masterNode    string
-	masterNodeMux sync.RWMutex
-)
-
-const (
-	heartbeatInterval = 5 * time.Second
-	heartbeatTimeout  = 15 * time.Second
-	electionTimeout   = 10 * time.Second
-)
-
-func sendHeartbeats() {
-	for {
-		if !isMaster {
-			return
-		}
-		for _, node := range peers {
-			msg := Message{
-				ID:      hostID,
-				Type:    "heartbeat",
-				Content: authCode,
-			}
-			err := sendMessage(node, msg)
-			if err != nil {
-				log.Printf("Error sending heartbeat to %s: %v", node, err)
-			}
-		}
-		time.Sleep(heartbeatInterval)
-	}
-}
-
-func checkMasterHeartbeat() {
-	for {
-		time.Sleep(heartbeatTimeout)
-		masterNodeMux.RLock()
-		if masterNode == authCode || masterNode == "" {
-			masterNodeMux.RUnlock()
-			continue
-		}
-		masterNodeMux.RUnlock()
-
-		masterNodeMux.Lock()
-		masterNode = ""
-		masterNodeMux.Unlock()
-		startElection()
-	}
-}
-
-func startElection() {
-	masterNodeMux.Lock()
-	defer masterNodeMux.Unlock()
-
-	for _, node := range peers {
-		msg := Message{
-			ID:      hostID,
-			Type:    "election",
-			Content: authCode,
-		}
-		err := sendMessage(node, msg)
-		if err != nil {
-			log.Printf("Error sending election message to %s: %v", node, err)
-		}
-	}
-
-	isMaster = true
-	go sendHeartbeats()
-}
-
-func handleHeartbeat(content string) {
-	masterNodeMux.Lock()
-	defer masterNodeMux.Unlock()
-	masterNode = content
-}
-
-func handleElection(content string) {
-	masterNodeMux.Lock()
-	defer masterNodeMux.Unlock()
-
-	if content < authCode {
-		masterNode = content
-	}
-}

node-request-files.go

@@ -1,19 +1,22 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (
-	"encoding/json"
+	"bytes"
+	"encoding/binary"
 	"fmt"
 	"time"
 )
 
-func tryOtherNodesForFileSearch(query, safe, lang string, page int, visitedNodes []string) []TorrentResult {
+var fileResultsChan = make(chan []TorrentResult)
-	for _, nodeAddr := range peers {
+
-		if contains(visitedNodes, nodeAddr) {
+func tryOtherNodesForFileSearch(query, safe, lang string, page int) []TorrentResult {
-			continue // Skip nodes already visited
+	for _, nodeTarget := range sockets {
-		}
+		results, err := sendFileSearchRequestToNode(nodeTarget, query, safe, lang, page)
-		results, err := sendFileSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
 		if err != nil {
-			printWarn("Error contacting node %s: %v", nodeAddr, err)
+			printWarn("Error contacting node %s: %v", nodeTarget, err)
 			continue
 		}
 		if len(results) > 0 {
@@ -23,60 +26,123 @@ func tryOtherNodesForFileSearch(query, safe, lang string, page int, visitedNodes
 	return nil
 }
 
-func sendFileSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]TorrentResult, error) {
+func sendFileSearchRequestToNode(target, query, safe, lang string, page int) ([]TorrentResult, error) {
-	visitedNodes = append(visitedNodes, nodeAddr)
+	payload, err := encodeSearchTextParams(query, safe, lang, page)
-	searchParams := struct {
-		Query        string   `json:"query"`
-		Safe         string   `json:"safe"`
-		Lang         string   `json:"lang"`
-		Page         int      `json:"page"`
-		ResponseAddr string   `json:"responseAddr"`
-		VisitedNodes []string `json:"visitedNodes"`
-	}{
-		Query:        query,
-		Safe:         safe,
-		Lang:         lang,
-		Page:         page,
-		ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
-		VisitedNodes: visitedNodes,
-	}
-
-	msgBytes, err := json.Marshal(searchParams)
 	if err != nil {
-		return nil, fmt.Errorf("failed to marshal search parameters: %v", err)
+		return nil, fmt.Errorf("encode error: %v", err)
 	}
 
 	msg := Message{
-		ID:      hostID,
+		ID:      generateMessageID(),
-		Type:    "search-file",
+		Type:    MsgTypeSearchFileRequest,
-		Content: string(msgBytes),
+		Content: payload,
+		Target:  target,
 	}
 
-	err = sendMessage(nodeAddr, msg)
+	if err := sendMessage(msg); err != nil {
-	if err != nil {
+		return nil, fmt.Errorf("send error: %v", err)
-		return nil, fmt.Errorf("failed to send search request to node %s: %v", nodeAddr, err)
 	}
 
-	// Wait for results
 	select {
 	case res := <-fileResultsChan:
 		return res, nil
 	case <-time.After(20 * time.Second):
-		return nil, fmt.Errorf("timeout waiting for results from node %s", nodeAddr)
+		return nil, fmt.Errorf("timeout waiting for results from node %s", target)
 	}
 }
 
 func handleFileResultsMessage(msg Message) {
-	var results []TorrentResult
+	results, err := decodeFileResults([]byte(msg.Content))
-	err := json.Unmarshal([]byte(msg.Content), &results)
 	if err != nil {
-		printWarn("Error unmarshalling file results: %v", err)
+		printWarn("Error decoding file results: %v", err)
 		return
 	}
-
 	printDebug("Received file results: %+v", results)
-	// Send results to fileResultsChan
+
 	go func() {
 		fileResultsChan <- results
 	}()
 }
+
+func encodeFileResults(results []TorrentResult) ([]byte, error) {
+	buf := new(bytes.Buffer)
+
+	if err := binary.Write(buf, binary.BigEndian, uint16(len(results))); err != nil {
+		return nil, err
+	}
+
+	for _, r := range results {
+		if err := writeString(buf, r.URL); err != nil {
+			return nil, err
+		}
+		if err := binary.Write(buf, binary.BigEndian, uint32(r.Seeders)); err != nil {
+			return nil, err
+		}
+		if err := binary.Write(buf, binary.BigEndian, uint32(r.Leechers)); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Magnet); err != nil {
+			return nil, err
+		}
+		if err := binary.Write(buf, binary.BigEndian, uint32(r.Views)); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Size); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Title); err != nil {
+			return nil, err
+		}
+	}
+	return buf.Bytes(), nil
+}
+
+func decodeFileResults(data []byte) ([]TorrentResult, error) {
+	buf := bytes.NewReader(data)
+
+	var count uint16
+	if err := binary.Read(buf, binary.BigEndian, &count); err != nil {
+		return nil, err
+	}
+
+	results := make([]TorrentResult, 0, count)
+	for i := 0; i < int(count); i++ {
+		url, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		var seeders, leechers, views uint32
+		if err := binary.Read(buf, binary.BigEndian, &seeders); err != nil {
+			return nil, err
+		}
+		if err := binary.Read(buf, binary.BigEndian, &leechers); err != nil {
+			return nil, err
+		}
+		magnet, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		if err := binary.Read(buf, binary.BigEndian, &views); err != nil {
+			return nil, err
+		}
+		size, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		title, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+
+		results = append(results, TorrentResult{
+			URL:      url,
+			Seeders:  int(seeders),
+			Leechers: int(leechers),
+			Magnet:   magnet,
+			Views:    int(views),
+			Size:     size,
+			Title:    title,
+		})
+	}
+	return results, nil
+}

node-request-forums.go

@@ -1,100 +1,129 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (
-	"encoding/json"
+	"bytes"
+	"encoding/binary"
 	"fmt"
 	"time"
 )
 
 var forumResultsChan = make(chan []ForumSearchResult)
 
-func tryOtherNodesForForumSearch(query, safe, lang string, page int) []ForumSearchResult {
+func sendForumSearchRequestToNode(target, query, safe, lang string, page int, visitedNodes []string) ([]ForumSearchResult, error) {
-	for _, nodeAddr := range peers {
+	payload, err := encodeSearchTextParams(query, safe, lang, page) // reuse text param encoding
-		results, err := sendForumSearchRequestToNode(nodeAddr, query, safe, lang, page, []string{})
-		if err != nil {
-			printWarn("Error contacting node %s: %v", nodeAddr, err)
-			continue
-		}
-		if len(results) > 0 {
-			return results
-		}
-	}
-	return nil
-}
-
-func sendForumSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]ForumSearchResult, error) {
-	// Check if the current node has already been visited
-	for _, node := range visitedNodes {
-		if node == hostID {
-			return nil, fmt.Errorf("loop detected: this node (%s) has already been visited", hostID)
-		}
-	}
-
-	// Add current node to the list of visited nodes
-	visitedNodes = append(visitedNodes, hostID)
-
-	searchParams := struct {
-		Query        string   `json:"query"`
-		Safe         string   `json:"safe"`
-		Lang         string   `json:"lang"`
-		Page         int      `json:"page"`
-		ResponseAddr string   `json:"responseAddr"`
-		VisitedNodes []string `json:"visitedNodes"`
-	}{
-		Query:        query,
-		Safe:         safe,
-		Lang:         lang,
-		Page:         page,
-		ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
-		VisitedNodes: visitedNodes,
-	}
-
-	msgBytes, err := json.Marshal(searchParams)
 	if err != nil {
-		return nil, fmt.Errorf("failed to marshal search parameters: %v", err)
+		return nil, fmt.Errorf("encode error: %v", err)
 	}
 
 	msg := Message{
-		ID:      hostID,
+		ID:      generateMessageID(),
-		Type:    "search-forum",
+		Type:    MsgTypeSearchForumRequest,
-		Content: string(msgBytes),
+		Content: payload,
+		Target:  target,
 	}
 
-	err = sendMessage(nodeAddr, msg)
+	if err := sendMessage(msg); err != nil {
-	if err != nil {
+		return nil, fmt.Errorf("send error: %v", err)
-		return nil, fmt.Errorf("failed to send search request to node %s: %v", nodeAddr, err)
 	}
 
-	// Wait for results
 	select {
 	case res := <-forumResultsChan:
 		return res, nil
 	case <-time.After(20 * time.Second):
-		return nil, fmt.Errorf("timeout waiting for results from node %s", nodeAddr)
+		return nil, fmt.Errorf("timeout waiting for results from node %s", target)
 	}
 }
 
 func handleForumResultsMessage(msg Message) {
-	var results []ForumSearchResult
+	results, err := decodeForumResults([]byte(msg.Content))
-	err := json.Unmarshal([]byte(msg.Content), &results)
 	if err != nil {
-		printWarn("Error unmarshalling forum results: %v", err)
+		printWarn("Error decoding forum results: %v", err)
 		return
 	}
-
 	printDebug("Received forum results: %+v", results)
-	// Send results to forumResultsChan
+
 	go func() {
 		forumResultsChan <- results
 	}()
 }
 
-// Used only to answer requests
+func encodeForumResults(results []ForumSearchResult) ([]byte, error) {
-func fetchForumResults(query, safe, lang string, page int) []ForumSearchResult {
+	buf := new(bytes.Buffer)
-	results, err := PerformRedditSearch(query, safe, page)
+
-	if err != nil {
+	if err := binary.Write(buf, binary.BigEndian, uint16(len(results))); err != nil {
-		printWarn("Error fetching forum results: %v", err)
+		return nil, err
-		return nil
 	}
-	return results
+
+	for _, r := range results {
+		if err := writeString(buf, r.URL); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Header); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Description); err != nil {
+			return nil, err
+		}
+		if err := binary.Write(buf, binary.BigEndian, r.PublishedDate.Unix()); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.ImgSrc); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.ThumbnailSrc); err != nil {
+			return nil, err
+		}
+	}
+
+	return buf.Bytes(), nil
+}
+
+func decodeForumResults(data []byte) ([]ForumSearchResult, error) {
+	buf := bytes.NewReader(data)
+
+	var count uint16
+	if err := binary.Read(buf, binary.BigEndian, &count); err != nil {
+		return nil, err
+	}
+
+	results := make([]ForumSearchResult, 0, count)
+	for i := 0; i < int(count); i++ {
+		url, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		header, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		desc, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		var unixTime int64
+		if err := binary.Read(buf, binary.BigEndian, &unixTime); err != nil {
+			return nil, err
+		}
+		imgSrc, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		thumbSrc, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+
+		results = append(results, ForumSearchResult{
+			URL:           url,
+			Header:        header,
+			Description:   desc,
+			PublishedDate: time.Unix(unixTime, 0),
+			ImgSrc:        imgSrc,
+			ThumbnailSrc:  thumbSrc,
+		})
+	}
+	return results, nil
 }

node-request-images.go

@@ -1,84 +1,198 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (
-	"encoding/json"
+	"bytes"
+	"encoding/binary"
 	"fmt"
 	"time"
 )
 
+// const (
+// 	MessageTypeSearchImage         uint8 = 11
+// 	MessageTypeSearchImageThumb    uint8 = 111
+// 	MessageTypeSearchImageFull     uint8 = 112
+// 	MessageTypeSearchImageAllProxy uint8 = 113
+// 	MessageTypeImageResults        uint8 = 22
+// )
+
 var imageResultsChan = make(chan []ImageSearchResult)
 
-func handleImageResultsMessage(msg Message) {
+func sendImageSearchRequestToNode(target, query, safe, lang string, page int, mode uint8) ([]ImageSearchResult, error) {
-	var results []ImageSearchResult
+	payload, err := encodeSearchTextParams(query, safe, lang, page)
-	err := json.Unmarshal([]byte(msg.Content), &results)
 	if err != nil {
-		printWarn("Error unmarshalling image results: %v", err)
+		return nil, fmt.Errorf("encode error: %v", err)
-		return
 	}
 
+	msg := Message{
+		ID:      generateMessageID(),
+		Type:    mode, // one of the image search types
+		Content: payload,
+		Target:  target,
+	}
+
+	if err := sendMessage(msg); err != nil {
+		return nil, fmt.Errorf("send error: %v", err)
+	}
+
+	select {
+	case res := <-imageResultsChan:
+		return res, nil
+	case <-time.After(30 * time.Second):
+		return nil, fmt.Errorf("timeout from %s", target)
+	}
+}
+
+func handleImageResultsMessage(msg Message) {
+	results, err := decodeImageResults([]byte(msg.Content))
+	if err != nil {
+		printWarn("Error decoding image results: %v", err)
+		return
+	}
 	printDebug("Received image results: %+v", results)
-	// Send results to imageResultsChan
+
 	go func() {
 		imageResultsChan <- results
 	}()
 }
 
-func sendImageSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]ImageSearchResult, error) {
+func encodeImageResults(results []ImageSearchResult) ([]byte, error) {
-	visitedNodes = append(visitedNodes, nodeAddr)
+	buf := new(bytes.Buffer)
-	searchParams := struct {
+
-		Query        string   `json:"query"`
+	if err := binary.Write(buf, binary.BigEndian, uint16(len(results))); err != nil {
-		Safe         string   `json:"safe"`
+		return nil, err
-		Lang         string   `json:"lang"`
-		Page         int      `json:"page"`
-		ResponseAddr string   `json:"responseAddr"`
-		VisitedNodes []string `json:"visitedNodes"`
-	}{
-		Query:        query,
-		Safe:         safe,
-		Lang:         lang,
-		Page:         page,
-		ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
-		VisitedNodes: visitedNodes,
 	}
 
-	msgBytes, err := json.Marshal(searchParams)
+	for _, r := range results {
-	if err != nil {
+		if err := writeString(buf, r.ID); err != nil {
-		return nil, fmt.Errorf("failed to marshal search parameters: %v", err)
+			return nil, err
+		}
+		if err := writeString(buf, r.Title); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Full); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Thumb); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.ProxyFull); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.ProxyThumb); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Source); err != nil {
+			return nil, err
+		}
+		if err := binary.Write(buf, binary.BigEndian, int32(r.Width)); err != nil {
+			return nil, err
+		}
+		if err := binary.Write(buf, binary.BigEndian, int32(r.Height)); err != nil {
+			return nil, err
+		}
 	}
 
-	msg := Message{
+	return buf.Bytes(), nil
-		ID:      hostID,
-		Type:    "search-image",
-		Content: string(msgBytes),
-	}
-
-	err = sendMessage(nodeAddr, msg)
-	if err != nil {
-		return nil, fmt.Errorf("failed to send search request to node %s: %v", nodeAddr, err)
-	}
-
-	// Wait for results
-	select {
-	case res := <-imageResultsChan:
-		return res, nil
-	case <-time.After(30 * time.Second):
-		return nil, fmt.Errorf("timeout waiting for results from node %s", nodeAddr)
-	}
 }
 
-func tryOtherNodesForImageSearch(query, safe, lang string, page int, visitedNodes []string) []ImageSearchResult {
+func decodeImageResults(data []byte) ([]ImageSearchResult, error) {
-	for _, nodeAddr := range peers {
+	buf := bytes.NewReader(data)
-		if contains(visitedNodes, nodeAddr) {
+
-			continue // Skip nodes already visited
+	var count uint16
+	if err := binary.Read(buf, binary.BigEndian, &count); err != nil {
+		return nil, err
+	}
+
+	results := make([]ImageSearchResult, 0, count)
+	for i := 0; i < int(count); i++ {
+		id, _ := readString(buf)
+		title, _ := readString(buf)
+		full, _ := readString(buf)
+		thumb, _ := readString(buf)
+		proxyFull, _ := readString(buf)
+		proxyThumb, _ := readString(buf)
+		source, _ := readString(buf)
+
+		var width, height int32
+		if err := binary.Read(buf, binary.BigEndian, &width); err != nil {
+			return nil, err
 		}
-		results, err := sendImageSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
+		if err := binary.Read(buf, binary.BigEndian, &height); err != nil {
-		if err != nil {
+			return nil, err
-			printWarn("Error contacting node %s: %v", nodeAddr, err)
-			continue
 		}
-		if len(results) > 0 {
+
-			return results
+		results = append(results, ImageSearchResult{
+			ID:         id,
+			Title:      title,
+			Full:       full,
+			Thumb:      thumb,
+			ProxyFull:  proxyFull,
+			ProxyThumb: proxyThumb,
+			Source:     source,
+			Width:      int(width),
+			Height:     int(height),
+		})
+	}
+
+	return results, nil
+}
+
+func prepareProxiedImages(results []ImageSearchResult, mode uint8) []ImageSearchResult {
+	for i := range results {
+		switch mode {
+		case MsgTypeSearchImageThumbRequest:
+			results[i].ProxyThumb = "/image/" + results[i].ID + "_thumb.webp"
+		case MsgTypeSearchImageFullRequest:
+			results[i].ProxyFull = "/image/" + results[i].ID + "_full.webp"
+		case MsgTypeSearchImageAllRequest:
+			results[i].ProxyThumb = "/image/" + results[i].ID + "_thumb.webp"
+			results[i].ProxyFull = "/image/" + results[i].ID + "_full.webp"
 		}
 	}
-	return nil
+	return results
 }
+
+// func handleSearchImageMessage(msg Message) {
+// 	query, safe, lang, page, err := decodeSearchTextParams([]byte(msg.Content))
+// 	if err != nil {
+// 		printWarn("Error decoding image search parameters: %v", err)
+// 		return
+// 	}
+
+// 	results := fetchImageResults(query, safe, lang, page, true, true)
+
+// 	switch msg.Type {
+// 	case MsgTypeSearchImageRawRequest:
+// 		// No proxy URLs needed
+
+// 	case MsgTypeSearchImageThumbRequest:
+// 		results = prepareProxiedImages(results, MsgTypeSearchImageThumbRequest)
+// 	case MsgTypeSearchImageFullRequest:
+// 		results = prepareProxiedImages(results, MsgTypeSearchImageFullRequest)
+// 	case MsgTypeSearchImageAllRequest:
+// 		results = prepareProxiedImages(results, MsgTypeSearchImageAllRequest)
+// 	default:
+// 		printWarn("Unknown image request mode: %d", msg.Type)
+// 		return
+// 	}
+
+// 	payload, err := encodeImageResults(results)
+// 	if err != nil {
+// 		printWarn("Error encoding image search results: %v", err)
+// 		return
+// 	}
+
+// 	response := Message{
+// 		ID:      msg.ID,
+// 		Type:    MsgTypeSearchImageResponse,
+// 		Content: string(payload),
+// 		Target:  msg.Source, // Reply to sender
+// 	}
+
+// 	if err := sendMessage(response); err != nil {
+// 		printWarn("Error sending image search response: %v", err)
+// 	}
+// }

node-request-music.go

@@ -0,0 +1,187 @@
+//go:build experimental
+// +build experimental
+
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"time"
+)
+
+var musicResultsChan = make(chan []MusicResult)
+
+func tryOtherNodesForMusicSearch(query, lang string, safe bool, page int) []MusicResult {
+	safeStr := "inactive"
+	if safe {
+		safeStr = "active"
+	}
+	for _, nodeTarget := range sockets {
+		results, err := sendMusicSearchRequestToNode(nodeTarget, query, safeStr, lang, page)
+		if err != nil {
+			printWarn("Error contacting node %s: %v", nodeTarget, err)
+			continue
+		}
+		if len(results) > 0 {
+			return results
+		}
+	}
+	return nil
+}
+
+func sendMusicSearchRequestToNode(target, query, safe, lang string, page int) ([]MusicResult, error) {
+	payload, err := encodeSearchTextParams(query, safe, lang, page)
+	if err != nil {
+		return nil, fmt.Errorf("encode error: %v", err)
+	}
+
+	msg := Message{
+		ID:      generateMessageID(),
+		Type:    MsgTypeSearchMusicRequest,
+		Content: payload,
+		Target:  target,
+	}
+
+	err = sendMessage(msg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send music request to node %s: %v", target, err)
+	}
+
+	select {
+	case res := <-musicResultsChan:
+		return res, nil
+	case <-time.After(20 * time.Second):
+		return nil, fmt.Errorf("timeout waiting for music results from node %s", target)
+	}
+}
+
+func handleMusicResultsMessage(msg Message) {
+	results, err := decodeMusicResults([]byte(msg.Content))
+	if err != nil {
+		printWarn("Error decoding music results: %v", err)
+		return
+	}
+	printDebug("Received music results: %+v", results)
+
+	go func() {
+		musicResultsChan <- results
+	}()
+}
+
+func encodeMusicResults(results []MusicResult) ([]byte, error) {
+	buf := new(bytes.Buffer)
+
+	if err := binary.Write(buf, binary.BigEndian, uint16(len(results))); err != nil {
+		return nil, err
+	}
+
+	for _, r := range results {
+		if err := writeString(buf, r.URL); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Title); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Artist); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Description); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.PublishedDate); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Thumbnail); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Source); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Duration); err != nil {
+			return nil, err
+		}
+	}
+
+	return buf.Bytes(), nil
+}
+
+func decodeMusicResults(data []byte) ([]MusicResult, error) {
+	buf := bytes.NewReader(data)
+
+	var count uint16
+	if err := binary.Read(buf, binary.BigEndian, &count); err != nil {
+		return nil, err
+	}
+
+	results := make([]MusicResult, 0, count)
+	for i := 0; i < int(count); i++ {
+		url, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		title, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		artist, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		description, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		date, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		thumb, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		source, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		duration, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+
+		results = append(results, MusicResult{
+			URL:           url,
+			Title:         title,
+			Artist:        artist,
+			Description:   description,
+			PublishedDate: date,
+			Thumbnail:     thumb,
+			Source:        source,
+			Duration:      duration,
+		})
+	}
+	return results, nil
+}
+
+// func handleMusicSearchRequest(msg Message) {
+// 	buf := bytes.NewReader([]byte(msg.Content))
+// 	query, _ := readString(buf)
+
+// 	var page uint16
+// 	binary.Read(buf, binary.BigEndian, &page)
+
+// 	results := fetchMusicResults(query, int(page))
+// 	encoded, err := encodeMusicResults(results)
+// 	if err != nil {
+// 		printWarn("Encoding music results failed: %v", err)
+// 		return
+// 	}
+
+// 	reply := Message{
+// 		ID:      msg.ID,
+// 		Type:    MsgTypeSearchMusicResponse,
+// 		Content: string(encoded),
+// 		Target:  msg.Target, // Send back to sender
+// 	}
+// 	sendMessage(reply)
+// }

node-request-text.go

@@ -1,21 +1,23 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (
-	"encoding/json"
+	"bytes"
+	"encoding/binary"
 	"fmt"
 	"time"
 )
 
 var textResultsChan = make(chan []TextSearchResult)
 
-func tryOtherNodesForTextSearch(query, safe, lang string, page int, visitedNodes []string) []TextSearchResult {
+// Try other nodes is not defined for every type
-	for _, nodeAddr := range peers {
+func tryOtherNodesForTextSearch(query, safe, lang string, page int) []TextSearchResult {
-		if contains(visitedNodes, nodeAddr) {
+	for _, nodeTarget := range sockets {
-			continue // Skip nodes already visited
+		results, err := sendTextSearchRequestToNode(nodeTarget, query, safe, lang, page)
-		}
-		results, err := sendTextSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
 		if err != nil {
-			printWarn("Error contacting node %s: %v", nodeAddr, err)
+			printWarn("Error contacting node %s: %v", nodeTarget, err)
 			continue
 		}
 		if len(results) > 0 {
@@ -25,60 +27,102 @@ func tryOtherNodesForTextSearch(query, safe, lang string, page int, visitedNodes
 	return nil
 }
 
-func sendTextSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]TextSearchResult, error) {
+func sendTextSearchRequestToNode(target, query, safe, lang string, page int) ([]TextSearchResult, error) {
-	visitedNodes = append(visitedNodes, nodeAddr)
+	payload, err := encodeSearchTextParams(query, safe, lang, page)
-	searchParams := struct {
-		Query        string   `json:"query"`
-		Safe         string   `json:"safe"`
-		Lang         string   `json:"lang"`
-		Page         int      `json:"page"`
-		ResponseAddr string   `json:"responseAddr"`
-		VisitedNodes []string `json:"visitedNodes"`
-	}{
-		Query:        query,
-		Safe:         safe,
-		Lang:         lang,
-		Page:         page,
-		ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
-		VisitedNodes: visitedNodes,
-	}
-
-	msgBytes, err := json.Marshal(searchParams)
 	if err != nil {
-		return nil, fmt.Errorf("failed to marshal search parameters: %v", err)
+		return nil, fmt.Errorf("encode error: %v", err)
 	}
 
 	msg := Message{
-		ID:      hostID,
+		ID:      generateMessageID(), // assume function returns uint32
-		Type:    "search-text",
+		Type:    MsgTypeSearchTextRequest,
-		Content: string(msgBytes),
+		Content: payload,
+		Target:  target,
 	}
 
-	err = sendMessage(nodeAddr, msg)
+	err = sendMessage(msg)
 	if err != nil {
-		return nil, fmt.Errorf("failed to send search request to node %s: %v", nodeAddr, err)
+		return nil, fmt.Errorf("failed to send search request to node %s: %v", target, err)
 	}
 
-	// Wait for results
 	select {
 	case res := <-textResultsChan:
 		return res, nil
 	case <-time.After(20 * time.Second):
-		return nil, fmt.Errorf("timeout waiting for results from node %s", nodeAddr)
+		return nil, fmt.Errorf("timeout waiting for results from node %s", target)
 	}
 }
 
 func handleTextResultsMessage(msg Message) {
-	var results []TextSearchResult
+	results, err := decodeTextResults([]byte(msg.Content))
-	err := json.Unmarshal([]byte(msg.Content), &results)
 	if err != nil {
-		printWarn("Error unmarshalling text results: %v", err)
+		printWarn("Error decoding text results: %v", err)
 		return
 	}
-
 	printDebug("Received text results: %+v", results)
-	// Send results to textResultsChan
+
 	go func() {
 		textResultsChan <- results
 	}()
 }
+
+func encodeTextResults(results []TextSearchResult) ([]byte, error) {
+	buf := new(bytes.Buffer)
+
+	if err := binary.Write(buf, binary.BigEndian, uint16(len(results))); err != nil {
+		return nil, err
+	}
+
+	for _, r := range results {
+		if err := writeString(buf, r.URL); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Header); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Description); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Source); err != nil {
+			return nil, err
+		}
+	}
+
+	return buf.Bytes(), nil
+}
+
+func decodeTextResults(data []byte) ([]TextSearchResult, error) {
+	buf := bytes.NewReader(data)
+
+	var count uint16
+	if err := binary.Read(buf, binary.BigEndian, &count); err != nil {
+		return nil, err
+	}
+
+	results := make([]TextSearchResult, 0, count)
+	for i := 0; i < int(count); i++ {
+		url, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		header, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		description, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		source, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		results = append(results, TextSearchResult{
+			URL:         url,
+			Header:      header,
+			Description: description,
+			Source:      source,
+		})
+	}
+	return results, nil
+}

node-request-video.go

@@ -1,19 +1,22 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (
-	"encoding/json"
+	"bytes"
+	"encoding/binary"
 	"fmt"
 	"time"
 )
 
-func tryOtherNodesForVideoSearch(query, safe, lang string, page int, visitedNodes []string) []VideoResult {
+var videoResultsChan = make(chan []VideoResult)
-	for _, nodeAddr := range peers {
+
-		if contains(visitedNodes, nodeAddr) {
+func tryOtherNodesForVideoSearch(query, safe, lang string, page int) []VideoResult {
-			continue // Skip nodes already visited
+	for _, node := range sockets {
-		}
+		results, err := sendVideoSearchRequestToNode(node, query, safe, lang, page)
-		results, err := sendVideoSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
 		if err != nil {
-			printWarn("Error contacting node %s: %v", nodeAddr, err)
+			printWarn("Error contacting node %s: %v", node, err)
 			continue
 		}
 		if len(results) > 0 {
@@ -23,60 +26,134 @@ func tryOtherNodesForVideoSearch(query, safe, lang string, page int, visitedNode
 	return nil
 }
 
-func sendVideoSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]VideoResult, error) {
+func sendVideoSearchRequestToNode(target, query, safe, lang string, page int) ([]VideoResult, error) {
-	visitedNodes = append(visitedNodes, nodeAddr)
+	payload, err := encodeSearchTextParams(query, safe, lang, page)
-	searchParams := struct {
-		Query        string   `json:"query"`
-		Safe         string   `json:"safe"`
-		Lang         string   `json:"lang"`
-		Page         int      `json:"page"`
-		ResponseAddr string   `json:"responseAddr"`
-		VisitedNodes []string `json:"visitedNodes"`
-	}{
-		Query:        query,
-		Safe:         safe,
-		Lang:         lang,
-		Page:         page,
-		ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
-		VisitedNodes: visitedNodes,
-	}
-
-	msgBytes, err := json.Marshal(searchParams)
 	if err != nil {
-		return nil, fmt.Errorf("failed to marshal search parameters: %v", err)
+		return nil, fmt.Errorf("encode error: %v", err)
 	}
 
 	msg := Message{
-		ID:      hostID,
+		ID:      generateMessageID(),
-		Type:    "search-video",
+		Type:    MsgTypeSearchVideoRequest,
-		Content: string(msgBytes),
+		Content: payload,
+		Target:  target,
 	}
 
-	err = sendMessage(nodeAddr, msg)
+	if err := sendMessage(msg); err != nil {
-	if err != nil {
+		return nil, fmt.Errorf("send error: %v", err)
-		return nil, fmt.Errorf("failed to send search request to node %s: %v", nodeAddr, err)
 	}
 
-	// Wait for results
 	select {
 	case res := <-videoResultsChan:
 		return res, nil
 	case <-time.After(20 * time.Second):
-		return nil, fmt.Errorf("timeout waiting for results from node %s", nodeAddr)
+		return nil, fmt.Errorf("timeout waiting for results from node %s", target)
 	}
 }
 
 func handleVideoResultsMessage(msg Message) {
-	var results []VideoResult
+	results, err := decodeVideoResults([]byte(msg.Content))
-	err := json.Unmarshal([]byte(msg.Content), &results)
 	if err != nil {
-		printWarn("Error unmarshalling video results: %v", err)
+		printWarn("Error decoding video results: %v", err)
 		return
 	}
-
 	printDebug("Received video results: %+v", results)
-	// Send results to videoResultsChan
+
 	go func() {
 		videoResultsChan <- results
 	}()
 }
+
+func encodeVideoResults(results []VideoResult) ([]byte, error) {
+	buf := new(bytes.Buffer)
+
+	if err := binary.Write(buf, binary.BigEndian, uint16(len(results))); err != nil {
+		return nil, err
+	}
+
+	for _, r := range results {
+		if err := writeString(buf, r.Href); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Title); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Date); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Views); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Creator); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Publisher); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Image); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Duration); err != nil {
+			return nil, err
+		}
+	}
+
+	return buf.Bytes(), nil
+}
+
+func decodeVideoResults(data []byte) ([]VideoResult, error) {
+	buf := bytes.NewReader(data)
+
+	var count uint16
+	if err := binary.Read(buf, binary.BigEndian, &count); err != nil {
+		return nil, err
+	}
+
+	results := make([]VideoResult, 0, count)
+	for i := 0; i < int(count); i++ {
+		href, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		title, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		date, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		views, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		creator, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		publisher, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		image, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		duration, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+
+		results = append(results, VideoResult{
+			Href:      href,
+			Title:     title,
+			Date:      date,
+			Views:     views,
+			Creator:   creator,
+			Publisher: publisher,
+			Image:     image,
+			Duration:  duration,
+		})
+	}
+	return results, nil
+}

node-update.go

@@ -1,28 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"time"
-)
-
-// Function to sync updates across all nodes
-func nodeUpdateSync() {
-	fmt.Println("Syncing updates across all nodes...")
-	for _, peerAddr := range peers {
-		fmt.Printf("Notifying node %s about update...\n", peerAddr)
-		msg := Message{
-			ID:      hostID,
-			Type:    "update",
-			Content: "Start update process",
-		}
-		err := sendMessage(peerAddr, msg)
-		if err != nil {
-			log.Printf("Failed to notify node %s: %v\n", peerAddr, err)
-			continue
-		}
-		fmt.Printf("Node %s notified. Waiting for it to update...\n", peerAddr)
-		time.Sleep(30 * time.Second) // Adjust sleep time as needed to allow for updates
-	}
-	fmt.Println("All nodes have been updated.")
-}

node.go

@@ -1,78 +1,152 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (
 	"bytes"
-	"crypto/rand"
+	"encoding/binary"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
+	"os"
 	"time"
 )
 
 var (
-	authCode string
+	sockets   []string
-	peers    []string
+	hostID    string
-	hostID   string
+	socketDir string
 )
 
 type Message struct {
-	ID           string   `json:"id"`
+	ID      uint32
-	Type         string   `json:"type"`
+	Type    uint8
-	Content      string   `json:"content"`
+	Content []byte
-	VisitedNodes []string `json:"visitedNodes"`
+	Target  string
 }
 
+const (
+	MsgTypeNone uint8 = 0
+	MsgTypeTest uint8 = 1
+
+	// Request types (10–99)
+	MsgTypeSearchTextRequest       uint8 = 10
+	MsgTypeSearchImageRawRequest   uint8 = 11
+	MsgTypeSearchImageThumbRequest uint8 = 12
+	MsgTypeSearchImageFullRequest  uint8 = 13
+	MsgTypeSearchImageAllRequest   uint8 = 14
+	MsgTypeSearchVideoRequest      uint8 = 15
+	MsgTypeSearchFileRequest       uint8 = 16
+	MsgTypeSearchForumRequest      uint8 = 17
+	MsgTypeSearchMusicRequest      uint8 = 18
+
+	// Response types (110–199)
+	MsgTypeSearchTextResponse  uint8 = 110
+	MsgTypeSearchImageResponse uint8 = 111
+	MsgTypeSearchVideoResponse uint8 = 112
+	MsgTypeSearchFileResponse  uint8 = 113
+	MsgTypeSearchForumResponse uint8 = 114
+	MsgTypeSearchMusicResponse uint8 = 115
+)
+
 func loadNodeConfig() {
-	authCode = config.AuthCode
+	sockets = config.Nodes
-	peers = config.Peers
+	socketDir = "/tmp/" // Directory where sockets are stored, for now fixed tmp dir, can be changed later
 }
 
-func generateHostID() (string, error) {
+var messageIDCounter uint32 = 0
-	bytes := make([]byte, 16)
+
-	_, err := rand.Read(bytes)
+func generateMessageID() uint32 {
-	if err != nil {
+	if messageIDCounter == ^uint32(0) { // 0xFFFFFFFF
-		return "", fmt.Errorf("failed to generate host ID: %v", err)
+		messageIDCounter = 1
+	} else {
+		messageIDCounter++
 	}
-	return fmt.Sprintf("%x", bytes), nil
+	return messageIDCounter
 }
 
-func sendMessage(serverAddr string, msg Message) error {
+func encodeSearchTextParams(query, safe, lang string, page int) ([]byte, error) {
-	if serverAddr == "" {
+	buf := new(bytes.Buffer)
-		return fmt.Errorf("server address is empty")
+
+	if err := writeString(buf, query); err != nil {
+		return nil, err
+	}
+	if err := writeString(buf, safe); err != nil {
+		return nil, err
+	}
+	if err := writeString(buf, lang); err != nil {
+		return nil, err
+	}
+	if err := binary.Write(buf, binary.BigEndian, uint16(page)); err != nil {
+		return nil, err
 	}
 
-	msgBytes, err := json.Marshal(msg)
+	return buf.Bytes(), nil
+}
+
+func sendMessage(msg Message) error {
+	socketPath := socketDir + msg.Target + ".sock"
+
+	conn, err := net.Dial("unix", socketPath)
 	if err != nil {
-		return fmt.Errorf("failed to marshal message: %v", err)
+		return fmt.Errorf("failed to connect to socket %s: %v", socketPath, err)
 	}
+	defer conn.Close()
 
-	req, err := http.NewRequest("POST", serverAddr, bytes.NewBuffer(msgBytes))
+	msgBytes, err := serializeMessage(msg)
 	if err != nil {
-		return fmt.Errorf("failed to create request: %v", err)
+		return fmt.Errorf("serialization error: %v", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Authorization", authCode)
-
-	client := &http.Client{
-		Timeout: time.Second * 10,
 	}
 
-	resp, err := client.Do(req)
+	_, err = conn.Write(msgBytes)
+	return err
+}
+
+func startUnixSocketServer(socketName string) {
+	socketPath := socketDir + socketName + ".sock"
+
+	if _, err := os.Stat(socketPath); err == nil {
+		os.Remove(socketPath)
+	}
+
+	listener, err := net.Listen("unix", socketPath)
 	if err != nil {
-		return fmt.Errorf("failed to send request: %v", err)
+		panic(fmt.Sprintf("Failed to listen on %s: %v", socketPath, err))
 	}
-	defer resp.Body.Close()
+	defer listener.Close()
+	os.Chmod(socketPath, 0666)
 
-	if resp.StatusCode != http.StatusOK {
+	printInfo("Listening on UNIX socket: %s", socketPath)
-		body, err := io.ReadAll(resp.Body)
+
+	for {
+		conn, err := listener.Accept()
 		if err != nil {
-			return fmt.Errorf("failed to read response body: %v", err)
+			printWarn("Accept error: %v", err)
+			continue
 		}
-		return fmt.Errorf("server error: %s", body)
-	}
 
-	return nil
+		go func(c net.Conn) {
+			defer c.Close()
+			buf, err := io.ReadAll(c)
+			if err != nil {
+				printWarn("Read error: %v", err)
+				return
+			}
+
+			msg, err := deserializeMessage(buf)
+			if err != nil {
+				printWarn("Deserialization error: %v", err)
+				return
+			}
+
+			printDebug("Received binary message: %+v", msg)
+			interpretMessage(msg)
+		}(conn)
+	}
 }
 
 func handleNodeRequest(w http.ResponseWriter, r *http.Request) {
@@ -81,12 +155,6 @@ func handleNodeRequest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	auth := r.Header.Get("Authorization")
-	if auth != authCode {
-		http.Error(w, "Unauthorized", http.StatusUnauthorized)
-		return
-	}
-
 	var msg Message
 	err := json.NewDecoder(r.Body).Decode(&msg)
 	if err != nil {
@@ -95,64 +163,177 @@ func handleNodeRequest(w http.ResponseWriter, r *http.Request) {
 	}
 	defer r.Body.Close()
 
-	printDebug("Received message: %+v\n", msg)
+	printDebug("Received HTTP message: %+v", msg)
-	w.Write([]byte("Message received"))
-
 	interpretMessage(msg)
 }
 
-func startNodeClient() {
+func startNodeClientUnix() {
-	for {
+	var idCounter uint32 = 0
-		for _, peerAddr := range peers {
-			msg := Message{
-				ID:      hostID,
-				Type:    "test",
-				Content: "This is a test message from the client node",
-			}
 
-			err := sendMessage(peerAddr, msg)
+	for {
-			if err != nil {
+		msg := Message{
-				printWarn("Error sending message to %s: %v", peerAddr, err)
+			ID:      idCounter,
-			} else {
+			Type:    MsgTypeTest,
-				printInfo("Message sent successfully to: %s", peerAddr)
+			Content: []byte("This is a test message via UNIX socket"),
-			}
+			Target:  "node2", ///!!!
+		}
+		idCounter++
+
+		if err := sendMessage(msg); err != nil {
+			printWarn("Send error: %v", err)
 		}
 		time.Sleep(10 * time.Second)
 	}
 }
 
 func interpretMessage(msg Message) {
+	printDebug("Received message: %s", msg.Content)
+
 	switch msg.Type {
-	case "test":
+	case MsgTypeTest:
-		printDebug("Received test message: %v", msg.Content)
+		handleTestMessage(msg)
-	case "update":
+	case MsgTypeSearchTextRequest:
-		printDebug("Received update message: %v", msg.Content)
-		go update()
-	case "heartbeat":
-		handleHeartbeat(msg.Content)
-	case "election":
-		handleElection(msg.Content)
-	case "search-text":
 		handleSearchTextMessage(msg)
-	case "search-image":
+	case MsgTypeSearchImageRawRequest, MsgTypeSearchImageThumbRequest, MsgTypeSearchImageFullRequest, MsgTypeSearchImageAllRequest:
 		handleSearchImageMessage(msg)
-	case "search-video":
+	case MsgTypeSearchVideoRequest:
 		handleSearchVideoMessage(msg)
-	case "search-file":
+	case MsgTypeSearchFileRequest:
 		handleSearchFileMessage(msg)
-	case "search-forum":
+	case MsgTypeSearchForumRequest:
 		handleSearchForumMessage(msg)
-	case "forum-results":
+	case MsgTypeSearchMusicRequest:
-		handleForumResultsMessage(msg)
+		handleSearchMusicMessage(msg)
-	case "text-results":
+
+	case MsgTypeSearchTextResponse:
 		handleTextResultsMessage(msg)
-	case "image-results":
+	case MsgTypeSearchImageResponse:
 		handleImageResultsMessage(msg)
-	case "video-results":
+	case MsgTypeSearchVideoResponse:
 		handleVideoResultsMessage(msg)
-	case "file-results":
+	case MsgTypeSearchFileResponse:
 		handleFileResultsMessage(msg)
+	case MsgTypeSearchForumResponse:
+		handleForumResultsMessage(msg)
+	case MsgTypeSearchMusicResponse:
+		handleMusicResultsMessage(msg)
+
 	default:
-		printWarn("Received unknown message type: %v", msg.Type)
+		printWarn("Unknown message type: %d", msg.Type)
+	}
+}
+
+// Serialize Message to binary
+func serializeMessage(msg Message) ([]byte, error) {
+	buf := new(bytes.Buffer)
+
+	if err := binary.Write(buf, binary.BigEndian, msg.ID); err != nil {
+		return nil, err
+	}
+	if err := binary.Write(buf, binary.BigEndian, msg.Type); err != nil {
+		return nil, err
+	}
+
+	// Content
+	contentBytes := []byte(msg.Content)
+	if len(contentBytes) > 65535 {
+		return nil, errors.New("content too long")
+	}
+	if err := binary.Write(buf, binary.BigEndian, uint16(len(contentBytes))); err != nil {
+		return nil, err
+	}
+	buf.Write(contentBytes)
+
+	// Target
+	targetBytes := []byte(msg.Target)
+	if len(targetBytes) > 255 {
+		return nil, errors.New("target name too long")
+	}
+	if err := buf.WriteByte(uint8(len(targetBytes))); err != nil {
+		return nil, err
+	}
+	buf.Write(targetBytes)
+
+	return buf.Bytes(), nil
+}
+
+// Deserialize binary to Message
+func deserializeMessage(data []byte) (Message, error) {
+	buf := bytes.NewReader(data)
+	var msg Message
+
+	if err := binary.Read(buf, binary.BigEndian, &msg.ID); err != nil {
+		return msg, err
+	}
+	if err := binary.Read(buf, binary.BigEndian, &msg.Type); err != nil {
+		return msg, err
+	}
+
+	var contentLen uint16
+	if err := binary.Read(buf, binary.BigEndian, &contentLen); err != nil {
+		return msg, err
+	}
+	content := make([]byte, contentLen)
+	if _, err := io.ReadFull(buf, content); err != nil {
+		return msg, err
+	}
+	msg.Content = content
+
+	var targetLen uint8
+	if err := binary.Read(buf, binary.BigEndian, &targetLen); err != nil {
+		return msg, err
+	}
+	target := make([]byte, targetLen)
+	if _, err := io.ReadFull(buf, target); err != nil {
+		return msg, err
+	}
+	msg.Target = string(target)
+
+	return msg, nil
+}
+
+func writeString(buf *bytes.Buffer, s string) error {
+	if err := binary.Write(buf, binary.BigEndian, uint16(len(s))); err != nil {
+		return err
+	}
+	_, err := buf.Write([]byte(s))
+	return err
+}
+
+func readString(buf *bytes.Reader) (string, error) {
+	var length uint16
+	if err := binary.Read(buf, binary.BigEndian, &length); err != nil {
+		return "", err
+	}
+	strBytes := make([]byte, length)
+	if _, err := io.ReadFull(buf, strBytes); err != nil {
+		return "", err
+	}
+	return string(strBytes), nil
+}
+
+type testPayload struct {
+	Message      string `json:"message"`
+	ResponseAddr string `json:"ResponseAddr"`
+}
+
+func handleTestMessage(msg Message) {
+	var payload testPayload
+	if err := json.Unmarshal([]byte(msg.Content), &payload); err != nil {
+		printWarn("Failed to parse test payload: %v", err)
+		return
+	}
+	printDebug("Received message: %s", payload.Message)
+	printInfo("Received TEST message: %s", payload.Message)
+
+	reply := Message{
+		ID:      msg.ID,
+		Type:    MsgTypeTest,
+		Content: []byte("hello test"),
+		Target:  payload.ResponseAddr,
+	}
+
+	if err := sendMessage(reply); err != nil {
+		printWarn("Failed to send test response: %v", err)
 	}
 }

text-extra.go

@@ -0,0 +1,272 @@
+//go:build experimental
+// +build experimental
+
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+var textSearchEngines []SearchEngine
+
+var allTextSearchEngines = []SearchEngine{
+	{Name: "Google", Func: wrapTextSearchFunc(PerformGoogleTextSearch)},
+	{Name: "LibreX", Func: wrapTextSearchFunc(PerformLibreXTextSearch)},
+	{Name: "Brave", Func: wrapTextSearchFunc(PerformBraveTextSearch)},
+	{Name: "DuckDuckGo", Func: wrapTextSearchFunc(PerformDuckDuckGoTextSearch)},
+	{Name: "Quant", Func: wrapTextSearchFunc(PerformQwantTextSearch)}, // Broken !
+	//{Name: "SearXNG", Func: wrapTextSearchFunc(PerformSearXTextSearch)}, // bruh
+}
+
+func initTextEngines() {
+	// textSearchEngines is your final slice (already declared globally)
+	textSearchEngines = nil // or make([]SearchEngine, 0)
+
+	for _, engineName := range config.MetaSearch.Text {
+		for _, candidate := range allTextSearchEngines {
+			if candidate.Name == engineName {
+				textSearchEngines = append(textSearchEngines, candidate)
+				break
+			}
+		}
+	}
+}
+
+func HandleTextSearch(w http.ResponseWriter, settings UserSettings, query string, page int) {
+	startTime := time.Now()
+
+	cacheKey := CacheKey{
+		Query: query,
+		Page:  page,
+		Safe:  settings.SafeSearch == "active",
+		Lang:  settings.SearchLanguage,
+		Type:  "text",
+	}
+	combinedResults := getTextResultsFromCacheOrFetch(cacheKey, query, settings.SafeSearch, settings.SearchLanguage, page)
+
+	hasPrevPage := page > 1
+
+	// Prefetch next and previous pages asynchronously
+	go prefetchPage(query, settings.SafeSearch, settings.SearchLanguage, page+1)
+	if hasPrevPage {
+		go prefetchPage(query, settings.SafeSearch, settings.SearchLanguage, page-1)
+	}
+
+	elapsedTime := time.Since(startTime)
+
+	// Simplified result structure without waiting for favicons
+	type DecoratedResult struct {
+		TextSearchResult
+		PrettyLink LinkParts
+		FaviconID  string // Just the ID, URL will be generated client-side
+	}
+
+	var decoratedResults []DecoratedResult
+	for _, r := range combinedResults {
+		if r.URL == "" {
+			continue
+		}
+
+		prettyLink := FormatLinkHTML(r.URL)
+		faviconID := faviconIDFromURL(prettyLink.RootURL)
+
+		decoratedResults = append(decoratedResults, DecoratedResult{
+			TextSearchResult: r,
+			PrettyLink:       prettyLink,
+			FaviconID:        faviconID,
+		})
+
+		// Start async favicon fetch if not already cached
+		go ensureFaviconIsCached(faviconID, prettyLink.RootURL)
+	}
+
+	data := map[string]interface{}{
+		"Results":          decoratedResults,
+		"Query":            query,
+		"Fetched":          FormatElapsedTime(elapsedTime),
+		"Page":             page,
+		"HasPrevPage":      hasPrevPage,
+		"HasNextPage":      len(combinedResults) >= 50,
+		"NoResults":        len(combinedResults) == 0,
+		"LanguageOptions":  languageOptions,
+		"CurrentLang":      settings.SearchLanguage,
+		"Theme":            settings.Theme,
+		"Safe":             settings.SafeSearch,
+		"IsThemeDark":      settings.IsThemeDark,
+		"Trans":            Translate,
+		"HardCacheEnabled": config.DriveCacheEnabled,
+	}
+
+	renderTemplate(w, "text.html", data)
+}
+
+func ensureFaviconIsCached(faviconID, rootURL string) {
+	// Check if already exists in cache
+	filename := fmt.Sprintf("%s_icon.webp", faviconID)
+	cachedPath := filepath.Join(config.DriveCache.Path, "images", filename)
+
+	if _, err := os.Stat(cachedPath); err == nil {
+		return // Already cached
+	}
+
+	// Not cached, initiate download
+	getFaviconProxyURL("", rootURL) // This will trigger async download
+}
+
+func getTextResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page int) []TextSearchResult {
+	cacheChan := make(chan []SearchResult)
+	var combinedResults []TextSearchResult
+
+	go func() {
+		results, exists := resultsCache.Get(cacheKey)
+		if exists {
+			printDebug("Cache hit")
+			cacheChan <- results
+		} else {
+			printDebug("Cache miss")
+			cacheChan <- nil
+		}
+	}()
+
+	select {
+	case results := <-cacheChan:
+		if results == nil {
+			// Always attempt to fetch results on a cache miss
+			combinedResults = fetchTextResults(query, safe, lang, page)
+			if len(combinedResults) > 0 {
+				resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
+			}
+		} else {
+			textResults, _, _, _, _ := convertToSpecificResults(results)
+			combinedResults = textResults
+		}
+	case <-time.After(2 * time.Second):
+		printInfo("Cache check timeout")
+		// Even on timeout, attempt to fetch results
+		combinedResults = fetchTextResults(query, safe, lang, page)
+		if len(combinedResults) > 0 {
+			resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
+		}
+	}
+
+	return combinedResults
+}
+
+func prefetchPage(query, safe, lang string, page int) {
+	cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "active", Lang: lang, Type: "text"}
+	if _, exists := resultsCache.Get(cacheKey); !exists {
+		printInfo("Page %d not cached, caching now...", page)
+		if config.MetaSearchEnabled {
+			pageResults := fetchTextResults(query, safe, lang, page)
+			if len(pageResults) > 0 {
+				resultsCache.Set(cacheKey, convertToSearchResults(pageResults))
+			}
+		} else {
+			printInfo("Crawler disabled; skipping prefetch for page %d", page)
+		}
+	} else {
+		printInfo("Page %d already cached", page)
+	}
+}
+
+// The logic in this function is rotating search engines instead of running them in order as noted in the wiki
+func fetchTextResults(query, safe, lang string, page int) []TextSearchResult {
+	var results []TextSearchResult
+
+	if !config.MetaSearchEnabled {
+		printDebug("Crawler is disabled; fetching from local index.")
+
+		// Calculate the starting position based on the page number
+		indexedResults, err := SearchIndex(query, page, 10)
+		if err != nil {
+			printErr("Error searching the index: %v", err)
+			return results // Return empty results on error
+		}
+
+		// Convert indexed results to TextSearchResult format
+		for _, doc := range indexedResults {
+			results = append(results, TextSearchResult{
+				URL:         doc.Link,
+				Header:      doc.Title,
+				Description: doc.Description,
+				Source:      doc.Tags,
+			})
+		}
+
+		return results
+	} else {
+		// Crawler is enabled, so use the search engines
+		engineCount := len(textSearchEngines)
+
+		// Determine which engine to use for the current page
+		engineIndex := (page - 1) % engineCount
+		engine := textSearchEngines[engineIndex]
+
+		// Calculate the page number for this engine
+		enginePage := (page-1)/engineCount + 1
+
+		printDebug("Fetching results for overall page %d using engine: %s (engine page %d)", page, engine.Name, enginePage)
+
+		// Fetch results from the selected engine
+		searchResults, _, err := engine.Func(query, safe, lang, enginePage)
+		if err != nil {
+			printWarn("Error performing search with %s: %v", engine.Name, err)
+		} else {
+			results = append(results, validateResults(searchResults)...)
+		}
+
+		// If no results are found with the selected engine, try the next in line
+		if len(results) == 0 {
+			for i := 1; i < engineCount; i++ {
+				nextEngine := textSearchEngines[(engineIndex+i)%engineCount]
+				enginePage = (page-1)/engineCount + 1
+				printInfo("No results found, trying next engine: %s (engine page %d)", nextEngine.Name, enginePage)
+
+				searchResults, _, err := nextEngine.Func(query, safe, lang, enginePage)
+				if err != nil {
+					printWarn("Error performing search with %s: %v", nextEngine.Name, err)
+					continue
+				}
+				results = append(results, validateResults(searchResults)...)
+				if len(results) > 0 {
+					break
+				}
+			}
+		}
+
+		printInfo("Fetched %d results for overall page %d", len(results), page)
+		return results
+	}
+}
+
+func validateResults(searchResults []SearchResult) []TextSearchResult {
+	var validResults []TextSearchResult
+
+	// Remove anything that is missing a URL or Header
+	for _, result := range searchResults {
+		textResult := result.(TextSearchResult)
+		if textResult.URL != "" || textResult.Header != "" {
+			validResults = append(validResults, textResult)
+		}
+	}
+
+	return validResults
+}
+
+func wrapTextSearchFunc(f func(string, string, string, int) ([]TextSearchResult, time.Duration, error)) func(string, string, string, int) ([]SearchResult, time.Duration, error) {
+	return func(query, safe, lang string, page int) ([]SearchResult, time.Duration, error) {
+		textResults, duration, err := f(query, safe, lang, page)
+		if err != nil {
+			return nil, duration, err
+		}
+		searchResults := make([]SearchResult, len(textResults))
+		for i, result := range textResults {
+			searchResults[i] = result
+		}
+		return searchResults, duration, nil
+	}
+}

text.go

@@ -1,3 +1,6 @@
+//go:build !experimental
+// +build !experimental
+
 package main
 
 import (
@@ -174,70 +177,47 @@ func prefetchPage(query, safe, lang string, page int) {
 func fetchTextResults(query, safe, lang string, page int) []TextSearchResult {
 	var results []TextSearchResult
 
-	if !config.MetaSearchEnabled {
+	// Crawler is enabled, so use the search engines
-		printDebug("Crawler is disabled; fetching from local index.")
+	engineCount := len(textSearchEngines)
 
-		// Calculate the starting position based on the page number
+	// Determine which engine to use for the current page
-		indexedResults, err := SearchIndex(query, page, 10)
+	engineIndex := (page - 1) % engineCount
-		if err != nil {
+	engine := textSearchEngines[engineIndex]
-			printErr("Error searching the index: %v", err)
-			return results // Return empty results on error
-		}
 
-		// Convert indexed results to TextSearchResult format
+	// Calculate the page number for this engine
-		for _, doc := range indexedResults {
+	enginePage := (page-1)/engineCount + 1
-			results = append(results, TextSearchResult{
-				URL:         doc.Link,
-				Header:      doc.Title,
-				Description: doc.Description,
-				Source:      doc.Tags,
-			})
-		}
 
-		return results
+	printDebug("Fetching results for overall page %d using engine: %s (engine page %d)", page, engine.Name, enginePage)
+
+	// Fetch results from the selected engine
+	searchResults, _, err := engine.Func(query, safe, lang, enginePage)
+	if err != nil {
+		printWarn("Error performing search with %s: %v", engine.Name, err)
 	} else {
-		// Crawler is enabled, so use the search engines
+		results = append(results, validateResults(searchResults)...)
-		engineCount := len(textSearchEngines)
+	}
 
-		// Determine which engine to use for the current page
+	// If no results are found with the selected engine, try the next in line
-		engineIndex := (page - 1) % engineCount
+	if len(results) == 0 {
-		engine := textSearchEngines[engineIndex]
+		for i := 1; i < engineCount; i++ {
+			nextEngine := textSearchEngines[(engineIndex+i)%engineCount]
+			enginePage = (page-1)/engineCount + 1
+			printInfo("No results found, trying next engine: %s (engine page %d)", nextEngine.Name, enginePage)
 
-		// Calculate the page number for this engine
+			searchResults, _, err := nextEngine.Func(query, safe, lang, enginePage)
-		enginePage := (page-1)/engineCount + 1
+			if err != nil {
-
+				printWarn("Error performing search with %s: %v", nextEngine.Name, err)
-		printDebug("Fetching results for overall page %d using engine: %s (engine page %d)", page, engine.Name, enginePage)
+				continue
-
+			}
-		// Fetch results from the selected engine
-		searchResults, _, err := engine.Func(query, safe, lang, enginePage)
-		if err != nil {
-			printWarn("Error performing search with %s: %v", engine.Name, err)
-		} else {
 			results = append(results, validateResults(searchResults)...)
-		}
+			if len(results) > 0 {
-
+				break
-		// If no results are found with the selected engine, try the next in line
-		if len(results) == 0 {
-			for i := 1; i < engineCount; i++ {
-				nextEngine := textSearchEngines[(engineIndex+i)%engineCount]
-				enginePage = (page-1)/engineCount + 1
-				printInfo("No results found, trying next engine: %s (engine page %d)", nextEngine.Name, enginePage)
-
-				searchResults, _, err := nextEngine.Func(query, safe, lang, enginePage)
-				if err != nil {
-					printWarn("Error performing search with %s: %v", nextEngine.Name, err)
-					continue
-				}
-				results = append(results, validateResults(searchResults)...)
-				if len(results) > 0 {
-					break
-				}
 			}
 		}
-
-		printInfo("Fetched %d results for overall page %d", len(results), page)
-		return results
 	}
+
+	printInfo("Fetched %d results for overall page %d", len(results), page)
+	return results
 }
 
 func validateResults(searchResults []SearchResult) []TextSearchResult {

update.go

@@ -1,52 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-	"os/exec"
-	"time"
-)
-
-// Function to check for updates and restart the server if an update is found
-func checkForUpdates() {
-	repoURL := "https://weforge.xyz/Spitfire/Search.git"
-	localDir := "." // Assume the repository is cloned in the current directory
-
-	for {
-		err := gitPull(repoURL, localDir)
-		if err != nil {
-			fmt.Println("Error checking for updates:", err)
-			time.Sleep(10 * time.Minute)
-			continue
-		}
-
-		fmt.Println("Update found. Syncing updates...")
-		nodeUpdateSync()
-
-		fmt.Println("Restarting server to apply updates...")
-		update()
-		time.Sleep(10 * time.Minute)
-	}
-}
-
-// Function to pull updates from the Git repository
-func gitPull(repoURL, localDir string) error {
-	cmd := exec.Command("git", "-C", localDir, "pull", repoURL)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
-}
-
-// Function to download updates and restart the server
-func update() {
-	cmd := exec.Command("sh", "run.sh")
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	err := cmd.Start()
-	if err != nil {
-		fmt.Println("Error starting the server:", err)
-		return
-	}
-
-	os.Exit(0)
-}

video.go

@@ -15,7 +15,6 @@ var (
 	pipedInstances    = []string{}
 	disabledInstances = make(map[string]bool)
 	mu                sync.Mutex
-	videoResultsChan  = make(chan []VideoResult) // Channel to receive video results from other nodes
 )
 
 func initPipedInstances() {
@@ -146,11 +145,6 @@ func handleVideoSearch(w http.ResponseWriter, settings UserSettings, query strin
 		results = fetchVideoResults(query, settings.SafeSearch, settings.SearchLanguage, page)
 	}
 
-	if len(results) == 0 {
-		printWarn("No results from primary search, trying other nodes")
-		results = tryOtherNodesForVideoSearch(query, settings.SafeSearch, settings.SearchLanguage, page, []string{hostID})
-	}
-
 	elapsed := time.Since(start)
 
 	// Prepare the data to pass to the template