added website crawling and indexing crawled results

init.go

@@ -3,6 +3,8 @@ package main
 import (
 	"flag"
 	"os"
+	"path/filepath"
+	"time"
 )
 
 var config Config
@@ -95,5 +97,28 @@ func main() {
 		printInfo("RAM cache is disabled.")
 	}
 
+	// Init indexer
+	if config.IndexerEnabled {
+		if err := downloadAndSetupDomainsCSV(); err != nil {
+			printErr("Failed to set up domains.csv: %v", err)
+			return
+		}
+
+		webCrawlerInit()
+
+		err := InitIndex()
+		if err != nil {
+			printErr("Failed to initialize index:", err)
+		}
+
+		// Start periodic indexing (every 2 minutes)
+		dataFilePath := filepath.Join(config.DriveCache.Path, "data_to_index.txt")
+		startPeriodicIndexing(dataFilePath, 2*time.Minute)
+
+		printInfo("Indexer is enabled.")
+	} else {
+		printInfo("Indexer is disabled.")
+	}
+
 	runServer()
 }