Changed self-crawling as experimental, cleanup unused features

init-extra.go

@@ -0,0 +1,142 @@
+//go:build experimental
+// +build experimental
+
+package main
+
+import (
+	"flag"
+	"os"
+	"path/filepath"
+)
+
+var config Config
+
+func main() {
+	// Command-line flags
+	portFlag := flag.Int("port", 0, "Port number to run the application (overrides config)")
+	domainFlag := flag.String("domain", "", "Domain address for the application (overrides config)")
+	skipConfigFlag := flag.Bool("skip-config-check", false, "Skip interactive prompts and load config.ini")
+	configFlag := flag.String("config", "", "Path to configuration file (overrides default)")
+
+	// Parse command-line flags
+	flag.Parse()
+
+	// Override global configFilePath if --config flag is provided
+	if *configFlag != "" {
+		configFilePath = *configFlag
+	}
+
+	if *skipConfigFlag {
+		// Skip interactive configuration
+		if _, err := os.Stat(configFilePath); err == nil {
+			// Load from config file if it exists
+			config = loadConfig()
+		} else {
+			// Use defaults if config file does not exist
+			config = defaultConfig
+			saveConfig(config) // Save the defaults to config.ini
+			printInfo("Configuration saved to %s", configFilePath)
+		}
+	} else {
+		// Initialize configuration interactively or from config file
+		err := initConfig()
+		if err != nil {
+			printErr("Error during initialization: %v", err)
+			return
+		}
+	}
+
+	// Override with command-line arguments if provided
+	if *portFlag != 0 {
+		config.Port = *portFlag
+	}
+	if *domainFlag != "" {
+		config.Domain = *domainFlag
+	}
+
+	loadNodeConfig()
+
+	if config.CrawlerProxyEnabled || config.MetaProxyEnabled {
+		InitProxies()
+	}
+
+	// Initiate Browser Agent updater
+	if config.MetaSearchEnabled || config.IndexerEnabled {
+		go periodicAgentUpdate()
+	}
+
+	// Load List of Meta Search Engines
+	if config.MetaSearchEnabled {
+		initTextEngines()
+		initImageEngines()
+		initFileEngines()
+		initPipedInstances()
+		initMusicEngines()
+	}
+
+	InitializeLanguage("en") // Initialize language before generating OpenSearch
+	generateOpenSearchXML(config)
+
+	// Start the node client only if NodesEnabled is true
+	if config.NodesEnabled {
+		go startUnixSocketServer(config.NodeID)
+		printInfo("Node client started.")
+	} else {
+		printInfo("Node client is disabled.")
+	}
+
+	// Check if the cache directory exists when caching is enabled
+	if config.DriveCacheEnabled {
+		cacheDir := config.DriveCache.Path
+		imagesDir := filepath.Join(cacheDir, "images")
+
+		// Check if the directory already exists
+		if _, err := os.Stat(imagesDir); os.IsNotExist(err) {
+			// Try to create the directory since it doesn't exist
+			if err := os.MkdirAll(imagesDir, os.ModePerm); err != nil {
+				printErr("Error: Failed to create cache or images directory '%s': %v", imagesDir, err)
+				os.Exit(1) // Exit with a non-zero status to indicate an error
+			}
+			// Print a warning if the directory had to be created
+			printWarn("Warning: Created missing directory '%s'.", imagesDir)
+		}
+	}
+
+	// Start periodic cleanup of expired cache files
+	if config.DriveCacheEnabled {
+		go cleanExpiredCachedImages()
+		printInfo("Drive cache started.")
+	} else {
+		printInfo("Drive cache is disabled.")
+	}
+
+	// Start periodic cleanup of expired cache files
+	if config.RamCacheEnabled {
+		resultsCache = NewResultsCache()
+		geocodeCache = NewGeocodeCache()
+		printInfo("RAM cache started.")
+	} else {
+		printInfo("RAM cache is disabled.")
+	}
+
+	// Init indexer
+	if config.IndexerEnabled {
+		if err := downloadAndSetupDomainsCSV(); err != nil {
+			printErr("Failed to set up domains.csv: %v", err)
+			return
+		}
+
+		err := InitIndex()
+		if err != nil {
+			printErr("Failed to initialize index: %v", err)
+		}
+
+		webCrawlerInit()
+
+		printInfo("Indexer is enabled.")
+	} else {
+		printInfo("Indexer is disabled.")
+	}
+
+	runServer()
+}