package main import ( "archive/zip" "fmt" "io" "net/http" "os" "path/filepath" ) func downloadAndSetupDomainsCSV() error { targetFilePath := filepath.Join(config.DriveCache.Path, "domains.csv") // Check if domains.csv already exists if _, err := os.Stat(targetFilePath); err == nil { printDebug("domains.csv already exists at %s", targetFilePath) return nil } downloadURL := "https://www.domcop.com/files/top/top10milliondomains.csv.zip" zipFilePath := filepath.Join(config.DriveCache.Path, "top10milliondomains.csv.zip") // Download the file printDebug("Downloading file from %s", downloadURL) resp, err := http.Get(downloadURL) if err != nil { return fmt.Errorf("failed to download file: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("failed to download file: received status code %d", resp.StatusCode) } // Create the zip file locally zipFile, err := os.Create(zipFilePath) if err != nil { return fmt.Errorf("failed to create local zip file: %v", err) } defer zipFile.Close() _, err = io.Copy(zipFile, resp.Body) if err != nil { return fmt.Errorf("failed to write downloaded zip file: %v", err) } // Unzip the file printDebug("Unzipping file %s", zipFilePath) if err := unzipFile(zipFilePath, config.DriveCache.Path); err != nil { return fmt.Errorf("failed to unzip file: %v", err) } // Find the .csv file and rename/move it to domains.csv csvFound := false dirEntries, err := os.ReadDir(config.DriveCache.Path) if err != nil { return fmt.Errorf("failed to read directory: %v", err) } for _, entry := range dirEntries { if !entry.IsDir() && filepath.Ext(entry.Name()) == ".csv" { csvPath := filepath.Join(config.DriveCache.Path, entry.Name()) if err := os.Rename(csvPath, targetFilePath); err != nil { return fmt.Errorf("failed to move %s to %s: %v", csvPath, targetFilePath, err) } csvFound = true break } } if !csvFound { return fmt.Errorf("no .csv file found in the downloaded archive") } // Clean up zip file if err := os.Remove(zipFilePath); err != nil { printWarn("failed to remove zip file %s: %v", zipFilePath, err) } printDebug("domains.csv successfully downloaded and placed at %s", targetFilePath) return nil } func unzipFile(zipFile, destDir string) error { reader, err := zip.OpenReader(zipFile) if err != nil { return err } defer reader.Close() for _, file := range reader.File { filePath := filepath.Join(destDir, file.Name) if file.FileInfo().IsDir() { os.MkdirAll(filePath, os.ModePerm) continue } srcFile, err := file.Open() if err != nil { return err } defer srcFile.Close() destFile, err := os.Create(filePath) if err != nil { return err } defer destFile.Close() if _, err := io.Copy(destFile, srcFile); err != nil { return err } } return nil }