Search/agent.go

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"sort"
	"sync"
	"time"
)

// BrowserVersion represents the version & global usage from the caniuse data
type BrowserVersion struct {
	Version string  `json:"version"`
	Global  float64 `json:"global"`
}

// BrowserData holds sets of versions for Firefox and Chromium
type BrowserData struct {
	Firefox  []BrowserVersion `json:"firefox"`
	Chromium []BrowserVersion `json:"chrome"`
}

var (
	cache = struct {
		sync.RWMutex
		data map[string]string
	}{
		data: make(map[string]string),
	}

	browserCache = struct {
		sync.RWMutex
		data    BrowserData
		expires time.Time
	}{
		expires: time.Now(),
	}
)

// fetchLatestBrowserVersions retrieves usage data from caniuse.com’s fulldata JSON.
func fetchLatestBrowserVersions() (BrowserData, error) {
	const urlCaniuse = "https://raw.githubusercontent.com/Fyrd/caniuse/master/fulldata-json/data-2.0.json"
	client := &http.Client{
		Timeout: 30 * time.Second,
	}

	req, err := http.NewRequest("GET", urlCaniuse, nil)
	if err != nil {
		return BrowserData{}, err
	}

	// Set a simple custom User-Agent and language
	req.Header.Set("User-Agent", "MyCustomAgent/1.0 (compatible; +https://example.com)")
	req.Header.Set("Accept-Language", "en-US,en;q=0.9")

	resp, err := client.Do(req)
	if err != nil {
		return BrowserData{}, err
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return BrowserData{}, err
	}

	var rawData map[string]any
	if err := json.Unmarshal(body, &rawData); err != nil {
		return BrowserData{}, err
	}

	stats, ok := rawData["agents"].(map[string]any)
	if !ok {
		return BrowserData{}, fmt.Errorf("unexpected JSON structure (no 'agents' field)")
	}

	var data BrowserData

	// Extract Firefox data
	if firefoxData, ok := stats["firefox"].(map[string]any); ok {
		if usageMap, ok := firefoxData["usage_global"].(map[string]any); ok {
			for version, usage := range usageMap {
				val, _ := usage.(float64)
				data.Firefox = append(data.Firefox, BrowserVersion{Version: version, Global: val})
			}
		}
	}

	// Extract Chrome data
	if chromeData, ok := stats["chrome"].(map[string]any); ok {
		if usageMap, ok := chromeData["usage_global"].(map[string]any); ok {
			for version, usage := range usageMap {
				val, _ := usage.(float64)
				data.Chromium = append(data.Chromium, BrowserVersion{Version: version, Global: val})
			}
		}
	}

	return data, nil
}

// getLatestBrowserVersions checks the cache and fetches new data if expired
func getLatestBrowserVersions() (BrowserData, error) {
	browserCache.RLock()
	if time.Now().Before(browserCache.expires) {
		data := browserCache.data
		browserCache.RUnlock()
		return data, nil
	}
	browserCache.RUnlock()

	data, err := fetchLatestBrowserVersions()
	if err != nil {
		return BrowserData{}, err
	}

	browserCache.Lock()
	browserCache.data = data
	browserCache.expires = time.Now().Add(24 * time.Hour) // Refresh daily
	browserCache.Unlock()

	return data, nil
}

// randomUserAgent picks a random browser (Firefox/Chromium), selects a version based on usage,
// picks an OS string, and composes a User-Agent header.
func randomUserAgent() (string, error) {
	browsers, err := getLatestBrowserVersions()
	if err != nil {
		return "", err
	}

	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	// Overall usage: 80% chance for Chromium, 20% for Firefox
	usageStats := map[string]float64{
		"Firefox":  20.0,
		"Chromium": 80.0,
	}

	// Weighted random selection of the browser type
	browserType := ""
	randVal := r.Float64() * 100
	cumulative := 0.0
	for bType, usage := range usageStats {
		cumulative += usage
		if randVal < cumulative {
			browserType = bType
			break
		}
	}

	var versions []BrowserVersion
	switch browserType {
	case "Firefox":
		versions = browsers.Firefox
	case "Chromium":
		versions = browsers.Chromium
	}

	if len(versions) == 0 {
		return "", fmt.Errorf("no versions found for browser: %s", browserType)
	}

	// Sort by global usage descending
	sort.Slice(versions, func(i, j int) bool {
		return versions[i].Global > versions[j].Global
	})

	// Probability distribution for top few versions
	probabilities := []float64{0.5, 0.25, 0.125, 0.0625, 0.03125, 0.015625, 0.0078125, 0.00390625}

	version := ""
	randVal = r.Float64()
	cumulative = 0.0
	for i, p := range probabilities {
		cumulative += p
		if randVal < cumulative && i < len(versions) {
			version = versions[i].Version
			break
		}
	}

	// Fallback to the least used version if none matched
	if version == "" {
		version = versions[len(versions)-1].Version
	}

	userAgent := generateUserAgent(browserType, version, r)
	return userAgent, nil
}

// generateUserAgent composes the final UA string given the browser, version, and OS.
func generateUserAgent(browser, version string, r *rand.Rand) string {
	oses := []struct {
		os          string
		probability float64
	}{
		{"Windows NT 10.0; Win64; x64", 44.0},
		{"X11; Linux x86_64", 2.0},
		{"X11; Ubuntu; Linux x86_64", 2.0},
		{"Macintosh; Intel Mac OS X 10_15_7", 10.0},
	}

	// Weighted random selection for OS
	randVal := r.Float64() * 100
	cumulative := 0.0
	selectedOS := oses[0].os // Default in case distribution is off
	for _, entry := range oses {
		cumulative += entry.probability
		if randVal < cumulative {
			selectedOS = entry.os
			break
		}
	}

	switch browser {
	case "Firefox":
		// Example: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:117.0) Gecko/20100101 Firefox/117.0
		return fmt.Sprintf("Mozilla/5.0 (%s; rv:%s) Gecko/20100101 Firefox/%s", selectedOS, version, version)
	case "Chromium":
		// Example: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Safari/537.36
		return fmt.Sprintf("Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", selectedOS, version)
	default:
		return ""
	}
}

// updateCachedUserAgents randomly updates half of the cached UAs to new versions
func updateCachedUserAgents(newVersions BrowserData) {
	cache.Lock()
	defer cache.Unlock()

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	for key, userAgent := range cache.data {
		if r.Float64() < 0.5 {
			updatedUserAgent := updateUserAgentVersion(userAgent, newVersions, r)
			cache.data[key] = updatedUserAgent
		}
	}
}

// updateUserAgentVersion tries to parse the old UA, detect its browser, and update the version
func updateUserAgentVersion(userAgent string, newVersions BrowserData, r *rand.Rand) string {
	var browserType, version string

	// Attempt to detect old UA patterns (Chromium or Firefox)
	if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil {
		browserType = "Chromium"
	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil {
		browserType = "Chromium"
	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil {
		browserType = "Chromium"
	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil {
		browserType = "Chromium"
	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil {
		browserType = "Firefox"
	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Linux x86_64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil {
		browserType = "Firefox"
	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil {
		browserType = "Firefox"
	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil {
		browserType = "Firefox"
	}

	// Grab the newest version from the fetched data
	var latestVersion string
	if browserType == "Firefox" && len(newVersions.Firefox) > 0 {
		// Sort by usage descending
		sort.Slice(newVersions.Firefox, func(i, j int) bool {
			return newVersions.Firefox[i].Global > newVersions.Firefox[j].Global
		})
		latestVersion = newVersions.Firefox[0].Version
	} else if browserType == "Chromium" && len(newVersions.Chromium) > 0 {
		// Sort by usage descending
		sort.Slice(newVersions.Chromium, func(i, j int) bool {
			return newVersions.Chromium[i].Global > newVersions.Chromium[j].Global
		})
		latestVersion = newVersions.Chromium[0].Version
	}

	// If we failed to detect the browser or have no data, just return the old UA
	if browserType == "" || latestVersion == "" {
		return userAgent
	}

	// Create a new random OS-based UA string with the latest version
	return generateUserAgent(browserType, latestVersion, r)
}

// periodicAgentUpdate periodically refreshes browser data and user agents
func periodicAgentUpdate() {
	for {
		// Sleep a random interval between 1 and 2 days
		r := rand.New(rand.NewSource(time.Now().UnixNano()))
		time.Sleep(time.Duration(24+r.Intn(24)) * time.Hour)

		// Fetch the latest browser versions
		newVersions, err := fetchLatestBrowserVersions()
		if err != nil {
			printWarn("Error fetching latest browser versions: %v", err)
			continue
		}

		// Update the browser version cache
		browserCache.Lock()
		browserCache.data = newVersions
		browserCache.expires = time.Now().Add(24 * time.Hour)
		browserCache.Unlock()

		// Update the cached user agents
		updateCachedUserAgents(newVersions)
	}
}

// GetUserAgent returns a cached UA for the given key or creates one if none exists.
func GetUserAgent(cacheKey string) (string, error) {
	cache.RLock()
	userAgent, found := cache.data[cacheKey]
	cache.RUnlock()

	if found {
		return userAgent, nil
	}

	userAgent, err := randomUserAgent()
	if err != nil {
		return "", err
	}

	cache.Lock()
	cache.data[cacheKey] = userAgent
	cache.Unlock()

	printDebug("Generated (cached or new) user agent: %s", userAgent)
	return userAgent, nil
}

// GetNewUserAgent always returns a newly generated UA, overwriting the cache.
func GetNewUserAgent(cacheKey string) (string, error) {
	userAgent, err := randomUserAgent()
	if err != nil {
		return "", err
	}

	cache.Lock()
	cache.data[cacheKey] = userAgent
	cache.Unlock()

	printDebug("Generated new user agent: %s", userAgent)
	return userAgent, nil
}

// func main() {
// 	go periodicAgentUpdate() // not needed here

// 	cacheKey := "image-search"
// 	userAgent, err := GetUserAgent(cacheKey)
// 	if err != nil {
// 		fmt.Println("Error:", err)
// 		return
// 	}

// 	fmt.Println("Generated User Agent:", userAgent)

// 	// Request a new user agent for the same key
// 	newUserAgent, err := GetNewUserAgent(cacheKey)
// 	if err != nil {
// 		fmt.Println("Error:", err)
// 		return
// 	}

// 	fmt.Println("New User Agent:", newUserAgent)

// 	AcacheKey := "image-search"
// 	AuserAgent, err := GetUserAgent(AcacheKey)
// 	if err != nil {
// 		fmt.Println("Error:", err)
// 		return
// 	}

// 	fmt.Println("Generated User Agent:", AuserAgent)

// 	DcacheKey := "image-search"
// 	DuserAgent, err := GetUserAgent(DcacheKey)
// 	if err != nil {
// 		fmt.Println("Error:", err)
// 		return
// 	}

// 	fmt.Println("Generated User Agent:", DuserAgent)

// }