package main import ( "encoding/json" "fmt" "io" "math/rand" "net/http" "sort" "sync" "time" ) // BrowserVersion represents the version & global usage from the caniuse data type BrowserVersion struct { Version string `json:"version"` Global float64 `json:"global"` } // BrowserData holds sets of versions for Firefox and Chromium type BrowserData struct { Firefox []BrowserVersion `json:"firefox"` Chromium []BrowserVersion `json:"chrome"` } var ( cache = struct { sync.RWMutex data map[string]string }{ data: make(map[string]string), } browserCache = struct { sync.RWMutex data BrowserData expires time.Time }{ expires: time.Now(), } ) // fetchLatestBrowserVersions retrieves usage data from caniuse.com’s fulldata JSON. func fetchLatestBrowserVersions() (BrowserData, error) { const urlCaniuse = "https://raw.githubusercontent.com/Fyrd/caniuse/master/fulldata-json/data-2.0.json" client := &http.Client{ Timeout: 30 * time.Second, } req, err := http.NewRequest("GET", urlCaniuse, nil) if err != nil { return BrowserData{}, err } // Set a simple custom User-Agent and language req.Header.Set("User-Agent", "MyCustomAgent/1.0 (compatible; +https://example.com)") req.Header.Set("Accept-Language", "en-US,en;q=0.9") resp, err := client.Do(req) if err != nil { return BrowserData{}, err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return BrowserData{}, err } var rawData map[string]any if err := json.Unmarshal(body, &rawData); err != nil { return BrowserData{}, err } stats, ok := rawData["agents"].(map[string]any) if !ok { return BrowserData{}, fmt.Errorf("unexpected JSON structure (no 'agents' field)") } var data BrowserData // Extract Firefox data if firefoxData, ok := stats["firefox"].(map[string]any); ok { if usageMap, ok := firefoxData["usage_global"].(map[string]any); ok { for version, usage := range usageMap { val, _ := usage.(float64) data.Firefox = append(data.Firefox, BrowserVersion{Version: version, Global: val}) } } } // Extract Chrome data if chromeData, ok := stats["chrome"].(map[string]any); ok { if usageMap, ok := chromeData["usage_global"].(map[string]any); ok { for version, usage := range usageMap { val, _ := usage.(float64) data.Chromium = append(data.Chromium, BrowserVersion{Version: version, Global: val}) } } } return data, nil } // getLatestBrowserVersions checks the cache and fetches new data if expired func getLatestBrowserVersions() (BrowserData, error) { browserCache.RLock() if time.Now().Before(browserCache.expires) { data := browserCache.data browserCache.RUnlock() return data, nil } browserCache.RUnlock() data, err := fetchLatestBrowserVersions() if err != nil { return BrowserData{}, err } browserCache.Lock() browserCache.data = data browserCache.expires = time.Now().Add(24 * time.Hour) // Refresh daily browserCache.Unlock() return data, nil } // randomUserAgent picks a random browser (Firefox/Chromium), selects a version based on usage, // picks an OS string, and composes a User-Agent header. func randomUserAgent() (string, error) { browsers, err := getLatestBrowserVersions() if err != nil { return "", err } r := rand.New(rand.NewSource(time.Now().UnixNano())) // Overall usage: 80% chance for Chromium, 20% for Firefox usageStats := map[string]float64{ "Firefox": 20.0, "Chromium": 80.0, } // Weighted random selection of the browser type browserType := "" randVal := r.Float64() * 100 cumulative := 0.0 for bType, usage := range usageStats { cumulative += usage if randVal < cumulative { browserType = bType break } } var versions []BrowserVersion switch browserType { case "Firefox": versions = browsers.Firefox case "Chromium": versions = browsers.Chromium } if len(versions) == 0 { return "", fmt.Errorf("no versions found for browser: %s", browserType) } // Sort by global usage descending sort.Slice(versions, func(i, j int) bool { return versions[i].Global > versions[j].Global }) // Probability distribution for top few versions probabilities := []float64{0.5, 0.25, 0.125, 0.0625, 0.03125, 0.015625, 0.0078125, 0.00390625} version := "" randVal = r.Float64() cumulative = 0.0 for i, p := range probabilities { cumulative += p if randVal < cumulative && i < len(versions) { version = versions[i].Version break } } // Fallback to the least used version if none matched if version == "" { version = versions[len(versions)-1].Version } userAgent := generateUserAgent(browserType, version, r) return userAgent, nil } // generateUserAgent composes the final UA string given the browser, version, and OS. func generateUserAgent(browser, version string, r *rand.Rand) string { oses := []struct { os string probability float64 }{ {"Windows NT 10.0; Win64; x64", 44.0}, {"X11; Linux x86_64", 2.0}, {"X11; Ubuntu; Linux x86_64", 2.0}, {"Macintosh; Intel Mac OS X 10_15_7", 10.0}, } // Weighted random selection for OS randVal := r.Float64() * 100 cumulative := 0.0 selectedOS := oses[0].os // Default in case distribution is off for _, entry := range oses { cumulative += entry.probability if randVal < cumulative { selectedOS = entry.os break } } switch browser { case "Firefox": // Example: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:117.0) Gecko/20100101 Firefox/117.0 return fmt.Sprintf("Mozilla/5.0 (%s; rv:%s) Gecko/20100101 Firefox/%s", selectedOS, version, version) case "Chromium": // Example: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Safari/537.36 return fmt.Sprintf("Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", selectedOS, version) default: return "" } } // updateCachedUserAgents randomly updates half of the cached UAs to new versions func updateCachedUserAgents(newVersions BrowserData) { cache.Lock() defer cache.Unlock() r := rand.New(rand.NewSource(time.Now().UnixNano())) for key, userAgent := range cache.data { if r.Float64() < 0.5 { updatedUserAgent := updateUserAgentVersion(userAgent, newVersions, r) cache.data[key] = updatedUserAgent } } } // updateUserAgentVersion tries to parse the old UA, detect its browser, and update the version func updateUserAgentVersion(userAgent string, newVersions BrowserData, r *rand.Rand) string { var browserType, version string // Attempt to detect old UA patterns (Chromium or Firefox) if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil { browserType = "Chromium" } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil { browserType = "Chromium" } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil { browserType = "Chromium" } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil { browserType = "Chromium" } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil { browserType = "Firefox" } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Linux x86_64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil { browserType = "Firefox" } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil { browserType = "Firefox" } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil { browserType = "Firefox" } // Grab the newest version from the fetched data var latestVersion string if browserType == "Firefox" && len(newVersions.Firefox) > 0 { // Sort by usage descending sort.Slice(newVersions.Firefox, func(i, j int) bool { return newVersions.Firefox[i].Global > newVersions.Firefox[j].Global }) latestVersion = newVersions.Firefox[0].Version } else if browserType == "Chromium" && len(newVersions.Chromium) > 0 { // Sort by usage descending sort.Slice(newVersions.Chromium, func(i, j int) bool { return newVersions.Chromium[i].Global > newVersions.Chromium[j].Global }) latestVersion = newVersions.Chromium[0].Version } // If we failed to detect the browser or have no data, just return the old UA if browserType == "" || latestVersion == "" { return userAgent } // Create a new random OS-based UA string with the latest version return generateUserAgent(browserType, latestVersion, r) } // periodicAgentUpdate periodically refreshes browser data and user agents func periodicAgentUpdate() { for { // Sleep a random interval between 1 and 2 days r := rand.New(rand.NewSource(time.Now().UnixNano())) time.Sleep(time.Duration(24+r.Intn(24)) * time.Hour) // Fetch the latest browser versions newVersions, err := fetchLatestBrowserVersions() if err != nil { printWarn("Error fetching latest browser versions: %v", err) continue } // Update the browser version cache browserCache.Lock() browserCache.data = newVersions browserCache.expires = time.Now().Add(24 * time.Hour) browserCache.Unlock() // Update the cached user agents updateCachedUserAgents(newVersions) } } // GetUserAgent returns a cached UA for the given key or creates one if none exists. func GetUserAgent(cacheKey string) (string, error) { cache.RLock() userAgent, found := cache.data[cacheKey] cache.RUnlock() if found { return userAgent, nil } userAgent, err := randomUserAgent() if err != nil { return "", err } cache.Lock() cache.data[cacheKey] = userAgent cache.Unlock() printDebug("Generated (cached or new) user agent: %s", userAgent) return userAgent, nil } // GetNewUserAgent always returns a newly generated UA, overwriting the cache. func GetNewUserAgent(cacheKey string) (string, error) { userAgent, err := randomUserAgent() if err != nil { return "", err } cache.Lock() cache.data[cacheKey] = userAgent cache.Unlock() printDebug("Generated new user agent: %s", userAgent) return userAgent, nil } // func main() { // go periodicAgentUpdate() // not needed here // cacheKey := "image-search" // userAgent, err := GetUserAgent(cacheKey) // if err != nil { // fmt.Println("Error:", err) // return // } // fmt.Println("Generated User Agent:", userAgent) // // Request a new user agent for the same key // newUserAgent, err := GetNewUserAgent(cacheKey) // if err != nil { // fmt.Println("Error:", err) // return // } // fmt.Println("New User Agent:", newUserAgent) // AcacheKey := "image-search" // AuserAgent, err := GetUserAgent(AcacheKey) // if err != nil { // fmt.Println("Error:", err) // return // } // fmt.Println("Generated User Agent:", AuserAgent) // DcacheKey := "image-search" // DuserAgent, err := GetUserAgent(DcacheKey) // if err != nil { // fmt.Println("Error:", err) // return // } // fmt.Println("Generated User Agent:", DuserAgent) // }