Qwant is still blocking me but this might help you
Some checks failed
Run Integration Tests / test (push) Failing after 42s

This commit is contained in:
partisan 2025-07-04 22:45:06 +02:00
parent a132ca7fd8
commit 89264b0f87
6 changed files with 214 additions and 115 deletions

View file

@ -1,7 +1,10 @@
package main
import (
"bytes"
"compress/gzip"
"fmt"
"io"
"net/http"
"strings"
"sync"
@ -217,17 +220,55 @@ func DoCrawlerProxyRequest(req *http.Request) (*http.Response, error) {
func tryRequestWithRetry(req *http.Request, client Doer, retries int) (*http.Response, error) {
var resp *http.Response
var err error
for i := 1; i <= retries; i++ {
if resp != nil {
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
printDebug("Attempt %d of %d with proxy/client...", i, retries)
resp, err = tryRequestOnce(req, client)
if err != nil {
printDebug("Request error: %v", err)
}
if resp == nil {
printDebug("No response received (nil)")
time.Sleep(200 * time.Millisecond)
continue
}
// Try to read and print the body
var bodyReader io.ReadCloser = resp.Body
if resp.Header.Get("Content-Encoding") == "gzip" {
gr, gzErr := gzip.NewReader(resp.Body)
if gzErr != nil {
printDebug("Failed to init gzip reader: %v", gzErr)
bodyReader = resp.Body
} else {
bodyReader = gr
defer gr.Close()
}
}
bodyBytes, readErr := io.ReadAll(bodyReader)
if readErr != nil {
printDebug("Failed to read body: %v", readErr)
} else {
printDebug("Response status: %d\n---\n%s\n---", resp.StatusCode, string(bodyBytes))
}
// Reset body for possible reuse
resp.Body = io.NopCloser(bytes.NewReader(bodyBytes))
if isSuccessful(resp, err) {
return resp, nil
}
time.Sleep(200 * time.Millisecond)
}
return resp, err
}