Qwant is still blocking me but this might help you
Some checks failed
Run Integration Tests / test (push) Failing after 42s
Some checks failed
Run Integration Tests / test (push) Failing after 42s
This commit is contained in:
parent
a132ca7fd8
commit
89264b0f87
6 changed files with 214 additions and 115 deletions
43
proxy.go
43
proxy.go
|
@ -1,7 +1,10 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
|
@ -217,17 +220,55 @@ func DoCrawlerProxyRequest(req *http.Request) (*http.Response, error) {
|
|||
func tryRequestWithRetry(req *http.Request, client Doer, retries int) (*http.Response, error) {
|
||||
var resp *http.Response
|
||||
var err error
|
||||
|
||||
for i := 1; i <= retries; i++ {
|
||||
if resp != nil {
|
||||
if resp != nil && resp.Body != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
printDebug("Attempt %d of %d with proxy/client...", i, retries)
|
||||
resp, err = tryRequestOnce(req, client)
|
||||
|
||||
if err != nil {
|
||||
printDebug("Request error: %v", err)
|
||||
}
|
||||
|
||||
if resp == nil {
|
||||
printDebug("No response received (nil)")
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
|
||||
// Try to read and print the body
|
||||
var bodyReader io.ReadCloser = resp.Body
|
||||
if resp.Header.Get("Content-Encoding") == "gzip" {
|
||||
gr, gzErr := gzip.NewReader(resp.Body)
|
||||
if gzErr != nil {
|
||||
printDebug("Failed to init gzip reader: %v", gzErr)
|
||||
bodyReader = resp.Body
|
||||
} else {
|
||||
bodyReader = gr
|
||||
defer gr.Close()
|
||||
}
|
||||
}
|
||||
|
||||
bodyBytes, readErr := io.ReadAll(bodyReader)
|
||||
if readErr != nil {
|
||||
printDebug("Failed to read body: %v", readErr)
|
||||
} else {
|
||||
printDebug("Response status: %d\n---\n%s\n---", resp.StatusCode, string(bodyBytes))
|
||||
}
|
||||
|
||||
// Reset body for possible reuse
|
||||
resp.Body = io.NopCloser(bytes.NewReader(bodyBytes))
|
||||
|
||||
if isSuccessful(resp, err) {
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
|
||||
return resp, err
|
||||
}
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue