From 6897aad91e44c8ea1ee75c307ab20e60f3f14cc7 Mon Sep 17 00:00:00 2001 From: Shinku <17696928+Shinku-Chen@users.noreply.github.com> Date: Fri, 15 May 2026 17:31:18 +0800 Subject: [PATCH 1/9] LimitRule_Init --- http_backend.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/http_backend.go b/http_backend.go index 90585e5e5..bffae24e9 100644 --- a/http_backend.go +++ b/http_backend.go @@ -67,7 +67,9 @@ type LimitRule struct { // Init initializes the private members of LimitRule func (r *LimitRule) Init() error { - r.waitChan = make(chan bool, max(r.Parallelism, 1)) + if r.waitChan == nil { + r.waitChan = make(chan bool, max(r.Parallelism, 1)) + } hasPattern := false if r.DomainRegexp != "" { c, err := regexp.Compile(r.DomainRegexp) From ff149e8f2a774657828dda8533b6084b2e369ea1 Mon Sep 17 00:00:00 2001 From: Shinku <17696928+Shinku-Chen@users.noreply.github.com> Date: Wed, 20 May 2026 16:56:39 +0800 Subject: [PATCH 2/9] ProxyURL --- colly.go | 21 +++++++++++----- proxy/proxy_test.go | 60 +++++++++++++++++++++++++++++++++++++++++++++ response.go | 3 +++ 3 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 proxy/proxy_test.go diff --git a/colly.go b/colly.go index b4b96e61e..48fad7802 100644 --- a/colly.go +++ b/colly.go @@ -729,7 +729,15 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct request.URL = req.URL request.Headers = &req.Header } - c.handleOnResponseHeaders(&Response{Ctx: ctx, Request: request, StatusCode: statusCode, Headers: &headers}) + // Read ProxyURLKey here, not after Cache returns. http.Client with a + // non-zero Timeout calls forkReq() before Transport.RoundTrip, so the + // ProxyFunc's *pr = *pr.WithContext(ctx) mutation lands on the fork. + // The fork is what gets surfaced as res.Request → finalRequest → + // this callback's req argument, so the context value is visible here. + if proxyURL, ok := req.Context().Value(ProxyURLKey).(string); ok { + request.ProxyURL = proxyURL + } + c.handleOnResponseHeaders(&Response{Ctx: ctx, Request: request, ProxyURL: request.ProxyURL, StatusCode: statusCode, Headers: &headers}) return !request.abort } checkRequestHeadersFunc := func(req *http.Request) bool { @@ -737,9 +745,6 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct return !request.abort } response, err := c.backend.Cache(req, c.MaxBodySize, checkRequestHeadersFunc, checkResponseHeadersFunc, c.CacheDir, c.CacheExpiration) - if proxyURL, ok := req.Context().Value(ProxyURLKey).(string); ok { - request.ProxyURL = proxyURL - } if err := c.handleOnError(response, err, request, ctx); err != nil { return err } @@ -747,6 +752,7 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct response.Ctx = ctx response.Request = request response.Trace = hTrace + response.ProxyURL = request.ProxyURL err = response.fixCharset(c.DetectCharset, request.ResponseCharacterEncoding) if err != nil { @@ -1324,9 +1330,12 @@ func (c *Collector) handleOnError(response *Response, err error, request *Reques } if response == nil { response = &Response{ - Request: request, - Ctx: ctx, + Request: request, + Ctx: ctx, + ProxyURL: request.ProxyURL, } + } else { + response.ProxyURL = request.ProxyURL } if c.debugger != nil { c.debugger.Event(createEvent("error", request.ID, c.ID, map[string]string{ diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go new file mode 100644 index 000000000..021cc48da --- /dev/null +++ b/proxy/proxy_test.go @@ -0,0 +1,60 @@ +// Copyright 2018 Adam Tauber +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proxy + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gocolly/colly/v2" +) + +// TestRoundRobinProxySwitcher_PropagatesProxyURL is the minimal smoke test: +// after a Visit through the switcher, the response must carry a non-empty +// ProxyURL on both Request and Response. +func TestRoundRobinProxySwitcher_PropagatesProxyURL(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "ok") + })) + defer ts.Close() + + rp, err := RoundRobinProxySwitcher(ts.URL) + if err != nil { + t.Fatalf("RoundRobinProxySwitcher: %v", err) + } + + c := colly.NewCollector(colly.IgnoreRobotsTxt()) + c.SetProxyFunc(rp) + + var called bool + c.OnResponse(func(r *colly.Response) { + called = true + if r.Request.ProxyURL == "" { + t.Errorf("Request.ProxyURL is empty — ProxyURLKey not propagated") + } + if r.ProxyURL == "" { + t.Errorf("Response.ProxyURL is empty") + } + }) + + if err := c.Visit("http://example.com/"); err != nil { + t.Fatalf("Visit: %v", err) + } + if !called { + t.Fatal("OnResponse never fired") + } +} diff --git a/response.go b/response.go index 30cdeae66..eb2f121c8 100644 --- a/response.go +++ b/response.go @@ -42,6 +42,9 @@ type Response struct { // Trace contains the HTTPTrace for the request. Will only be set by the // collector if Collector.TraceHTTP is set to true. Trace *HTTPTrace + // ProxyURL is the proxy address that handled the request, mirrored from + // Request.ProxyURL for convenience. + ProxyURL string } // Save writes response body to disk From f7dc02ef849d585739f8d520fe0e68d5c2e13869 Mon Sep 17 00:00:00 2001 From: Shinku <17696928+Shinku-Chen@users.noreply.github.com> Date: Wed, 20 May 2026 17:05:58 +0800 Subject: [PATCH 3/9] ProxyURL --- colly.go | 8 +++----- http_backend.go | 4 +--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/colly.go b/colly.go index 48fad7802..a7a7de4c9 100644 --- a/colly.go +++ b/colly.go @@ -1330,13 +1330,11 @@ func (c *Collector) handleOnError(response *Response, err error, request *Reques } if response == nil { response = &Response{ - Request: request, - Ctx: ctx, - ProxyURL: request.ProxyURL, + Request: request, + Ctx: ctx, } - } else { - response.ProxyURL = request.ProxyURL } + response.ProxyURL = request.ProxyURL if c.debugger != nil { c.debugger.Event(createEvent("error", request.ID, c.ID, map[string]string{ "url": request.URL.String(), diff --git a/http_backend.go b/http_backend.go index bffae24e9..90585e5e5 100644 --- a/http_backend.go +++ b/http_backend.go @@ -67,9 +67,7 @@ type LimitRule struct { // Init initializes the private members of LimitRule func (r *LimitRule) Init() error { - if r.waitChan == nil { - r.waitChan = make(chan bool, max(r.Parallelism, 1)) - } + r.waitChan = make(chan bool, max(r.Parallelism, 1)) hasPattern := false if r.DomainRegexp != "" { c, err := regexp.Compile(r.DomainRegexp) From 0e8b39c2357120982f855920a1f476a8188219d0 Mon Sep 17 00:00:00 2001 From: Shinku <17696928+Shinku-Chen@users.noreply.github.com> Date: Wed, 20 May 2026 18:24:49 +0800 Subject: [PATCH 4/9] ProxyURL --- colly.go | 33 +++++++++++++++++++++++---------- proxy/proxy.go | 11 +++++++---- proxy/proxy_test.go | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 14 deletions(-) diff --git a/colly.go b/colly.go index a7a7de4c9..d1a57e6b3 100644 --- a/colly.go +++ b/colly.go @@ -208,7 +208,15 @@ var collectorCounter uint32 // other packages. type key int -// ProxyURLKey is the context key for the request proxy address. +// ProxyURLKey is the context key for the per-request proxy URL holder +// (a *string shared across the request and any clone Go's http.Client may +// fork during Do, so the value survives forkReq on the error path). +// +// ProxyFunc implementations should write the chosen proxy via this pattern: +// +// if h, _ := req.Context().Value(colly.ProxyURLKey).(*string); h != nil { +// *h = chosen.String() +// } const ( ProxyURLKey key = iota CheckRevisitKey @@ -673,7 +681,9 @@ func (c *Collector) scrape(u, method string, depth int, requestData io.Reader, c } // note: once 1.13 is minimum supported Go version, // replace this with http.NewRequestWithContext - req = req.WithContext(context.WithValue(c.Context, CheckRevisitKey, checkRevisit)) + req = req.WithContext(context.WithValue( + context.WithValue(c.Context, CheckRevisitKey, checkRevisit), + ProxyURLKey, new(string))) if err := c.requestCheck(parsedURL, method, req.GetBody, depth, checkRevisit); err != nil { return err @@ -724,19 +734,21 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct req = hTrace.WithTrace(req) } origURL := req.URL + // Read the per-request *string holder set by the ProxyFunc. Called twice: + // inside checkResponseHeadersFunc so OnResponseHeaders sees it, and again + // after Cache returns so the error path (where the headers callback + // never fires) still surfaces the proxy URL. + syncProxyURL := func(r *http.Request) { + if h, _ := r.Context().Value(ProxyURLKey).(*string); h != nil && *h != "" { + request.ProxyURL = *h + } + } checkResponseHeadersFunc := func(req *http.Request, statusCode int, headers http.Header) bool { if req.URL != origURL { request.URL = req.URL request.Headers = &req.Header } - // Read ProxyURLKey here, not after Cache returns. http.Client with a - // non-zero Timeout calls forkReq() before Transport.RoundTrip, so the - // ProxyFunc's *pr = *pr.WithContext(ctx) mutation lands on the fork. - // The fork is what gets surfaced as res.Request → finalRequest → - // this callback's req argument, so the context value is visible here. - if proxyURL, ok := req.Context().Value(ProxyURLKey).(string); ok { - request.ProxyURL = proxyURL - } + syncProxyURL(req) c.handleOnResponseHeaders(&Response{Ctx: ctx, Request: request, ProxyURL: request.ProxyURL, StatusCode: statusCode, Headers: &headers}) return !request.abort } @@ -745,6 +757,7 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct return !request.abort } response, err := c.backend.Cache(req, c.MaxBodySize, checkRequestHeadersFunc, checkResponseHeadersFunc, c.CacheDir, c.CacheExpiration) + syncProxyURL(req) if err := c.handleOnError(response, err, request, ctx); err != nil { return err } diff --git a/proxy/proxy.go b/proxy/proxy.go index a4bd84852..e32ba1838 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -15,7 +15,6 @@ package proxy import ( - "context" "net/http" "net/url" "sync/atomic" @@ -31,9 +30,13 @@ type roundRobinSwitcher struct { func (r *roundRobinSwitcher) GetProxy(pr *http.Request) (*url.URL, error) { index := atomic.AddUint32(&r.index, 1) - 1 u := r.proxyURLs[index%uint32(len(r.proxyURLs))] - - ctx := context.WithValue(pr.Context(), colly.ProxyURLKey, u.String()) - *pr = *pr.WithContext(ctx) + // Write through the per-request *string holder colly placed in the + // context, so the chosen proxy is visible on Request.ProxyURL even when + // the request fails before response headers (forkReq isolates ctx field + // rewrites; pointer writes survive). + if h, _ := pr.Context().Value(colly.ProxyURLKey).(*string); h != nil { + *h = u.String() + } return u, nil } diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 021cc48da..23e87b145 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -58,3 +58,37 @@ func TestRoundRobinProxySwitcher_PropagatesProxyURL(t *testing.T) { t.Fatal("OnResponse never fired") } } + +// TestRoundRobinProxySwitcher_ProxyURLOnError ensures the chosen proxy URL +// is still recorded when the request fails before any response headers +// arrive (e.g. dial refused) — so OnError can report which proxy was tried. +func TestRoundRobinProxySwitcher_ProxyURLOnError(t *testing.T) { + ln := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + dead := ln.URL + ln.Close() // guarantees dial refused on dead + + rp, err := RoundRobinProxySwitcher(dead) + if err != nil { + t.Fatalf("RoundRobinProxySwitcher: %v", err) + } + c := colly.NewCollector(colly.IgnoreRobotsTxt()) + c.SetProxyFunc(rp) + + var called bool + c.OnError(func(r *colly.Response, _ error) { + called = true + if r.Request.ProxyURL != dead { + t.Errorf("Request.ProxyURL = %q, want %q", r.Request.ProxyURL, dead) + } + if r.ProxyURL != dead { + t.Errorf("Response.ProxyURL = %q, want %q", r.ProxyURL, dead) + } + }) + + if err := c.Visit("http://example.com/"); err == nil { + t.Fatal("expected Visit to fail") + } + if !called { + t.Fatal("OnError never fired") + } +} From dc2f2f734252d944bd3df0ac89f6fe92752344ce Mon Sep 17 00:00:00 2001 From: Shinku <17696928+Shinku-Chen@users.noreply.github.com> Date: Tue, 2 Jun 2026 17:18:12 +0800 Subject: [PATCH 5/9] change Context in SetProxyFunc --- colly.go | 11 ++++++++++- proxy/proxy.go | 6 +++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/colly.go b/colly.go index d1a57e6b3..16a4b4f93 100644 --- a/colly.go +++ b/colly.go @@ -1127,7 +1127,16 @@ func (c *Collector) SetProxy(proxyURL string) error { // The proxy type is determined by the URL scheme. "http" // and "socks5" are supported. If the scheme is empty, // "http" is assumed. -func (c *Collector) SetProxyFunc(p ProxyFunc) { +func (c *Collector) SetProxyFunc(f ProxyFunc) { + + var p ProxyFunc = func(pr *http.Request) (*url.URL, error) { + u, e := f(pr) + if h, _ := pr.Context().Value(ProxyURLKey).(*string); h != nil && u != nil { + *h = u.String() + } + return u, e + } + t, ok := c.backend.Client.Transport.(*http.Transport) if c.backend.Client.Transport != nil && ok { t.Proxy = p diff --git a/proxy/proxy.go b/proxy/proxy.go index e32ba1838..6474d72c9 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -34,9 +34,9 @@ func (r *roundRobinSwitcher) GetProxy(pr *http.Request) (*url.URL, error) { // context, so the chosen proxy is visible on Request.ProxyURL even when // the request fails before response headers (forkReq isolates ctx field // rewrites; pointer writes survive). - if h, _ := pr.Context().Value(colly.ProxyURLKey).(*string); h != nil { - *h = u.String() - } + //if h, _ := pr.Context().Value(colly.ProxyURLKey).(*string); h != nil { + // *h = u.String() + //} return u, nil } From 3d8e9643cbeb1df935377cde1bb35cc360554112 Mon Sep 17 00:00:00 2001 From: Shinku <17696928+Shinku-Chen@users.noreply.github.com> Date: Wed, 3 Jun 2026 09:03:03 +0800 Subject: [PATCH 6/9] change Context in SetProxyFunc --- colly.go | 37 ++++++++-------- proxy/proxy_test.go | 101 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 17 deletions(-) diff --git a/colly.go b/colly.go index 16a4b4f93..73a7ec2e1 100644 --- a/colly.go +++ b/colly.go @@ -208,15 +208,7 @@ var collectorCounter uint32 // other packages. type key int -// ProxyURLKey is the context key for the per-request proxy URL holder -// (a *string shared across the request and any clone Go's http.Client may -// fork during Do, so the value survives forkReq on the error path). -// -// ProxyFunc implementations should write the chosen proxy via this pattern: -// -// if h, _ := req.Context().Value(colly.ProxyURLKey).(*string); h != nil { -// *h = chosen.String() -// } +// ProxyURLKey is the context key for the request proxy address. const ( ProxyURLKey key = iota CheckRevisitKey @@ -681,6 +673,10 @@ func (c *Collector) scrape(u, method string, depth int, requestData io.Reader, c } // note: once 1.13 is minimum supported Go version, // replace this with http.NewRequestWithContext + // Place a *string holder in the context so the proxy URL chosen by the + // ProxyFunc survives net/http's send() forkReq (triggered by Client.Timeout). + // The fork shallow-copies the request and shares the context pointer, + // so pointer writes through the holder remain visible on the original req. req = req.WithContext(context.WithValue( context.WithValue(c.Context, CheckRevisitKey, checkRevisit), ProxyURLKey, new(string))) @@ -734,10 +730,10 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct req = hTrace.WithTrace(req) } origURL := req.URL - // Read the per-request *string holder set by the ProxyFunc. Called twice: - // inside checkResponseHeadersFunc so OnResponseHeaders sees it, and again - // after Cache returns so the error path (where the headers callback - // never fires) still surfaces the proxy URL. + // Read the per-request proxy URL holder set by the ProxyFunc wrapper. + // Called twice: inside checkResponseHeadersFunc so OnResponseHeaders sees + // it, and again after Cache returns so the error path (where the headers + // callback never fires) still surfaces the proxy URL. syncProxyURL := func(r *http.Request) { if h, _ := r.Context().Value(ProxyURLKey).(*string); h != nil && *h != "" { request.ProxyURL = *h @@ -1130,11 +1126,18 @@ func (c *Collector) SetProxy(proxyURL string) error { func (c *Collector) SetProxyFunc(f ProxyFunc) { var p ProxyFunc = func(pr *http.Request) (*url.URL, error) { - u, e := f(pr) - if h, _ := pr.Context().Value(ProxyURLKey).(*string); h != nil && u != nil { - *h = u.String() + // Capture the context before invoking the user's f. Legacy custom + // ProxyFuncs may do *pr = *pr.WithContext(WithValue(..., ProxyURLKey, "...")), + // which shadows the holder on pr but leaves the original chain (and + // our *string holder) reachable through origCtx. + origCtx := pr.Context() + proxyURL, err := f(pr) + if proxyURL != nil { + if h, _ := origCtx.Value(ProxyURLKey).(*string); h != nil { + *h = proxyURL.String() + } } - return u, e + return proxyURL, err } t, ok := c.backend.Client.Transport.(*http.Transport) diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 23e87b145..23e682614 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -15,9 +15,11 @@ package proxy import ( + "context" "fmt" "net/http" "net/http/httptest" + "net/url" "testing" "github.com/gocolly/colly/v2" @@ -92,3 +94,102 @@ func TestRoundRobinProxySwitcher_ProxyURLOnError(t *testing.T) { t.Fatal("OnError never fired") } } + +// TestSetProxyFunc_LegacyContextStringPropagates documents the interaction +// between a custom ProxyFunc that follows the legacy "WithContext+string" +// pattern and the current SetProxyFunc wrapper. +// +// The user's *pr = *pr.WithContext(...) mutation only affects the fork that +// net/http.send() created (Client.Timeout triggers forkReq), so the string +// the user writes into ProxyURLKey is discarded along with the fork. What +// actually surfaces on Request.ProxyURL is the *url.URL the ProxyFunc +// returns, written by the wrapper through the *string holder colly placed +// in the (shared) context. To make this concrete the test has the user +// write a marker string that intentionally differs from the returned URL, +// then asserts the URL — not the marker — is what propagates. +func TestSetProxyFunc_LegacyContextStringPropagates(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "ok") + })) + defer ts.Close() + + proxyURL, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("url.Parse: %v", err) + } + const userMarker = "user-wrote-this-but-it-should-be-ignored" + + c := colly.NewCollector(colly.IgnoreRobotsTxt()) + c.SetProxyFunc(func(pr *http.Request) (*url.URL, error) { + ctx := context.WithValue(pr.Context(), colly.ProxyURLKey, userMarker) + *pr = *pr.WithContext(ctx) + return proxyURL, nil + }) + + var called bool + c.OnResponse(func(r *colly.Response) { + called = true + if r.Request.ProxyURL != proxyURL.String() { + t.Errorf("Request.ProxyURL = %q, want %q (from returned *url.URL)", r.Request.ProxyURL, proxyURL.String()) + } + if r.ProxyURL != proxyURL.String() { + t.Errorf("Response.ProxyURL = %q, want %q", r.ProxyURL, proxyURL.String()) + } + if r.Request.ProxyURL == userMarker { + t.Errorf("Request.ProxyURL leaked the user marker %q — the WithContext+string write must be isolated by forkReq", userMarker) + } + }) + + if err := c.Visit("http://example.com/"); err != nil { + t.Fatalf("Visit: %v", err) + } + if !called { + t.Fatal("OnResponse never fired") + } +} + +// TestSetProxyFunc_LegacyContextStringOnError is the error-path counterpart: +// the same legacy WithContext+string ProxyFunc, but the proxy is a dead port +// so the request fails before any response headers. The returned *url.URL +// (not the user's discarded ctx string) must still be reflected on +// Request.ProxyURL / Response.ProxyURL — proving the *string holder write +// from SetProxyFunc's wrapper survives both forkReq and the error path. +func TestSetProxyFunc_LegacyContextStringOnError(t *testing.T) { + ln := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + dead := ln.URL + ln.Close() // guarantees dial refused on dead + + proxyURL, err := url.Parse(dead) + if err != nil { + t.Fatalf("url.Parse: %v", err) + } + const userMarker = "user-wrote-this-but-it-should-be-ignored" + + c := colly.NewCollector(colly.IgnoreRobotsTxt()) + c.SetProxyFunc(func(pr *http.Request) (*url.URL, error) { + ctx := context.WithValue(pr.Context(), colly.ProxyURLKey, userMarker) + *pr = *pr.WithContext(ctx) + return proxyURL, nil + }) + + var called bool + c.OnError(func(r *colly.Response, _ error) { + called = true + if r.Request.ProxyURL != dead { + t.Errorf("Request.ProxyURL = %q, want %q (from returned *url.URL)", r.Request.ProxyURL, dead) + } + if r.ProxyURL != dead { + t.Errorf("Response.ProxyURL = %q, want %q", r.ProxyURL, dead) + } + if r.Request.ProxyURL == userMarker { + t.Errorf("Request.ProxyURL leaked the user marker %q", userMarker) + } + }) + + if err := c.Visit("http://example.com/"); err == nil { + t.Fatal("expected Visit to fail") + } + if !called { + t.Fatal("OnError never fired") + } +} From 1295e3951424f02c73abb4e05e8e6b81b9ff7cba Mon Sep 17 00:00:00 2001 From: Shinku <17696928+Shinku-Chen@users.noreply.github.com> Date: Wed, 3 Jun 2026 09:17:32 +0800 Subject: [PATCH 7/9] change Context in SetProxyFunc --- proxy/proxy.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index 6474d72c9..a2de86355 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -30,13 +30,9 @@ type roundRobinSwitcher struct { func (r *roundRobinSwitcher) GetProxy(pr *http.Request) (*url.URL, error) { index := atomic.AddUint32(&r.index, 1) - 1 u := r.proxyURLs[index%uint32(len(r.proxyURLs))] - // Write through the per-request *string holder colly placed in the - // context, so the chosen proxy is visible on Request.ProxyURL even when - // the request fails before response headers (forkReq isolates ctx field - // rewrites; pointer writes survive). - //if h, _ := pr.Context().Value(colly.ProxyURLKey).(*string); h != nil { - // *h = u.String() - //} + // SetProxyFunc wraps this and writes the chosen proxy URL through the + // *string holder in the request context, so GetProxy itself only needs + // to return the *url.URL. return u, nil } From f7e6fb96d6a192013355934b265ffbfcabf69b3a Mon Sep 17 00:00:00 2001 From: Shinku <17696928+Shinku-Chen@users.noreply.github.com> Date: Wed, 3 Jun 2026 18:16:52 +0800 Subject: [PATCH 8/9] change Context in SetProxyFunc --- colly.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/colly.go b/colly.go index 73a7ec2e1..423da62f5 100644 --- a/colly.go +++ b/colly.go @@ -730,22 +730,12 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct req = hTrace.WithTrace(req) } origURL := req.URL - // Read the per-request proxy URL holder set by the ProxyFunc wrapper. - // Called twice: inside checkResponseHeadersFunc so OnResponseHeaders sees - // it, and again after Cache returns so the error path (where the headers - // callback never fires) still surfaces the proxy URL. - syncProxyURL := func(r *http.Request) { - if h, _ := r.Context().Value(ProxyURLKey).(*string); h != nil && *h != "" { - request.ProxyURL = *h - } - } checkResponseHeadersFunc := func(req *http.Request, statusCode int, headers http.Header) bool { if req.URL != origURL { request.URL = req.URL request.Headers = &req.Header } - syncProxyURL(req) - c.handleOnResponseHeaders(&Response{Ctx: ctx, Request: request, ProxyURL: request.ProxyURL, StatusCode: statusCode, Headers: &headers}) + c.handleOnResponseHeaders(&Response{Ctx: ctx, Request: request, StatusCode: statusCode, Headers: &headers}) return !request.abort } checkRequestHeadersFunc := func(req *http.Request) bool { @@ -753,7 +743,9 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct return !request.abort } response, err := c.backend.Cache(req, c.MaxBodySize, checkRequestHeadersFunc, checkResponseHeadersFunc, c.CacheDir, c.CacheExpiration) - syncProxyURL(req) + if proxyURL, ok := req.Context().Value(ProxyURLKey).(*string); ok { + request.ProxyURL = *proxyURL + } if err := c.handleOnError(response, err, request, ctx); err != nil { return err } From 34fbd02b033fc552d47971255317f2bb09b09622 Mon Sep 17 00:00:00 2001 From: Shinku <17696928+Shinku-Chen@users.noreply.github.com> Date: Wed, 3 Jun 2026 18:23:30 +0800 Subject: [PATCH 9/9] change Context in SetProxyFunc --- colly.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/colly.go b/colly.go index 423da62f5..5447624e2 100644 --- a/colly.go +++ b/colly.go @@ -1351,7 +1351,6 @@ func (c *Collector) handleOnError(response *Response, err error, request *Reques Ctx: ctx, } } - response.ProxyURL = request.ProxyURL if c.debugger != nil { c.debugger.Event(createEvent("error", request.ID, c.ID, map[string]string{ "url": request.URL.String(), @@ -1364,6 +1363,9 @@ func (c *Collector) handleOnError(response *Response, err error, request *Reques if response.Ctx == nil { response.Ctx = request.Ctx } + if response.ProxyURL == "" { + response.ProxyURL = request.ProxyURL + } for _, f := range c.errorCallbacks { f(response, err) }