diff --git a/colly.go b/colly.go index ae74b7c3e..83bdcadb3 100644 --- a/colly.go +++ b/colly.go @@ -90,7 +90,7 @@ type Collector struct { CacheDir string // IgnoreRobotsTxt allows the Collector to ignore any restrictions set by // the target host's robots.txt file. See http://www.robotstxt.org/ for more - // information. + // information. It defaults to false, meaning robots.txt is respected. IgnoreRobotsTxt bool // Async turns on asynchronous network communication. Use Collector.Wait() to // be sure all requests have been finished. @@ -481,7 +481,7 @@ func (c *Collector) Init() { c.wg = &sync.WaitGroup{} c.lock = &sync.RWMutex{} c.robotsMap = make(map[string]*robotstxt.RobotsData) - c.IgnoreRobotsTxt = true + c.IgnoreRobotsTxt = false c.ID = atomic.AddUint32(&collectorCounter, 1) c.TraceHTTP = false c.Context = context.Background() @@ -1468,7 +1468,6 @@ func createMultipartReader(boundary string, data map[string][]byte) io.Reader { } buffer.WriteString(dashBoundary + "--\n\n") return bytes.NewReader(buffer.Bytes()) - } // randomBoundary was borrowed from diff --git a/colly_test.go b/colly_test.go index e70d2774e..01238c0e3 100644 --- a/colly_test.go +++ b/colly_test.go @@ -467,6 +467,14 @@ func TestNoAcceptHeader(t *testing.T) { } func TestNewCollector(t *testing.T) { + t.Run("Defaults", func(t *testing.T) { + c := NewCollector() + + if c.IgnoreRobotsTxt { + t.Fatal("c.IgnoreRobotsTxt = true, want false") + } + }) + t.Run("Functional Options", func(t *testing.T) { for name, test := range newCollectorTests { t.Run(name, test) @@ -1200,6 +1208,25 @@ func TestRobotsWhenDisallowedWithQueryParameter(t *testing.T) { } } +// TestRobotsDisallowedByDefault guards the default itself: the other robots +// tests set Collector.IgnoreRobotsTxt explicitly, so none of them would catch +// the default flipping back to ignoring robots.txt. +func TestRobotsDisallowedByDefault(t *testing.T) { + ts := newTestServer() + defer ts.Close() + + c := NewCollector() + + c.OnResponse(func(resp *Response) { + t.Fatalf("Received response: %d", resp.StatusCode) + }) + + err := c.Visit(ts.URL + "/disallowed") + if err != ErrRobotsTxtBlocked { + t.Fatalf("wrong error: %v, want %v", err, ErrRobotsTxtBlocked) + } +} + func TestIgnoreRobotsWhenDisallowed(t *testing.T) { ts := newTestServer() defer ts.Close() diff --git a/queue/queue_test.go b/queue/queue_test.go index 1d10f8377..d14e047dc 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -42,8 +42,12 @@ func TestQueue(t *testing.T) { put() storage.AddRequest([]byte("error request")) } + // The test server only serves /delay and hijacks every other route, so a + // robots.txt probe would fail and abort each request before OnRequest. + // This test covers queue mechanics, not robots.txt handling. c := colly.NewCollector( colly.AllowURLRevisit(), + colly.IgnoreRobotsTxt(), ) c.OnRequest(func(req *colly.Request) { atomic.AddUint32(&requests, 1)