Respect robots.txt by default

colly.go

@@ -90,7 +90,7 @@ type Collector struct {
 	CacheDir string
 	// IgnoreRobotsTxt allows the Collector to ignore any restrictions set by
 	// the target host's robots.txt file.  See http://www.robotstxt.org/ for more
-	// information.
+	// information.  It defaults to false, meaning robots.txt is respected.
 	IgnoreRobotsTxt bool
 	// Async turns on asynchronous network communication. Use Collector.Wait() to
 	// be sure all requests have been finished.
@@ -481,7 +481,7 @@ func (c *Collector) Init() {
 	c.wg = &sync.WaitGroup{}
 	c.lock = &sync.RWMutex{}
 	c.robotsMap = make(map[string]*robotstxt.RobotsData)
-	c.IgnoreRobotsTxt = true
+	c.IgnoreRobotsTxt = false
 	c.ID = atomic.AddUint32(&collectorCounter, 1)
 	c.TraceHTTP = false
 	c.Context = context.Background()
@@ -1468,7 +1468,6 @@ func createMultipartReader(boundary string, data map[string][]byte) io.Reader {
 	}
 	buffer.WriteString(dashBoundary + "--\n\n")
 	return bytes.NewReader(buffer.Bytes())
-
 }
 
 // randomBoundary was borrowed from

colly_test.go

@@ -467,6 +467,14 @@ func TestNoAcceptHeader(t *testing.T) {
 }
 
 func TestNewCollector(t *testing.T) {
+	t.Run("Defaults", func(t *testing.T) {
+		c := NewCollector()
+
+		if c.IgnoreRobotsTxt {
+			t.Fatal("c.IgnoreRobotsTxt = true, want false")
+		}
+	})
+
 	t.Run("Functional Options", func(t *testing.T) {
 		for name, test := range newCollectorTests {
 			t.Run(name, test)
@@ -1200,6 +1208,25 @@ func TestRobotsWhenDisallowedWithQueryParameter(t *testing.T) {
 	}
 }
 
+// TestRobotsDisallowedByDefault guards the default itself: the other robots
+// tests set Collector.IgnoreRobotsTxt explicitly, so none of them would catch
+// the default flipping back to ignoring robots.txt.
+func TestRobotsDisallowedByDefault(t *testing.T) {
+	ts := newTestServer()
+	defer ts.Close()
+
+	c := NewCollector()
+
+	c.OnResponse(func(resp *Response) {
+		t.Fatalf("Received response: %d", resp.StatusCode)
+	})
+
+	err := c.Visit(ts.URL + "/disallowed")
+	if err != ErrRobotsTxtBlocked {
+		t.Fatalf("wrong error: %v, want %v", err, ErrRobotsTxtBlocked)
+	}
+}
+
 func TestIgnoreRobotsWhenDisallowed(t *testing.T) {
 	ts := newTestServer()
 	defer ts.Close()

queue/queue_test.go

@@ -42,8 +42,12 @@ func TestQueue(t *testing.T) {
 		put()
 		storage.AddRequest([]byte("error request"))
 	}
+	// The test server only serves /delay and hijacks every other route, so a
+	// robots.txt probe would fail and abort each request before OnRequest.
+	// This test covers queue mechanics, not robots.txt handling.
 	c := colly.NewCollector(
 		colly.AllowURLRevisit(),
+		colly.IgnoreRobotsTxt(),
 	)
 	c.OnRequest(func(req *colly.Request) {
 		atomic.AddUint32(&requests, 1)