From 89d8a4b726eba4ad13089ab1d2a33ec391190eab Mon Sep 17 00:00:00 2001
From: jstrmby <48553432+jstmrby@users.noreply.github.com>
Date: Mon, 3 Aug 2026 14:25:53 +0200
Subject: [PATCH 1/2] respect robots.txt by default

---
 colly.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/colly.go b/colly.go
index ae74b7c3e..f0c321e58 100644
--- a/colly.go
+++ b/colly.go
@@ -481,7 +481,7 @@ func (c *Collector) Init() {
 	c.wg = &sync.WaitGroup{}
 	c.lock = &sync.RWMutex{}
 	c.robotsMap = make(map[string]*robotstxt.RobotsData)
-	c.IgnoreRobotsTxt = true
+	c.IgnoreRobotsTxt = false
 	c.ID = atomic.AddUint32(&collectorCounter, 1)
 	c.TraceHTTP = false
 	c.Context = context.Background()
@@ -1468,7 +1468,6 @@ func createMultipartReader(boundary string, data map[string][]byte) io.Reader {
 	}
 	buffer.WriteString(dashBoundary + "--\n\n")
 	return bytes.NewReader(buffer.Bytes())
-
 }
 
 // randomBoundary was borrowed from

From aa80da792402d0f07992829f210d3c1f78cfc51e Mon Sep 17 00:00:00 2001
From: jstrmby <48553432+jstmrby@users.noreply.github.com>
Date: Mon, 3 Aug 2026 14:25:56 +0200
Subject: [PATCH 2/2] add tests for the robots.txt default and align queue test

- Add TestNewCollector/Defaults and TestRobotsDisallowedByDefault to cover the new default
- Opt queue.TestQueue out of robots.txt, as its test server only serves /delay
- Document the IgnoreRobotsTxt default on the field
---
 colly.go            |  2 +-
 colly_test.go       | 27 +++++++++++++++++++++++++++
 queue/queue_test.go |  4 ++++
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/colly.go b/colly.go
index f0c321e58..83bdcadb3 100644
--- a/colly.go
+++ b/colly.go
@@ -90,7 +90,7 @@ type Collector struct {
 	CacheDir string
 	// IgnoreRobotsTxt allows the Collector to ignore any restrictions set by
 	// the target host's robots.txt file.  See http://www.robotstxt.org/ for more
-	// information.
+	// information.  It defaults to false, meaning robots.txt is respected.
 	IgnoreRobotsTxt bool
 	// Async turns on asynchronous network communication. Use Collector.Wait() to
 	// be sure all requests have been finished.
diff --git a/colly_test.go b/colly_test.go
index e70d2774e..01238c0e3 100644
--- a/colly_test.go
+++ b/colly_test.go
@@ -467,6 +467,14 @@ func TestNoAcceptHeader(t *testing.T) {
 }
 
 func TestNewCollector(t *testing.T) {
+	t.Run("Defaults", func(t *testing.T) {
+		c := NewCollector()
+
+		if c.IgnoreRobotsTxt {
+			t.Fatal("c.IgnoreRobotsTxt = true, want false")
+		}
+	})
+
 	t.Run("Functional Options", func(t *testing.T) {
 		for name, test := range newCollectorTests {
 			t.Run(name, test)
@@ -1200,6 +1208,25 @@ func TestRobotsWhenDisallowedWithQueryParameter(t *testing.T) {
 	}
 }
 
+// TestRobotsDisallowedByDefault guards the default itself: the other robots
+// tests set Collector.IgnoreRobotsTxt explicitly, so none of them would catch
+// the default flipping back to ignoring robots.txt.
+func TestRobotsDisallowedByDefault(t *testing.T) {
+	ts := newTestServer()
+	defer ts.Close()
+
+	c := NewCollector()
+
+	c.OnResponse(func(resp *Response) {
+		t.Fatalf("Received response: %d", resp.StatusCode)
+	})
+
+	err := c.Visit(ts.URL + "/disallowed")
+	if err != ErrRobotsTxtBlocked {
+		t.Fatalf("wrong error: %v, want %v", err, ErrRobotsTxtBlocked)
+	}
+}
+
 func TestIgnoreRobotsWhenDisallowed(t *testing.T) {
 	ts := newTestServer()
 	defer ts.Close()
diff --git a/queue/queue_test.go b/queue/queue_test.go
index 1d10f8377..d14e047dc 100644
--- a/queue/queue_test.go
+++ b/queue/queue_test.go
@@ -42,8 +42,12 @@ func TestQueue(t *testing.T) {
 		put()
 		storage.AddRequest([]byte("error request"))
 	}
+	// The test server only serves /delay and hijacks every other route, so a
+	// robots.txt probe would fail and abort each request before OnRequest.
+	// This test covers queue mechanics, not robots.txt handling.
 	c := colly.NewCollector(
 		colly.AllowURLRevisit(),
+		colly.IgnoreRobotsTxt(),
 	)
 	c.OnRequest(func(req *colly.Request) {
 		atomic.AddUint32(&requests, 1)