diff --git a/queue/queue.go b/queue/queue.go index 0d0d78a66..82869c7b0 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -90,6 +90,7 @@ func (q *Queue) AddURL(URL string) error { r := &colly.Request{ URL: u2, Method: "GET", + Depth: 1, } d, err := r.Marshal() if err != nil { @@ -131,6 +132,10 @@ func (q *Queue) Size() (int, error) { // to perform requests. Run blocks while the queue has active requests // The given Storage must not be used directly while Run blocks. func (q *Queue) Run(c *colly.Collector) error { + if c.Async { + // Async causes Queue crawling to silently fail, since all requests finish "Instantly" + panic("Cannot run Async collector in Queue!") + } q.mut.Lock() if q.wake != nil && q.running == true { q.mut.Unlock() diff --git a/queue/queue_test.go b/queue/queue_test.go index 1d10f8377..c92139203 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -1,6 +1,7 @@ package queue import ( + "fmt" "math/rand" "net/http" "net/http/httptest" @@ -75,6 +76,59 @@ func TestQueue(t *testing.T) { } } +func TestCollectorDepth(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(serverHandler)) + defer server.Close() + maxDepth := 4 + c := colly.NewCollector( + colly.MaxDepth(maxDepth), + colly.AllowURLRevisit(), + ) + + storage := &InMemoryQueueStorage{MaxSize: 100000} + q, err := New(10, storage) + if err != nil { + panic(err) + } + + c.OnRequest(func(req *colly.Request) { + if req.Depth > maxDepth { + errMsg := fmt.Sprintf("Invalid depth value! Expected %d, got %d", maxDepth, req.Depth) + panic(errMsg) + } + req.Visit(server.URL) + }) + + q.AddURL(server.URL) + + err = q.Run(c) + if err != nil { + t.Fatalf("Queue.Run() return an error: %v", err) + } +} + +func TestAsyncPanic(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("The code did not panic") + } + }() + + c := colly.NewCollector( + colly.MaxDepth(5), + colly.AllowURLRevisit(), + colly.Async(true), + ) + + storage := &InMemoryQueueStorage{MaxSize: 100000} + q, err := New(10, storage) + if err != nil { + panic(err) + } + + q.Run(c) +} + func serverHandler(w http.ResponseWriter, req *http.Request) { if !serverRoute(w, req) { shutdown(w) diff --git a/request.go b/request.go index 6beef8347..d09cd8b58 100644 --- a/request.go +++ b/request.go @@ -158,7 +158,7 @@ func (r *Request) Retry() error { // Do submits the request func (r *Request) Do() error { - return r.collector.scrape(r.URL.String(), r.Method, r.Depth, r.Body, r.Ctx, *r.Headers, !r.collector.AllowURLRevisit) + return r.collector.scrape(r.URL.String(), r.Method, r.Depth+1, r.Body, r.Ctx, *r.Headers, !r.collector.AllowURLRevisit) } // Marshal serializes the Request