From fa8ea89b10deac54cf0232d6762b08187c493e1e Mon Sep 17 00:00:00 2001 From: Cedric Brisson Date: Mon, 14 Nov 2022 17:29:35 -0500 Subject: [PATCH] Use c.OnRequest to print the URL that is actively being crawled rather than the URL it scrapped from a page --- pkg/crawler/colly.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index 07001d6..f430db1 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -112,6 +112,11 @@ func New(target string, txt string, html string, delayTime int, concurrency int, // crawler creation c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy, insecure, userAgent, target) + // On every request that Colly is making, print the URL it's currently visiting + c.OnRequest(func(e *colly.Request) { + fmt.Println(e.URL.String()) + }) + // On every a element which has href attribute call callback c.OnHTML("a[href]", func(e *colly.HTMLElement) { link := e.Attr("href") @@ -210,8 +215,6 @@ func New(target string, txt string, html string, delayTime int, concurrency int, } c.OnResponse(func(r *colly.Response) { - fmt.Println(r.Request.URL.String()) - minBodyLentgh := 10 lengthOk := len(string(r.Body)) > minBodyLentgh