From e4cfd30d49f3712a71b465a14bc79abad14194c7 Mon Sep 17 00:00:00 2001 From: Cedric Brisson Date: Mon, 14 Nov 2022 17:13:34 -0500 Subject: [PATCH 1/3] simplify the final list append (recommended by go-staticcheck). Instead of iterating through the list, we simply unfurl it and append it to the final list --- pkg/crawler/colly.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index a7cf19b..14faeaa 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -221,9 +221,7 @@ func New(target string, txt string, html string, delayTime int, concurrency int, // HERE SCAN FOR SECRETS if secretsFlag && lengthOk { secretsSlice := huntSecrets(secretsFile, r.Request.URL.String(), string(r.Body)) - for _, elem := range secretsSlice { - FinalSecrets = append(FinalSecrets, elem) - } + FinalSecrets = append(FinalSecrets, secretsSlice...) } // HERE SCAN FOR ENDPOINTS if endpointsFlag { From 4ae6be404e7591688552d9e466c5462d09ba0bd5 Mon Sep 17 00:00:00 2001 From: Cedric Brisson Date: Mon, 14 Nov 2022 17:17:07 -0500 Subject: [PATCH 2/3] simplify the final list append (recommended by go-staticcheck). Instead of iterating through the list, we simply unfurl it and append it to the final list (again) --- pkg/crawler/colly.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index 14faeaa..07001d6 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -242,17 +242,13 @@ func New(target string, txt string, html string, delayTime int, concurrency int, // HERE SCAN FOR ERRORS if errorsFlag { errorsSlice := huntErrors(r.Request.URL.String(), string(r.Body)) - for _, elem := range errorsSlice { - FinalErrors = append(FinalErrors, elem) - } + FinalErrors = append(FinalErrors, errorsSlice...) } // HERE SCAN FOR INFOS if infoFlag { infosSlice := huntInfos(r.Request.URL.String(), string(r.Body)) - for _, elem := range infosSlice { - FinalInfos = append(FinalInfos, elem) - } + FinalInfos = append(FinalInfos, infosSlice...) } } }) From fa8ea89b10deac54cf0232d6762b08187c493e1e Mon Sep 17 00:00:00 2001 From: Cedric Brisson Date: Mon, 14 Nov 2022 17:29:35 -0500 Subject: [PATCH 3/3] Use c.OnRequest to print the URL that is actively being crawled rather than the URL it scrapped from a page --- pkg/crawler/colly.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index 07001d6..f430db1 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -112,6 +112,11 @@ func New(target string, txt string, html string, delayTime int, concurrency int, // crawler creation c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy, insecure, userAgent, target) + // On every request that Colly is making, print the URL it's currently visiting + c.OnRequest(func(e *colly.Request) { + fmt.Println(e.URL.String()) + }) + // On every a element which has href attribute call callback c.OnHTML("a[href]", func(e *colly.HTMLElement) { link := e.Attr("href") @@ -210,8 +215,6 @@ func New(target string, txt string, html string, delayTime int, concurrency int, } c.OnResponse(func(r *colly.Response) { - fmt.Println(r.Request.URL.String()) - minBodyLentgh := 10 lengthOk := len(string(r.Body)) > minBodyLentgh