From 9a253c2670535914a06ff0159ccdbca7839f029e Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sun, 31 Oct 2021 22:14:19 +0100 Subject: [PATCH 1/4] Fix help (-h) output --- output/help.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/output/help.go b/output/help.go index ee138ac..e868b56 100644 --- a/output/help.go +++ b/output/help.go @@ -44,9 +44,9 @@ func PrintHelp() { Hunt for juicy file extensions. Integer from 1(juicy) to 7(not juicy). -h Print the help. -headers string - Use custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2". + Use custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2". -headersfile string - Read from an external file custom headers (same format of headers flag). + Read from an external file custom headers (same format of headers flag). -i string Ignore the URL containing at least one of the elements of this array. -intensive @@ -60,7 +60,7 @@ func PrintHelp() { -plain Print only the results. -proxy string - Set a Proxy to be used (http and socks5 supported). + Set a Proxy to be used (http and socks5 supported). -rua Use a random browser user agent on every request. -s Hunt for secrets. From a385de12ea1d17a64d536b0f26319c53a03810f4 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Mon, 1 Nov 2021 18:55:09 +0100 Subject: [PATCH 2/4] Don't ignore robots.txt --- crawler/colly.go | 1 + 1 file changed, 1 insertion(+) diff --git a/crawler/colly.go b/crawler/colly.go index 9b0b66b..6ebb196 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -292,6 +292,7 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int, c := colly.NewCollector( colly.Async(true), ) + c.IgnoreRobotsTxt = false c.Limit( &colly.LimitRule{ From 042ad24bb754b63ad7f33fcce0e3c88fa32673e9 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Thu, 4 Nov 2021 12:21:50 +0100 Subject: [PATCH 3/4] Update Ignore match --- crawler/colly.go | 54 +++++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/crawler/colly.go b/crawler/colly.go index 6ebb196..b83cf88 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -101,11 +101,9 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool { - if !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) - } + if ignoreBool && !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl) @@ -123,11 +121,9 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool { - if !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) - } + if ignoreBool && !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl) @@ -145,11 +141,9 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool { - if !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) - } + if ignoreBool && !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl) @@ -167,11 +161,9 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool { - if !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) - } + if ignoreBool && !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl) @@ -189,11 +181,9 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool { - if !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) - } + if ignoreBool && !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl) @@ -211,11 +201,9 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool { - if !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) - } + if ignoreBool && !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl) @@ -235,7 +223,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency c.OnResponse(func(r *colly.Response) { - fmt.Println(r.Request.URL.String()) + if ignoreBool && !IgnoreMatch(r.Request.URL.String(), ignoreSlice) { + fmt.Println(r.Request.URL.String()) + } else { + fmt.Println(r.Request.URL.String()) + } lengthOk := len(string(r.Body)) > 10 From 82a499bbb4cd2e405b3510eda741d27baf4c2836 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Fri, 19 Nov 2021 17:26:02 +0100 Subject: [PATCH 4/4] Fix Ignore option --- crawler/colly.go | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/crawler/colly.go b/crawler/colly.go index b83cf88..47741e8 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -101,9 +101,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool && !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + if ignoreBool { + if !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl) @@ -121,9 +123,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool && !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + if ignoreBool { + if !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl) @@ -141,9 +145,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool && !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + if ignoreBool { + if !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl) @@ -161,9 +167,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool && !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + if ignoreBool { + if !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl) @@ -181,9 +189,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool && !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + if ignoreBool { + if !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl) @@ -201,9 +211,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Only those links are visited which are in AllowedDomains if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || (intensive && intensiveOk(targetTemp, absoluteUrl)) { - if ignoreBool && !IgnoreMatch(link, ignoreSlice) { - FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + if ignoreBool { + if !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } } else { FinalResults = append(FinalResults, absoluteUrl) c.Visit(absoluteUrl)