From b5d63417f7c8fe41b809cd1c9b4815d009ed8a1c Mon Sep 17 00:00:00 2001 From: edoardottt Date: Thu, 8 Dec 2022 12:11:17 +0100 Subject: [PATCH] fix robots and sitemap --- pkg/crawler/colly.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index 5668cc6..1153bc9 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -298,19 +298,21 @@ func New(scan *Scan) *Results { addPath = "/" } - absoluteURL = protocolTemp + "://" + scan.Target + addPath + "robots.txt" - if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) { - err = c.Visit(absoluteURL) - if err != nil && scan.Debug && !errors.Is(err, colly.ErrAlreadyVisited) { - log.Println(err) + if path == "" || path == "/" { + absoluteURL = protocolTemp + "://" + scan.Target + addPath + "robots.txt" + if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) { + err = c.Visit(absoluteURL) + if err != nil && scan.Debug && !errors.Is(err, colly.ErrAlreadyVisited) { + log.Println(err) + } } - } - absoluteURL = protocolTemp + "://" + scan.Target + addPath + "sitemap.xml" - if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) { - err = c.Visit(absoluteURL) - if err != nil && scan.Debug && !errors.Is(err, colly.ErrAlreadyVisited) { - log.Println(err) + absoluteURL = protocolTemp + "://" + scan.Target + addPath + "sitemap.xml" + if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) { + err = c.Visit(absoluteURL) + if err != nil && scan.Debug && !errors.Is(err, colly.ErrAlreadyVisited) { + log.Println(err) + } } } }