diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index 2bfd26e..2b634ea 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -120,7 +120,7 @@ func New(target string, txt string, html string, delayTime int, concurrency int, // Visit link found on page // Only those links are visited which are in AllowedDomains if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) || - (intensive && intensiveOk(targetTemp, absoluteURL)) { + (intensive && intensiveOk(targetTemp, absoluteURL, debug)) { if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) { err := c.Visit(absoluteURL) if !errors.Is(err, colly.ErrAlreadyVisited) { @@ -368,7 +368,7 @@ func visitHTMLLink(link, protocolTemp, targetTemp, target string, intensive, ign // Visit link found on page // Only those links are visited which are in AllowedDomains if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) || - (intensive && intensiveOk(targetTemp, absoluteURL)) { + (intensive && intensiveOk(targetTemp, absoluteURL, debug)) { if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) { err := c.Visit(absoluteURL) if !errors.Is(err, colly.ErrAlreadyVisited) { @@ -391,7 +391,7 @@ func visitXMLLink(link, protocolTemp, targetTemp, target string, intensive, igno // Visit link found on page // Only those links are visited which are in AllowedDomains if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) || - (intensive && intensiveOk(targetTemp, absoluteURL)) { + (intensive && intensiveOk(targetTemp, absoluteURL, debug)) { if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) { err := c.Visit(absoluteURL) if !errors.Is(err, colly.ErrAlreadyVisited) { @@ -581,11 +581,14 @@ func IgnoreMatch(url string, ignoreSlice []string) bool { // intensiveOk checks if a given url can be crawled // in intensive mode (if the 2nd level domain matches with // the inputted target). -func intensiveOk(target string, urlInput string) bool { +func intensiveOk(target string, urlInput string, debug bool) bool { root, err := urlUtils.GetRootHost(urlInput) if err != nil { - fmt.Println(err.Error()) - os.Exit(1) + if debug { + fmt.Println(err.Error() + ": " + urlInput) + } + + return false } return root == target