From 41c3c38b5a01dfd5faff7ae6fc43cf4de31b4f11 Mon Sep 17 00:00:00 2001
From: edoardottt <edoardott@gmail.com>
Date: Tue, 28 Sep 2021 22:36:45 +0200
Subject: [PATCH] Add onXML event to catch sitemaps

---
 crawler/colly.go | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/crawler/colly.go b/crawler/colly.go
index 8789511..c058618 100644
--- a/crawler/colly.go
+++ b/crawler/colly.go
@@ -237,9 +237,34 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 		}
 	})
 
+	// Create a callback on the XPath query searching for the URLs
+	c.OnXML("//urlset/url/loc", func(e *colly.XMLElement) {
+		link := e.Text
+		output.EncapsulateCustomRed("XML URL", link)
+		if len(link) != 0 {
+			absoluteUrl := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
+			// Visit link found on page
+			// Only those links are visited which are in AllowedDomains
+			if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
+				(intensive && intensiveOk(targetTemp, absoluteUrl)) {
+				if ignoreBool {
+					if !IgnoreMatch(link, ignoreSlice) {
+						FinalResults = append(FinalResults, absoluteUrl)
+						c.Visit(absoluteUrl)
+					}
+				} else {
+					FinalResults = append(FinalResults, absoluteUrl)
+					c.Visit(absoluteUrl)
+				}
+			}
+		}
+	})
+
 	c.OnResponse(func(r *colly.Response) {
 
-		fmt.Println(r.Request.URL.String())
+		if utils.SameDomain(protocolTemp+"://"+target, r.Request.URL.String()) {
+			fmt.Println(r.Request.URL.String())
+		}
 
 		lengthOk := len(string(r.Body)) > 10