WEBDIENSTE/cariddi
1
0
Fork 0
mirror of https://github.com/edoardottt/cariddi.git synced 2026-08-17 21:25:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix sitemap URLs discovery

Browse Source
This commit is contained in:
edoardottt
2026-01-11 11:43:30 +01:00
parent c7cf4d5cbf
commit 1e72b1830a
1 changed files with 1 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/crawler/colly.go
+1 -6
View File
@@ -409,7 +409,7 @@ func registerHTMLEvents(c *colly.Collector, event *Event) {
// XML event triggering an action.
func registerXMLEvents(c *colly.Collector, event *Event) {
// Create a callback on the XPath query searching for the URLs
c.OnXML("//url", func(e *colly.XMLElement) {
c.OnXML("//url/loc", func(e *colly.XMLElement) {
visitXMLLink(e.Text, event, e, c)
})
@@ -423,11 +423,6 @@ func registerXMLEvents(c *colly.Collector, event *Event) {
visitXMLLink(e.Text, event, e, c)
})
// Create a callback on the XPath query searching for the URLs
c.OnXML("//loc", func(e *colly.XMLElement) {
visitXMLLink(e.Text, event, e, c)
})
// Create a callback on the XPath query searching for the URLs
c.OnXML("//fileurl", func(e *colly.XMLElement) {
visitXMLLink(e.Text, event, e, c)