WEBDIENSTE/cariddi
1
0
Fork 0
mirror of https://github.com/edoardottt/cariddi.git synced 2026-09-13 13:17:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix robots and sitemap

Browse Source
This commit is contained in:
edoardottt
2022-12-08 12:11:17 +01:00
parent 43fe15cf58
commit b5d63417f7
1 changed files with 13 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/crawler/colly.go
+13 -11
View File
@@ -298,19 +298,21 @@ func New(scan *Scan) *Results {
addPath = "/"
}
absoluteURL = protocolTemp + "://" + scan.Target + addPath + "robots.txt"
if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) {
err = c.Visit(absoluteURL)
if err != nil && scan.Debug && !errors.Is(err, colly.ErrAlreadyVisited) {
log.Println(err)
if path == "" || path == "/" {
absoluteURL = protocolTemp + "://" + scan.Target + addPath + "robots.txt"
if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) {
err = c.Visit(absoluteURL)
if err != nil && scan.Debug && !errors.Is(err, colly.ErrAlreadyVisited) {
log.Println(err)
}
}
}
absoluteURL = protocolTemp + "://" + scan.Target + addPath + "sitemap.xml"
if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) {
err = c.Visit(absoluteURL)
if err != nil && scan.Debug && !errors.Is(err, colly.ErrAlreadyVisited) {
log.Println(err)
absoluteURL = protocolTemp + "://" + scan.Target + addPath + "sitemap.xml"
if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) {
err = c.Visit(absoluteURL)
if err != nil && scan.Debug && !errors.Is(err, colly.ErrAlreadyVisited) {
log.Println(err)
}
}
}
}