WEBDIENSTE/cariddi
1
0
Fork 0
mirror of https://github.com/edoardottt/cariddi.git synced 2026-09-10 19:57:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

Regex (or other method) for Intensive mode #12

Browse Source
This commit is contained in:
edoardottt
2021-09-25 10:54:54 +02:00
parent 0399486239
commit b02974f015
2 changed files with 71 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crawler/colly.go
+58 -40
View File
@@ -99,25 +99,16 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
},
)
c.AllowURLRevisit = false
// Using timeout if needed
if timeout != 10 {
c.SetRequestTimeout(time.Second * time.Duration(timeout))
}
// Using cache if needed
if cache {
c.CacheDir = ".cariddi_cache"
}
// if not intensive
if !intensive {
c.AllowedDomains = []string{targetTemp}
} else {
// otherwise, intensive set
targetTemp = "." + targetTemp
targetTemp = strings.ReplaceAll(targetTemp, ".", "\\.")
targetRegex := "([-a-z0-9.]*)" + targetTemp + "([-a-z0-9.]*)"
c.URLFilters =
[]*regexp.Regexp{regexp.MustCompile(targetRegex)}
}
// Use a Random User Agent for each request if needed
if rua {
@@ -137,16 +128,20 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
link := e.Attr("href")
if len(link) != 0 && link[0] != '#' {
absoluteUrl := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
c.Visit(e.Request.AbsoluteURL(link))
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
(intensive && intensiveOk(targetTemp, absoluteUrl)) {
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults,
utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)))
c.Visit(utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)))
}
}
})
@@ -155,16 +150,20 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
c.OnHTML("script[src]", func(e *colly.HTMLElement) {
link := e.Attr("src")
if len(link) != 0 {
absoluteUrl := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
c.Visit(e.Request.AbsoluteURL(link))
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
(intensive && intensiveOk(targetTemp, absoluteUrl)) {
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults,
utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)))
c.Visit(utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)))
}
}
})
@@ -173,16 +172,20 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
c.OnHTML("link[href]", func(e *colly.HTMLElement) {
link := e.Attr("href")
if len(link) != 0 {
absoluteUrl := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
c.Visit(e.Request.AbsoluteURL(link))
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
(intensive && intensiveOk(targetTemp, absoluteUrl)) {
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults,
utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)))
c.Visit(utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)))
}
}
})
@@ -191,16 +194,20 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
c.OnHTML("iframe[src]", func(e *colly.HTMLElement) {
link := e.Attr("src")
if len(link) != 0 {
absoluteUrl := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
c.Visit(e.Request.AbsoluteURL(link))
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
(intensive && intensiveOk(targetTemp, absoluteUrl)) {
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults,
utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)))
c.Visit(utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)))
}
}
})
@@ -350,7 +357,7 @@ func RetrieveBody(target string) string {
return ""
}
//isLinkOkay checks if a link is buit in a proper way
//isLinkOkay checks if a link is built in a proper way
func isLinkOkay(input string) bool {
_, err := url.Parse(input)
return err == nil
@@ -366,3 +373,14 @@ func IgnoreMatch(url string, ignoreSlice []string) bool {
}
return false
}
//intensiveOk checks if a given url can be crawled
//in intensive mode (if the 2nd level domain matches with
//the inputted target).
func intensiveOk(target string, urlInput string) bool {
u, err := url.Parse(urlInput)
if err != nil {
return false
}
return utils.GetRootHost(u.Host) == target
}
utils/urls.go
+13
View File
@@ -117,3 +117,16 @@ func AbsoluteURL(protocol string, target string, path string) string {
}
return protocol + "://" + target + "/" + path
}
//SameDomain checks if two urls have the same domain
func SameDomain(url1 string, url2 string) bool {
u1, err := url.Parse(url1)
if err != nil {
return false
}
u2, err := url.Parse(url2)
if err != nil {
return false
}
return u1.Host == u2.Host
}