WEBDIENSTE/cariddi
1
0
Fork 0
mirror of https://github.com/edoardottt/cariddi.git synced 2026-09-27 12:04:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix Ignore option

Browse Source
This commit is contained in:
edoardottt
2021-11-19 17:26:02 +01:00
parent 9eba34c051
commit 82a499bbb4
1 changed files with 30 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crawler/colly.go
+30 -18
View File
@@ -101,9 +101,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
// Only those links are visited which are in AllowedDomains
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
(intensive && intensiveOk(targetTemp, absoluteUrl)) {
if ignoreBool && !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
@@ -121,9 +123,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
// Only those links are visited which are in AllowedDomains
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
(intensive && intensiveOk(targetTemp, absoluteUrl)) {
if ignoreBool && !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
@@ -141,9 +145,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
// Only those links are visited which are in AllowedDomains
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
(intensive && intensiveOk(targetTemp, absoluteUrl)) {
if ignoreBool && !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
@@ -161,9 +167,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
// Only those links are visited which are in AllowedDomains
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
(intensive && intensiveOk(targetTemp, absoluteUrl)) {
if ignoreBool && !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
@@ -181,9 +189,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
// Only those links are visited which are in AllowedDomains
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
(intensive && intensiveOk(targetTemp, absoluteUrl)) {
if ignoreBool && !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
@@ -201,9 +211,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
// Only those links are visited which are in AllowedDomains
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
(intensive && intensiveOk(targetTemp, absoluteUrl)) {
if ignoreBool && !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)