WEBDIENSTE/cariddi
1
0
Fork 0
mirror of https://github.com/edoardottt/cariddi.git synced 2026-09-12 04:37:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

update ignore

Browse Source
This commit is contained in:
edoardottt
2022-11-14 21:36:41 +01:00
parent 71e1301b8e
commit da540eba55
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/crawler/colly.go
+3 -3
View File
@@ -121,7 +121,7 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
// Only those links are visited which are in AllowedDomains
if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
(intensive && intensiveOk(targetTemp, absoluteURL, debug)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) {
err := c.Visit(absoluteURL)
if !errors.Is(err, colly.ErrAlreadyVisited) {
FinalResults = append(FinalResults, absoluteURL)
@@ -369,7 +369,7 @@ func visitHTMLLink(link, protocolTemp, targetTemp, target string, intensive, ign
// Only those links are visited which are in AllowedDomains
if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
(intensive && intensiveOk(targetTemp, absoluteURL, debug)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) {
err := c.Visit(absoluteURL)
if !errors.Is(err, colly.ErrAlreadyVisited) {
*finalResults = append(*finalResults, absoluteURL)
@@ -392,7 +392,7 @@ func visitXMLLink(link, protocolTemp, targetTemp, target string, intensive, igno
// Only those links are visited which are in AllowedDomains
if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
(intensive && intensiveOk(targetTemp, absoluteURL, debug)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) {
err := c.Visit(absoluteURL)
if !errors.Is(err, colly.ErrAlreadyVisited) {
*finalResults = append(*finalResults, absoluteURL)