WEBDIENSTE/cariddi
1
0
Fork 0
mirror of https://github.com/edoardottt/cariddi.git synced 2026-09-26 19:44:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix #78

Browse Source
This commit is contained in:
edoardottt
2022-11-14 21:18:40 +01:00
parent b488b3005c
commit e52c4ae792
1 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/crawler/colly.go
+9 -6
View File
@@ -120,7 +120,7 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
// Visit link found on page
// Only those links are visited which are in AllowedDomains
if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
(intensive && intensiveOk(targetTemp, absoluteURL)) {
(intensive && intensiveOk(targetTemp, absoluteURL, debug)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) {
err := c.Visit(absoluteURL)
if !errors.Is(err, colly.ErrAlreadyVisited) {
@@ -368,7 +368,7 @@ func visitHTMLLink(link, protocolTemp, targetTemp, target string, intensive, ign
// Visit link found on page
// Only those links are visited which are in AllowedDomains
if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
(intensive && intensiveOk(targetTemp, absoluteURL)) {
(intensive && intensiveOk(targetTemp, absoluteURL, debug)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) {
err := c.Visit(absoluteURL)
if !errors.Is(err, colly.ErrAlreadyVisited) {
@@ -391,7 +391,7 @@ func visitXMLLink(link, protocolTemp, targetTemp, target string, intensive, igno
// Visit link found on page
// Only those links are visited which are in AllowedDomains
if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
(intensive && intensiveOk(targetTemp, absoluteURL)) {
(intensive && intensiveOk(targetTemp, absoluteURL, debug)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) {
err := c.Visit(absoluteURL)
if !errors.Is(err, colly.ErrAlreadyVisited) {
@@ -581,11 +581,14 @@ func IgnoreMatch(url string, ignoreSlice []string) bool {
// intensiveOk checks if a given url can be crawled
// in intensive mode (if the 2nd level domain matches with
// the inputted target).
func intensiveOk(target string, urlInput string) bool {
func intensiveOk(target string, urlInput string, debug bool) bool {
root, err := urlUtils.GetRootHost(urlInput)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
if debug {
fmt.Println(err.Error() + ": " + urlInput)
}
return false
}
return root == target