Merge pull request #84 from edoardottt/devel

Devel

internal/slice/slice.go

@@ -27,6 +27,7 @@ along with this program.  If not, see http://www.gnu.org/licenses/.
 package slice
 
 import (
+	"math/rand"
 	"net/http"
 	"strings"
 )
@@ -93,3 +94,15 @@ func CheckCookies(input string) []*http.Cookie {
 
 	return result
 }
+
+// RandSeq produces a random sequence of letters.
+func RandSeq(n int) string {
+	b := make([]rune, n)
+	letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+
+	return string(b)
+}

pkg/crawler/colly.go

@@ -31,8 +31,10 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"math/rand"
 	"net/http"
 	"os"
+	"os/signal"
 	"regexp"
 	"strings"
 	"time"
@@ -259,23 +261,26 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
 	// Start scraping on target
 	path, err := urlUtils.GetPath(protocolTemp + "://" + target)
 	if err == nil {
+		var (
+			addPath     string
+			absoluteURL string
+		)
+
 		if path == "" {
-			err = c.Visit(protocolTemp + "://" + target + "/" + "robots.txt")
-			if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) {
-				log.Println(err)
-			}
+			addPath = "/"
+		}
 
-			err = c.Visit(protocolTemp + "://" + target + "/" + "sitemap.xml")
-			if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) {
-				log.Println(err)
-			}
-		} else if path == "/" {
-			err = c.Visit(protocolTemp + "://" + target + "robots.txt")
+		absoluteURL = protocolTemp + "://" + target + addPath + "robots.txt"
+		if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) {
+			err = c.Visit(absoluteURL)
 			if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) {
 				log.Println(err)
 			}
+		}
 
-			err = c.Visit(protocolTemp + "://" + target + "sitemap.xml")
+		absoluteURL = protocolTemp + "://" + target + addPath + "sitemap.xml"
+		if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) {
+			err = c.Visit(absoluteURL)
 			if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) {
 				log.Println(err)
 			}
@@ -287,6 +292,24 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
 		log.Println(err)
 	}
 
+	// Setup graceful exits
+	chanC := make(chan os.Signal, 1)
+	lettersNum := 23
+
+	signal.Notify(chanC, os.Interrupt)
+	rand.Seed(time.Now().UnixNano())
+
+	go func() {
+		for range chanC {
+			if !plain {
+				fmt.Fprint(os.Stdout, "\r")
+				fmt.Println("CTRL+C pressed: Exiting")
+			}
+
+			c.AllowedDomains = []string{sliceUtils.RandSeq(lettersNum)}
+		}
+	}()
+
 	c.Wait()
 
 	if html != "" {
@@ -306,10 +329,6 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
 	c.IgnoreRobotsTxt = true
 	c.AllowURLRevisit = false
 
-	if userAgent != "" {
-		c.UserAgent = userAgent
-	}
-
 	err := c.Limit(
 		&colly.LimitRule{
 			Parallelism: concurrency,
@@ -339,6 +358,10 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
 		c.UserAgent = GenerateRandomUserAgent()
 	}
 
+	if userAgent != "" {
+		c.UserAgent = userAgent
+	}
+
 	// Use a Proxy if needed
 	if proxy != "" {
 		err := c.SetProxy(proxy)