From d9d74ed498135cec88fa467b44cba8ac06bc8e01 Mon Sep 17 00:00:00 2001
From: edoardottt <edoardott@gmail.com>
Date: Tue, 15 Nov 2022 13:55:54 +0100
Subject: [PATCH 1/3] Fix Initial call - ignore (#81)

---
 pkg/crawler/colly.go | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go
index a7cf19b..556eb9a 100644
--- a/pkg/crawler/colly.go
+++ b/pkg/crawler/colly.go
@@ -262,23 +262,26 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
 	// Start scraping on target
 	path, err := urlUtils.GetPath(protocolTemp + "://" + target)
 	if err == nil {
+		var (
+			addPath     string
+			absoluteURL string
+		)
+
 		if path == "" {
-			err = c.Visit(protocolTemp + "://" + target + "/" + "robots.txt")
-			if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) {
-				log.Println(err)
-			}
+			addPath = "/"
+		}
 
-			err = c.Visit(protocolTemp + "://" + target + "/" + "sitemap.xml")
-			if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) {
-				log.Println(err)
-			}
-		} else if path == "/" {
-			err = c.Visit(protocolTemp + "://" + target + "robots.txt")
+		absoluteURL = protocolTemp + "://" + target + addPath + "robots.txt"
+		if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) {
+			err = c.Visit(absoluteURL)
 			if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) {
 				log.Println(err)
 			}
+		}
 
-			err = c.Visit(protocolTemp + "://" + target + "sitemap.xml")
+		absoluteURL = protocolTemp + "://" + target + addPath + "sitemap.xml"
+		if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) {
+			err = c.Visit(absoluteURL)
 			if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) {
 				log.Println(err)
 			}

From ba714c026b0969bf0d4941d1fa9e381f8b3e5a19 Mon Sep 17 00:00:00 2001
From: edoardottt <edoardott@gmail.com>
Date: Tue, 15 Nov 2022 14:16:05 +0100
Subject: [PATCH 2/3] Fix user agent

---
 pkg/crawler/colly.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go
index 556eb9a..5df23f6 100644
--- a/pkg/crawler/colly.go
+++ b/pkg/crawler/colly.go
@@ -312,10 +312,6 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
 	c.IgnoreRobotsTxt = true
 	c.AllowURLRevisit = false
 
-	if userAgent != "" {
-		c.UserAgent = userAgent
-	}
-
 	err := c.Limit(
 		&colly.LimitRule{
 			Parallelism: concurrency,
@@ -345,6 +341,10 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
 		c.UserAgent = GenerateRandomUserAgent()
 	}
 
+	if userAgent != "" {
+		c.UserAgent = userAgent
+	}
+
 	// Use a Proxy if needed
 	if proxy != "" {
 		err := c.SetProxy(proxy)

From b4c7b133f00e45b23a98da23c73afe94ceac7d57 Mon Sep 17 00:00:00 2001
From: edoardottt <edoardott@gmail.com>
Date: Tue, 15 Nov 2022 14:42:50 +0100
Subject: [PATCH 3/3] Add CTRL-C handle

---
 internal/slice/slice.go | 13 +++++++++++++
 pkg/crawler/colly.go    | 20 ++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/internal/slice/slice.go b/internal/slice/slice.go
index e67314b..c0f9880 100644
--- a/internal/slice/slice.go
+++ b/internal/slice/slice.go
@@ -27,6 +27,7 @@ along with this program.  If not, see http://www.gnu.org/licenses/.
 package slice
 
 import (
+	"math/rand"
 	"net/http"
 	"strings"
 )
@@ -93,3 +94,15 @@ func CheckCookies(input string) []*http.Cookie {
 
 	return result
 }
+
+// RandSeq produces a random sequence of letters.
+func RandSeq(n int) string {
+	b := make([]rune, n)
+	letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+
+	return string(b)
+}
diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go
index 21c4c38..6bde8cc 100644
--- a/pkg/crawler/colly.go
+++ b/pkg/crawler/colly.go
@@ -31,8 +31,10 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"math/rand"
 	"net/http"
 	"os"
+	"os/signal"
 	"regexp"
 	"strings"
 	"time"
@@ -290,6 +292,24 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
 		log.Println(err)
 	}
 
+	// Setup graceful exits
+	chanC := make(chan os.Signal, 1)
+	lettersNum := 23
+
+	signal.Notify(chanC, os.Interrupt)
+	rand.Seed(time.Now().UnixNano())
+
+	go func() {
+		for range chanC {
+			if !plain {
+				fmt.Fprint(os.Stdout, "\r")
+				fmt.Println("CTRL+C pressed: Exiting")
+			}
+
+			c.AllowedDomains = []string{sliceUtils.RandSeq(lettersNum)}
+		}
+	}()
+
 	c.Wait()
 
 	if html != "" {