diff --git a/internal/slice/slice.go b/internal/slice/slice.go index e67314b..c0f9880 100644 --- a/internal/slice/slice.go +++ b/internal/slice/slice.go @@ -27,6 +27,7 @@ along with this program. If not, see http://www.gnu.org/licenses/. package slice import ( + "math/rand" "net/http" "strings" ) @@ -93,3 +94,15 @@ func CheckCookies(input string) []*http.Cookie { return result } + +// RandSeq produces a random sequence of letters. +func RandSeq(n int) string { + b := make([]rune, n) + letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + + return string(b) +} diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index f430db1..6bde8cc 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -31,8 +31,10 @@ import ( "errors" "fmt" "log" + "math/rand" "net/http" "os" + "os/signal" "regexp" "strings" "time" @@ -259,23 +261,26 @@ func New(target string, txt string, html string, delayTime int, concurrency int, // Start scraping on target path, err := urlUtils.GetPath(protocolTemp + "://" + target) if err == nil { + var ( + addPath string + absoluteURL string + ) + if path == "" { - err = c.Visit(protocolTemp + "://" + target + "/" + "robots.txt") - if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) { - log.Println(err) - } + addPath = "/" + } - err = c.Visit(protocolTemp + "://" + target + "/" + "sitemap.xml") - if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) { - log.Println(err) - } - } else if path == "/" { - err = c.Visit(protocolTemp + "://" + target + "robots.txt") + absoluteURL = protocolTemp + "://" + target + addPath + "robots.txt" + if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) { + err = c.Visit(absoluteURL) if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) { log.Println(err) } + } - err = c.Visit(protocolTemp + "://" + target + "sitemap.xml") + absoluteURL = protocolTemp + "://" + target + addPath + "sitemap.xml" + if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) { + err = c.Visit(absoluteURL) if err != nil && debug && !errors.Is(err, colly.ErrAlreadyVisited) { log.Println(err) } @@ -287,6 +292,24 @@ func New(target string, txt string, html string, delayTime int, concurrency int, log.Println(err) } + // Setup graceful exits + chanC := make(chan os.Signal, 1) + lettersNum := 23 + + signal.Notify(chanC, os.Interrupt) + rand.Seed(time.Now().UnixNano()) + + go func() { + for range chanC { + if !plain { + fmt.Fprint(os.Stdout, "\r") + fmt.Println("CTRL+C pressed: Exiting") + } + + c.AllowedDomains = []string{sliceUtils.RandSeq(lettersNum)} + } + }() + c.Wait() if html != "" { @@ -306,10 +329,6 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int, c.IgnoreRobotsTxt = true c.AllowURLRevisit = false - if userAgent != "" { - c.UserAgent = userAgent - } - err := c.Limit( &colly.LimitRule{ Parallelism: concurrency, @@ -339,6 +358,10 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int, c.UserAgent = GenerateRandomUserAgent() } + if userAgent != "" { + c.UserAgent = userAgent + } + // Use a Proxy if needed if proxy != "" { err := c.SetProxy(proxy)