Add Random User Agent mode #14

README.md

@@ -128,7 +128,9 @@ Usage of cariddi:
   -ot string
     	Write the output into a TXT file.
   -plain
-    	Print only the results.
+      Print only the results.
+  -rua
+      Use a random browser user agent on every request.
   -s	Hunt for secrets.
   -sf string
     	Use an external file (txt, one per line) to use custom regexes for secrets hunting.

crawler/colly.go

@@ -35,17 +35,20 @@ import (
 	"github.com/edoardottt/cariddi/scanner"
 	"github.com/edoardottt/cariddi/utils"
 	"github.com/gocolly/colly"
+	"github.com/gocolly/colly/extensions"
 )
 
 //Crawler it's the actual crawler core
-func Crawler(target string, txt string, html string, delayTime int, concurrency int, ignore string, ignoreTxt string,
-	cache bool, timeout int, intensive bool, secrets bool, secretsFile []string, plain bool, endpoints bool, endpointsFile []string,
+func Crawler(target string, txt string, html string, delayTime int, concurrency int, ignore string,
+	ignoreTxt string, cache bool, timeout int, intensive bool, rua bool, secrets bool, secretsFile []string,
+	plain bool, endpoints bool, endpointsFile []string,
 	fileType int) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched, []scanner.FileTypeMatched) {
 
 	// This is to avoid to insert into the crawler target regular
 	// expression directories passed as input.
 	var targetTemp string
 	var protocolTemp string
+
 	// if there isn't a scheme use http.
 	if !utils.HasScheme(target) {
 		protocolTemp = "http"
@@ -54,6 +57,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 		protocolTemp = utils.GetProtocol(target)
 		targetTemp = utils.GetHost(target)
 	}
+
 	if intensive {
 		targetTemp = utils.GetRootHost(targetTemp)
 	}
@@ -114,6 +118,12 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 		c.URLFilters =
 			[]*regexp.Regexp{regexp.MustCompile(targetRegex)}
 	}
+
+	// Use a Random User Agent for each request if needed
+	if rua {
+		extensions.RandomUserAgent(c)
+	}
+
 	// On every a element which has href attribute call callback
 	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
 		link := e.Attr("href")

input/flags.go

@@ -42,6 +42,7 @@ type Input struct {
 	Cache         bool
 	Timeout       int
 	Intensive     bool
+	Rua           bool
 	Secrets       bool
 	SecretsFile   string
 	Endpoints     bool
@@ -66,6 +67,7 @@ func ScanFlag() Input {
 	cachePtr := flag.Bool("cache", false, "Use the .cariddi_cache folder as cache.")
 	timeoutPtr := flag.Int("t", 10, "Set timeout for the requests.")
 	intensivePtr := flag.Bool("intensive", false, "Crawl searching for resources matching 2nd level domain.")
+	ruaPtr := flag.Bool("rua", false, "Use a random browser user agent on every request.")
 
 	secretsPtr := flag.Bool("s", false, "Hunt for secrets.")
 	secretsFilePtr := flag.String("sf", "", "Use an external file (txt, one per line) to use custom regexes for secrets hunting.")
@@ -91,6 +93,7 @@ func ScanFlag() Input {
 		*cachePtr,
 		*timeoutPtr,
 		*intensivePtr,
+		*ruaPtr,
 		*secretsPtr,
 		*secretsFilePtr,
 		*endpointsPtr,

main.go

@@ -90,9 +90,10 @@ func main() {
 
 	for _, inp := range targets {
 
-		results, secrets, endpoints, extensions := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay, flags.Concurrency, flags.Ignore,
-			flags.IgnoreTxt, flags.Cache, flags.Timeout, flags.Intensive, flags.Secrets, secretsFileSlice, flags.Plain,
-			flags.Endpoints, endpointsFileSlice, flags.Extensions)
+		results, secrets, endpoints, extensions := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay,
+			flags.Concurrency, flags.Ignore, flags.IgnoreTxt, flags.Cache, flags.Timeout, flags.Intensive,
+			flags.Rua, flags.Secrets, secretsFileSlice, flags.Plain, flags.Endpoints, endpointsFileSlice,
+			flags.Extensions)
 
 		finalResults = append(finalResults, results...)
 		finalSecret = append(finalSecret, secrets...)

output/examples.go

@@ -63,5 +63,7 @@ func PrintExamples() {
 
 	cat urls | cariddi -t 5 (Set the timeout for the requests)
 
-	cat urls | cariddi -intensive (Crawl searching for resources matching 2nd level domain)`)
+	cat urls | cariddi -intensive (Crawl searching for resources matching 2nd level domain)
+
+	cat urls | cariddi -rua (Use a random browser user agent on every request)`)
 }

output/help.go

@@ -55,6 +55,8 @@ func PrintHelp() {
 		Write the output into a TXT file.
 	-plain
 		Print only the results.
+	-rua
+		Use a random browser user agent on every request.
 	-s	Hunt for secrets.
 	-sf string
 		Use an external file (txt, one per line) to use custom regexes for secrets hunting.