diff --git a/README.md b/README.md index 79fe072..7afdb94 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,9 @@ Usage of cariddi: -ot string Write the output into a TXT file. -plain - Print only the results. + Print only the results. + -rua + Use a random browser user agent on every request. -s Hunt for secrets. -sf string Use an external file (txt, one per line) to use custom regexes for secrets hunting. diff --git a/crawler/colly.go b/crawler/colly.go index 558f95f..08298b6 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -35,17 +35,20 @@ import ( "github.com/edoardottt/cariddi/scanner" "github.com/edoardottt/cariddi/utils" "github.com/gocolly/colly" + "github.com/gocolly/colly/extensions" ) //Crawler it's the actual crawler core -func Crawler(target string, txt string, html string, delayTime int, concurrency int, ignore string, ignoreTxt string, - cache bool, timeout int, intensive bool, secrets bool, secretsFile []string, plain bool, endpoints bool, endpointsFile []string, +func Crawler(target string, txt string, html string, delayTime int, concurrency int, ignore string, + ignoreTxt string, cache bool, timeout int, intensive bool, rua bool, secrets bool, secretsFile []string, + plain bool, endpoints bool, endpointsFile []string, fileType int) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched, []scanner.FileTypeMatched) { // This is to avoid to insert into the crawler target regular // expression directories passed as input. var targetTemp string var protocolTemp string + // if there isn't a scheme use http. if !utils.HasScheme(target) { protocolTemp = "http" @@ -54,6 +57,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency protocolTemp = utils.GetProtocol(target) targetTemp = utils.GetHost(target) } + if intensive { targetTemp = utils.GetRootHost(targetTemp) } @@ -114,6 +118,12 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency c.URLFilters = []*regexp.Regexp{regexp.MustCompile(targetRegex)} } + + // Use a Random User Agent for each request if needed + if rua { + extensions.RandomUserAgent(c) + } + // On every a element which has href attribute call callback c.OnHTML("a[href]", func(e *colly.HTMLElement) { link := e.Attr("href") diff --git a/input/flags.go b/input/flags.go index 3f89f70..c44c294 100644 --- a/input/flags.go +++ b/input/flags.go @@ -42,6 +42,7 @@ type Input struct { Cache bool Timeout int Intensive bool + Rua bool Secrets bool SecretsFile string Endpoints bool @@ -66,6 +67,7 @@ func ScanFlag() Input { cachePtr := flag.Bool("cache", false, "Use the .cariddi_cache folder as cache.") timeoutPtr := flag.Int("t", 10, "Set timeout for the requests.") intensivePtr := flag.Bool("intensive", false, "Crawl searching for resources matching 2nd level domain.") + ruaPtr := flag.Bool("rua", false, "Use a random browser user agent on every request.") secretsPtr := flag.Bool("s", false, "Hunt for secrets.") secretsFilePtr := flag.String("sf", "", "Use an external file (txt, one per line) to use custom regexes for secrets hunting.") @@ -91,6 +93,7 @@ func ScanFlag() Input { *cachePtr, *timeoutPtr, *intensivePtr, + *ruaPtr, *secretsPtr, *secretsFilePtr, *endpointsPtr, diff --git a/main.go b/main.go index c1dfbef..4cb95c0 100644 --- a/main.go +++ b/main.go @@ -90,9 +90,10 @@ func main() { for _, inp := range targets { - results, secrets, endpoints, extensions := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay, flags.Concurrency, flags.Ignore, - flags.IgnoreTxt, flags.Cache, flags.Timeout, flags.Intensive, flags.Secrets, secretsFileSlice, flags.Plain, - flags.Endpoints, endpointsFileSlice, flags.Extensions) + results, secrets, endpoints, extensions := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay, + flags.Concurrency, flags.Ignore, flags.IgnoreTxt, flags.Cache, flags.Timeout, flags.Intensive, + flags.Rua, flags.Secrets, secretsFileSlice, flags.Plain, flags.Endpoints, endpointsFileSlice, + flags.Extensions) finalResults = append(finalResults, results...) finalSecret = append(finalSecret, secrets...) diff --git a/output/examples.go b/output/examples.go index be9746d..0f0a471 100644 --- a/output/examples.go +++ b/output/examples.go @@ -63,5 +63,7 @@ func PrintExamples() { cat urls | cariddi -t 5 (Set the timeout for the requests) - cat urls | cariddi -intensive (Crawl searching for resources matching 2nd level domain)`) + cat urls | cariddi -intensive (Crawl searching for resources matching 2nd level domain) + + cat urls | cariddi -rua (Use a random browser user agent on every request)`) } diff --git a/output/help.go b/output/help.go index 2ef9534..b089cc7 100644 --- a/output/help.go +++ b/output/help.go @@ -55,6 +55,8 @@ func PrintHelp() { Write the output into a TXT file. -plain Print only the results. + -rua + Use a random browser user agent on every request. -s Hunt for secrets. -sf string Use an external file (txt, one per line) to use custom regexes for secrets hunting.