From ee0fed981a616adb56023cbb96afccfb25e207a8 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Wed, 13 Oct 2021 15:00:28 +0200 Subject: [PATCH 1/5] Add Bugsnag secrets detection --- scanner/secrets.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scanner/secrets.go b/scanner/secrets.go index a3ebf67..3abeade 100644 --- a/scanner/secrets.go +++ b/scanner/secrets.go @@ -308,6 +308,13 @@ func GetRegexes() []Secret { []string{}, "?", }, + { + "Bugsnag API Key", + "Bugsnag API Key", + "(?i)(bs|bugsnag)(.{0,20})?[0-9a-f]{32}", + []string{}, + "?", + }, { "S3 Bucket", "S3 Bucket", From 9968e6e587b2b17afb82a20c25719509fb4d9cc5 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 26 Oct 2021 21:49:00 +0200 Subject: [PATCH 2/5] Add HTTP reqs. function and comments --- main.go | 27 ++++++++++++++++++++------- utils/files.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 6610b5a..79cc553 100644 --- a/main.go +++ b/main.go @@ -33,39 +33,50 @@ import ( "github.com/edoardottt/cariddi/utils" ) -//main +//main function > func main() { + // Scan flags. flags := input.ScanFlag() + //Print version and exit. if flags.Version { output.Beautify() os.Exit(0) } + //Print help and exit. if flags.Help { output.PrintHelp() os.Exit(0) } + //Print examples and exit. if flags.Examples { output.PrintExamples() os.Exit(0) } + //If it's possible print the cariddi banner. if !flags.Plain { output.Beautify() } + //Read the targets from standard input. targets := input.ScanTargets() + //Check if there are errors in the flags definition. input.CheckFlags(flags) + //If it is needed, read custom endpoints definition + //from the specified file. var endpointsFileSlice []string if flags.EndpointsFile != "" { endpointsFileSlice = utils.ReadFile(flags.EndpointsFile) } + //If it is needed, read custom secrets definition + //from the specified file. var secretsFileSlice []string if flags.SecretsFile != "" { secretsFileSlice = utils.ReadFile(flags.SecretsFile) @@ -76,7 +87,7 @@ func main() { var finalEndpoints []scanner.EndpointMatched var finalExtensions []scanner.FileTypeMatched - // output files + //Create output files if needed (txt / html). var ResultTxt = "" if flags.Txt != "" { ResultTxt = utils.CreateOutputFile(flags.Txt, "results", "txt") @@ -88,6 +99,7 @@ func main() { output.HeaderHTML("Results", ResultHtml) } + //For each target generate a crawler and collect all the results. for _, inp := range targets { results, secrets, endpoints, extensions := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay, @@ -101,29 +113,30 @@ func main() { finalExtensions = append(finalExtensions, extensions...) } + //Remove duplicates from all the results. finalResults = utils.RemoveDuplicateValues(finalResults) finalSecret = scanner.RemoveDuplicateSecrets(finalSecret) finalEndpoints = scanner.RemovDuplicateEndpoints(finalEndpoints) finalExtensions = scanner.RemoveDuplicateExtensions(finalExtensions) - // IF TXT OUTPUT + // IF TXT OUTPUT > if flags.Txt != "" { output.TxtOutput(flags, finalResults, finalSecret, finalEndpoints, finalExtensions) } - // IF HTML OUTPUT + // IF HTML OUTPUT > if flags.Html != "" { output.HtmlOutput(flags, ResultHtml, finalResults, finalSecret, finalEndpoints, finalExtensions) } - // if needed print secrets + //If needed print secrets. if !flags.Plain && len(finalSecret) != 0 { for _, elem := range finalSecret { output.EncapsulateCustomGreen(elem.Secret.Name, elem.Match+" in "+elem.Url) } } - // if needed print endpoints + //If needed print endpoints. if !flags.Plain && len(finalEndpoints) != 0 { for _, elem := range finalEndpoints { for _, parameter := range elem.Parameters { @@ -140,7 +153,7 @@ func main() { } } - // if needed print extensions + //If needed print extensions. if !flags.Plain && len(finalExtensions) != 0 { for _, elem := range finalExtensions { output.EncapsulateCustomGreen(elem.Filetype.Extension, elem.Url+" matched!") diff --git a/utils/files.go b/utils/files.go index c65f138..aa3c407 100644 --- a/utils/files.go +++ b/utils/files.go @@ -26,7 +26,9 @@ package utils import ( "bufio" "fmt" + "io/ioutil" "log" + "net/http" "os" "strings" ) @@ -115,3 +117,41 @@ func ElementExists(path string) (bool, error) { } return false, err } + +//ReadHTTPRequestFromFile reads from a file an HTTP +//request and returns a *http.Request object +func ReadHTTPFromFile(inputFile string) (*http.Request, error) { + f, err := os.Open(inputFile) + if err != nil { + fmt.Println("Cannot open input file.") + os.Exit(1) + } + defer f.Close() + + buf := bufio.NewReader(f) + req, err := http.ReadRequest(buf) + if err != nil { + fmt.Println("Cannot read request from input file.") + os.Exit(1) + } + return req, nil + +} + +//ReadEntireFile returns the content of the inputted file. +func ReadEntireFile(inputFile string) []byte { + file, err := os.Open(inputFile) + if err != nil { + fmt.Println("Cannot open input file.") + os.Exit(1) + } + defer func() { + if err = file.Close(); err != nil { + fmt.Println("Cannot close input file.") + os.Exit(1) + } + }() + + b, err := ioutil.ReadAll(file) + return b +} From 0f3329ed522aca7dff8adff92a596cccb595a712 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sat, 30 Oct 2021 17:14:37 +0200 Subject: [PATCH 3/5] Fix results output on stdout --- crawler/colly.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crawler/colly.go b/crawler/colly.go index 0349a45..2fd6440 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -226,9 +226,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency c.OnResponse(func(r *colly.Response) { - if utils.SameDomain(protocolTemp+"://"+target, r.Request.URL.String()) { - fmt.Println(r.Request.URL.String()) - } + fmt.Println(r.Request.URL.String()) lengthOk := len(string(r.Body)) > 10 From 9b1d2d5b102732599982e57508b0b9d173239f79 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sat, 30 Oct 2021 17:32:50 +0200 Subject: [PATCH 4/5] Add default requests for robots and sitemap #26 --- crawler/colly.go | 10 ++++++++++ utils/urls.go | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/crawler/colly.go b/crawler/colly.go index 2fd6440..e5a5be0 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -257,6 +257,16 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency }) // Start scraping on target + path, err := utils.GetPath(protocolTemp + "://" + target) + if err == nil { + if path == "" { + c.Visit(protocolTemp + "://" + target + "/" + "robots.txt") + c.Visit(protocolTemp + "://" + target + "/" + "sitemap.xml") + } else if path == "/" { + c.Visit(protocolTemp + "://" + target + "robots.txt") + c.Visit(protocolTemp + "://" + target + "sitemap.xml") + } + } c.Visit(protocolTemp + "://" + target) c.Wait() if html != "" { diff --git a/utils/urls.go b/utils/urls.go index 6feb7eb..1f72f05 100644 --- a/utils/urls.go +++ b/utils/urls.go @@ -130,3 +130,12 @@ func SameDomain(url1 string, url2 string) bool { } return u1.Host == u2.Host } + +//GetPath returns the path of the input URL +func GetPath(input string) (string, error) { + u, err := url.Parse(input) + if err != nil { + return "", err + } + return u.Path, nil +} From 22a14bfbf68894d6e8bc227cd4381e343e52d617 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sat, 30 Oct 2021 17:36:53 +0200 Subject: [PATCH 5/5] v1.1.3 --- main.go | 2 +- output/beautify.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 79cc553..7d92323 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ /* ========== -Cariddi v1.1.2 +Cariddi v1.1.3 ========== This program is free software: you can redistribute it and/or modify diff --git a/output/beautify.go b/output/beautify.go index eec8b1a..e889ca5 100644 --- a/output/beautify.go +++ b/output/beautify.go @@ -35,7 +35,7 @@ func Beautify() { banner2 := " ___ __ _ _ __(_) __| | __| (_)\n" banner3 := " / __/ _` | '__| |/ _` |/ _` | |\n" banner4 := " | (_| (_| | | | | (_| | (_| | |\n" - banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.1.2\n" + banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.1.3\n" banner6 := "" banner7 := " > github.com/edoardottt/cariddi\n" banner8 := " > edoardoottavianelli.it\n"