From 849405ffdde67d810dd677fb0ebc19d51442fe89 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 6 Jun 2023 08:37:11 +0200 Subject: [PATCH 01/14] update golangcilint --- .golangci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 6c36914..cfc0617 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -5,7 +5,6 @@ linters: enable: - asciicheck - bodyclose - - depguard - dogsled - dupl - errcheck From 8fffb90eed1760c21d8a0674b7560c69453479e9 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 6 Jun 2023 08:37:50 +0200 Subject: [PATCH 02/14] update main --- cmd/cariddi/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/cariddi/main.go b/cmd/cariddi/main.go index 23f9c31..649995b 100644 --- a/cmd/cariddi/main.go +++ b/cmd/cariddi/main.go @@ -86,6 +86,7 @@ func main() { InfoFlag: flags.Info, Debug: flags.Debug, UserAgent: flags.UserAgent, + StoreResp: flags.StoreResp, } // Read the targets from standard input. @@ -126,6 +127,10 @@ func main() { output.HeaderHTML("Results", ResultHTML) } + if config.StoreResp { + fileUtils.CreateOutputFile("index", "responses", "txt") + } + // Read headers if needed if flags.HeadersFile != "" || flags.Headers != "" { var headersInput string From 3b9baf38ad2f255fc8c083030a78029e621dc1c4 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 6 Jun 2023 08:38:12 +0200 Subject: [PATCH 03/14] add CreateHostOutputFolder --- internal/file/file.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/file/file.go b/internal/file/file.go index d2e0b4d..bd3b2fc 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -33,6 +33,7 @@ import ( "log" "net/http" "os" + "path/filepath" "strings" ) @@ -52,11 +53,23 @@ func CreateOutputFolder() { } } +// CreateHostOutputFolder creates the host output folder +// for the HTTP responses. +// If it fails exits with an error message. +func CreateHostOutputFolder(host string) { + // Create a folder/directory at a full qualified path + err := os.MkdirAll(filepath.Join("output-cariddi", host), Permission0755) + if err != nil { + fmt.Println("Can't create host output folder.") + os.Exit(1) + } +} + // CreateOutputFile takes a target (of the attack), a subcommand // (PORT-DNS-DIR-SUBDOMAIN-REPORT) and a format (json-html-txt). // It creates the output folder if needed, then checks if the output file -// already exists, if yes asks the user if scilla has to overwrite it; -// if no scilla creates it. +// already exists, if yes asks the user if cariddi has to overwrite it; +// if no cariddi creates it. // Whenever an instruction fails, it exits with an error message. func CreateOutputFile(target string, subcommand string, format string) string { target = ReplaceBadCharacterOutput(target) From 85dae448f885a526b4a6f1215b917e7500096078 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 6 Jun 2023 08:38:30 +0200 Subject: [PATCH 04/14] add -sr option --- pkg/input/flags.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/input/flags.go b/pkg/input/flags.go index c3fcfb0..be2fdca 100644 --- a/pkg/input/flags.go +++ b/pkg/input/flags.go @@ -65,6 +65,7 @@ type Input struct { Info bool Debug bool UserAgent string + StoreResp bool } // ScanFlag defines all the options taken @@ -111,6 +112,8 @@ func ScanFlag() Input { userAgentPtr := flag.String("ua", "", "Use a custom User Agent.") + storeRespPtr := flag.Bool("sr", false, "Store HTTP responses.") + flag.Parse() result := Input{ @@ -141,6 +144,7 @@ func ScanFlag() Input { *infoPtr, *debugPtr, *userAgentPtr, + *storeRespPtr, } return result From 457df74a6cb72112ffedf612016da7f636515a16 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 6 Jun 2023 08:38:46 +0200 Subject: [PATCH 05/14] add -sr option --- pkg/output/examples.go | 16 ++++++++-------- pkg/output/help.go | 4 +++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/output/examples.go b/pkg/output/examples.go index 279013e..0adb87b 100644 --- a/pkg/output/examples.go +++ b/pkg/output/examples.go @@ -72,19 +72,19 @@ func PrintExamples() { cat urls | cariddi -proxy http://127.0.0.1:8080 (Set a Proxy to be used (http and socks5 supported)) - cat urls | cariddi -headers "Cookie: auth=admin;type=2;; X-Custom: customHeader" + cat urls | cariddi -headers "Cookie: auth=admin;type=2;; X-Custom: customHeader (Use custom headers)" - cat urls | cariddi -headersfile headers.txt + cat urls | cariddi -headersfile headers.txt (Read from an external file custom headers) - cat urls | cariddi -err + cat urls | cariddi -err (Hunt for errors) - cat urls | cariddi -info + cat urls | cariddi -info (Hunt for useful information) - cat urls | cariddi -debug + cat urls | cariddi -debug (Print debug information) - cat urls | cariddi -ua "Custom User Agent" + cat urls | cariddi -ua "Custom User Agent" (Use a custom User Agent) - cat urls | cariddi -json + cat urls | cariddi -json (Print the output as JSON) - cat urls | cariddi -json | jq .`) + cat urls | cariddi -sr (Store HTTP responses)`) } diff --git a/pkg/output/help.go b/pkg/output/help.go index 80b0a5b..961d6d2 100644 --- a/pkg/output/help.go +++ b/pkg/output/help.go @@ -59,7 +59,7 @@ func PrintHelp() { -i string Ignore the URL containing at least one of the elements of this array. -info - Hunt for useful informations in websites. + Hunt for useful information in websites. -intensive Crawl searching for resources matching 2nd level domain. -it string @@ -77,6 +77,8 @@ func PrintHelp() { -s Hunt for secrets. -sf string Use an external file (txt, one per line) to use custom regexes for secrets hunting. + -sr + Store HTTP responses. -t int Set timeout for the requests. (default 10) -ua From 4a056fc3ab880bff3719a7bcdef242ee3b76e219 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 6 Jun 2023 08:39:08 +0200 Subject: [PATCH 06/14] add -sr option --- pkg/output/output.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/output/output.go b/pkg/output/output.go index c849bec..43fceee 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -36,6 +36,10 @@ import ( "github.com/edoardottt/cariddi/pkg/scanner" ) +const ( + CariddiOutputFolder = "output-cariddi" +) + // PrintSimpleOutput prints line by line. func PrintSimpleOutput(out []string) { for _, elem := range out { @@ -48,7 +52,7 @@ func PrintSimpleOutput(out []string) { func TxtOutput(flags input.Input, finalResults []string, finalSecret []scanner.SecretMatched, finalEndpoints []scanner.EndpointMatched, finalExtensions []scanner.FileTypeMatched, finalErrors []scanner.ErrorMatched, finalInfos []scanner.InfoMatched) { - exists, err := fileUtils.ElementExists("output-cariddi") + exists, err := fileUtils.ElementExists(CariddiOutputFolder) if err != nil { fmt.Println("Error while creating the output directory.") os.Exit(1) @@ -120,7 +124,7 @@ func TxtOutput(flags input.Input, finalResults []string, finalSecret []scanner.S func HTMLOutput(flags input.Input, resultFilename string, finalResults []string, finalSecret []scanner.SecretMatched, finalEndpoints []scanner.EndpointMatched, finalExtensions []scanner.FileTypeMatched, finalErrors []scanner.ErrorMatched, finalInfos []scanner.InfoMatched) { - exists, err := fileUtils.ElementExists("output-cariddi") + exists, err := fileUtils.ElementExists(CariddiOutputFolder) if err != nil { fmt.Println("Error while creating the output directory.") From 3aa80149bbfd0fe4bcbd7c509d018c899c945116 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 6 Jun 2023 08:39:17 +0200 Subject: [PATCH 07/14] add -sr option --- pkg/crawler/colly.go | 73 ++++++++------------------------------------ 1 file changed, 13 insertions(+), 60 deletions(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index a21b920..ae4d3f7 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -48,59 +48,6 @@ import ( "github.com/gocolly/colly/extensions" ) -type Results struct { - URLs []string - Secrets []scanner.SecretMatched - Endpoints []scanner.EndpointMatched - Extensions []scanner.FileTypeMatched - Errors []scanner.ErrorMatched - Infos []scanner.InfoMatched -} - -type Scan struct { - // Flags - Cache bool - Debug bool - EndpointsFlag bool - ErrorsFlag bool - InfoFlag bool - Intensive bool - Plain bool - Rua bool - SecretsFlag bool - Ignore string - IgnoreTxt string - JSON bool - HTML string - Proxy string - Target string - Txt string - UserAgent string - FileType int - Headers map[string]string - - // Settings - Concurrency int - Delay int - Timeout int - - // Storage - SecretsSlice []string - EndpointsSlice []string -} - -type Event struct { - ProtocolTemp string - TargetTemp string - Target string - Intensive bool - Ignore bool - Debug bool - JSON bool - IgnoreSlice []string - URLs *[]string -} - // New it's the actual crawler engine. // It controls all the behaviours of a scan // (event handlers, secrets, errors, extensions and endpoints scanning). @@ -182,6 +129,17 @@ func New(scan *Scan) *Results { } c.OnResponse(func(r *colly.Response) { + if !scan.JSON { + fmt.Println(r.Request.URL) + } + + if scan.StoreResp { + err := output.StoreHTTPResponse(r) + if err != nil { + log.Println(err) + } + } + minBodyLentgh := 10 lengthOk := len(string(r.Body)) > minBodyLentgh secrets := []scanner.SecretMatched{} @@ -231,10 +189,12 @@ func New(scan *Scan) *Results { infos = append(infos, infosSlice...) } } + if scan.JSON { jsonOutput, err := output.GetJSONString( r, secrets, parameters, filetype, errors, infos, ) + if err == nil { fmt.Println(string(jsonOutput)) } else { @@ -374,13 +334,6 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int, // registerHTMLEvents registers the associated functions for each // HTML event triggering an action. func registerHTMLEvents(c *colly.Collector, event *Event) { - // On every request that Colly is making, print the URL it's currently visiting - c.OnRequest(func(e *colly.Request) { - if !event.JSON { - fmt.Println(e.URL.String()) - } - }) - // On every a element which has href attribute call callback c.OnHTML("a[href]", func(e *colly.HTMLElement) { link := e.Attr("href") From d3c11781c9e9aa8786f9e1c491478002a7d91d06 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 6 Jun 2023 08:39:35 +0200 Subject: [PATCH 08/14] add responses --- pkg/crawler/options.go | 83 ++++++++++++++++++ pkg/output/responses.go | 185 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 pkg/crawler/options.go create mode 100644 pkg/output/responses.go diff --git a/pkg/crawler/options.go b/pkg/crawler/options.go new file mode 100644 index 0000000..73ab694 --- /dev/null +++ b/pkg/crawler/options.go @@ -0,0 +1,83 @@ +/* +========== +Cariddi +========== + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see http://www.gnu.org/licenses/. + + @Repository: https://github.com/edoardottt/cariddi + + @Author: edoardottt, https://www.edoardoottavianelli.it + + @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE + +*/ + +package crawler + +import "github.com/edoardottt/cariddi/pkg/scanner" + +type Results struct { + URLs []string + Secrets []scanner.SecretMatched + Endpoints []scanner.EndpointMatched + Extensions []scanner.FileTypeMatched + Errors []scanner.ErrorMatched + Infos []scanner.InfoMatched +} + +type Scan struct { + // Flags + Cache bool + Debug bool + EndpointsFlag bool + ErrorsFlag bool + InfoFlag bool + Intensive bool + Plain bool + Rua bool + SecretsFlag bool + Ignore string + IgnoreTxt string + JSON bool + HTML string + Proxy string + Target string + Txt string + UserAgent string + FileType int + Headers map[string]string + StoreResp bool + + // Settings + Concurrency int + Delay int + Timeout int + + // Storage + SecretsSlice []string + EndpointsSlice []string +} + +type Event struct { + ProtocolTemp string + TargetTemp string + Target string + Intensive bool + Ignore bool + Debug bool + JSON bool + IgnoreSlice []string + URLs *[]string +} diff --git a/pkg/output/responses.go b/pkg/output/responses.go new file mode 100644 index 0000000..d983e28 --- /dev/null +++ b/pkg/output/responses.go @@ -0,0 +1,185 @@ +/* +========== +Cariddi +========== + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see http://www.gnu.org/licenses/. + + @Repository: https://github.com/edoardottt/cariddi + + @Author: edoardottt, https://www.edoardoottavianelli.it + + @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE + +*/ + +package output + +import ( + "bytes" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/gocolly/colly" + + fileUtils "github.com/edoardottt/cariddi/internal/file" +) + +const ( + index = "index.responses.txt" +) + +var ( + ErrHTTPResp = errors.New("cannot store HTTP response") +) + +func getResponseHash(url string) string { + hash := sha1.Sum([]byte(url)) + return hex.EncodeToString(hash[:]) +} + +// FormatResponse formats an HTTP response ready to be written in a file. +func FormatResponse(resp *colly.Response) ([]byte, error) { + builder := &bytes.Buffer{} + + builder.WriteString(resp.Request.URL.String()) + builder.WriteString("\n\n\n") + + builder.WriteString(resp.Request.Method) + builder.WriteString(" ") + + path := resp.Request.URL.Path + if resp.Request.URL.Fragment != "" { + path = path + "#" + resp.Request.URL.Fragment + } + + builder.WriteString(path) + builder.WriteString(" ") + builder.WriteString("HTTP/1.1") + builder.WriteString("\n") + builder.WriteString("Host: " + resp.Request.URL.Host) + builder.WriteRune('\n') + + for k, v := range *resp.Request.Headers { + builder.WriteString(k + ": " + strings.Join(v, "; ") + "\n") + } + + if resp.Request.Body != nil { + bodyResp, _ := io.ReadAll(resp.Request.Body) + if string(bodyResp) != "" { + builder.WriteString("\n") + builder.WriteString(string(bodyResp)) + } + } + + builder.WriteString("\n\n") + builder.WriteString("HTTP/1.1") + builder.WriteString(" ") + builder.WriteString(fmt.Sprint(resp.StatusCode)) + builder.WriteString("\n") + + for k, v := range *resp.Headers { + builder.WriteString(k + ": " + strings.Join(v, "; ") + "\n") + } + + builder.WriteString("\n") + + body, _ := io.ReadAll(bytes.NewReader(resp.Body)) + + builder.WriteString(string(body)) + + return builder.Bytes(), nil +} + +func getResponseFileName(folder, url string) string { + file := getResponseHash(url) + ".txt" + return filepath.Join(folder, file) +} + +// UpdateIndex updates the index file with the +// correct information linking to HTTP responses files. +// If it fails returns an error. +func UpdateIndex(resp *colly.Response) error { + index, err := os.OpenFile(filepath.Join(CariddiOutputFolder, index), os.O_APPEND|os.O_WRONLY, fileUtils.Permission0644) + if err != nil { + return err + } + + defer index.Close() + + builder := &bytes.Buffer{} + + builder.WriteString(getResponseFileName(filepath.Join(CariddiOutputFolder, resp.Request.URL.Host), + resp.Request.URL.String())) + builder.WriteRune(' ') + builder.WriteString(resp.Request.URL.String()) + builder.WriteRune(' ') + builder.WriteString("(" + fmt.Sprint(resp.StatusCode) + ")") + builder.WriteRune('\n') + + if _, writeErr := index.Write(builder.Bytes()); writeErr != nil { + return fmt.Errorf("%w %s", err, "could not update index") + } + + return nil +} + +// WriteHTTPResponse creates an HTTP response output file and +// writes the HTTP response inside it. +// If it fails returns an error. +func WriteHTTPResponse(inputURL *url.URL, response []byte) error { + file := getResponseFileName(filepath.Join(CariddiOutputFolder, inputURL.Host), inputURL.String()) + + outFile, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY, fileUtils.Permission0644) + if err != nil { + return err + } + + if _, writeErr := outFile.Write(response); writeErr != nil { + return ErrHTTPResp + } + + return nil +} + +// StoreHTTPResponse stores an HTTP response in a file. +// If it fails returns an error. +func StoreHTTPResponse(r *colly.Response) error { + fileUtils.CreateHostOutputFolder(r.Request.URL.Host) + + err := UpdateIndex(r) + if err != nil { + log.Println(err) + } + + response, err := FormatResponse(r) + if err != nil { + log.Println(err) + } + + err = WriteHTTPResponse(r.Request.URL, response) + if err != nil { + log.Println(err) + } + + return nil +} From 2a8b61652ec9629675fa01d1cf23b32638b53b91 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 6 Jun 2023 08:42:05 +0200 Subject: [PATCH 09/14] update --- README.md | 131 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 68 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index e487455..fbfcc8b 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ License

-Preview :bar_chart: +Preview :bar_chart ---------- @@ -69,11 +69,13 @@ Installation 📡 ---------- ### Using Snap + ```bash sudo snap install cariddi ``` ### Using Go + ```bash go install -v github.com/edoardottt/cariddi/cmd/cariddi@latest ``` @@ -97,7 +99,7 @@ You need [Go](https://golang.org/). - `git clone https://github.com/edoardottt/cariddi.git` - `cd cariddi` - `go get ./...` - - `.\make.bat windows` (to install) + - `.\make.bat windows` (to install) - `.\make.bat unwindows` (to uninstall) Get Started 🎉 @@ -108,99 +110,101 @@ Get Started 🎉 ``` Usage of cariddi: -c int - Concurrency level. (default 20) + Concurrency level. (default 20) -cache - Use the .cariddi_cache folder as cache. + Use the .cariddi_cache folder as cache. -d int - Delay between a page crawled and another. + Delay between a page crawled and another. -debug - Print debug information while crawling. - -e Hunt for juicy endpoints. + Print debug information while crawling. + -e Hunt for juicy endpoints. -ef string - Use an external file (txt, one per line) to use custom parameters for endpoints hunting. + Use an external file (txt, one per line) to use custom parameters for endpoints hunting. -err - Hunt for errors in websites. + Hunt for errors in websites. -examples - Print the examples. + Print the examples. -ext int - Hunt for juicy file extensions. Integer from 1(juicy) to 7(not juicy). - -h Print the help. + Hunt for juicy file extensions. Integer from 1(juicy) to 7(not juicy). + -h Print the help. -headers string - Use custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2". + Use custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2". -headersfile string - Read from an external file custom headers (same format of headers flag). + Read from an external file custom headers (same format of headers flag). -json - Print the output as JSON in stdout. + Print the output as JSON in stdout. -i string - Ignore the URL containing at least one of the elements of this array. + Ignore the URL containing at least one of the elements of this array. -info - Hunt for useful informations in websites. + Hunt for useful informations in websites. -intensive - Crawl searching for resources matching 2nd level domain. + Crawl searching for resources matching 2nd level domain. -it string - Ignore the URL containing at least one of the lines of this file. + Ignore the URL containing at least one of the lines of this file. -oh string - Write the output into an HTML file. + Write the output into an HTML file. -ot string - Write the output into a TXT file. + Write the output into a TXT file. -plain - Print only the results. + Print only the results. -proxy string - Set a Proxy to be used (http and socks5 supported). + Set a Proxy to be used (http and socks5 supported). -rua - Use a random browser user agent on every request. - -s Hunt for secrets. + Use a random browser user agent on every request. + -s Hunt for secrets. -sf string - Use an external file (txt, one per line) to use custom regexes for secrets hunting. + Use an external file (txt, one per line) to use custom regexes for secrets hunting. + -sr + Store HTTP responses. -t int - Set timeout for the requests. (default 10) + Set timeout for the requests. (default 10) -ua string - Use a custom User Agent. + Use a custom User Agent. -version - Print the version. + Print the version. ``` - Examples 💡 ---------- - - `cariddi -version` (Print the version) - - `cariddi -h` (Print the help) - - `cariddi -examples` (Print the examples) - - `cat urls | cariddi -s` (Hunt for secrets) - - `cat urls | cariddi -d 2` (2 seconds between a page crawled and another) - - `cat urls | cariddi -c 200` (Set the concurrency level to 200) - - `cat urls | cariddi -e` (Hunt for juicy endpoints) - - `cat urls | cariddi -plain` (Print only results) - - `cat urls | cariddi -ot target_name` (Results in txt file) - - `cat urls | cariddi -oh target_name` (Results in html file) - - `cat urls | cariddi -ext 2` (Hunt for juicy (level 2 out of 7) files) - - `cat urls | cariddi -e -ef endpoints_file` (Hunt for custom endpoints) - - `cat urls | cariddi -s -sf secrets_file` (Hunt for custom secrets) - - `cat urls | cariddi -i forum,blog,community,open` (Ignore urls containing these words) - - `cat urls | cariddi -it ignore_file` (Ignore urls containing at least one line in the input file) - - `cat urls | cariddi -cache` (Use the .cariddi_cache folder as cache) - - `cat urls | cariddi -t 5` (Set the timeout for the requests) - - `cat urls | cariddi -intensive` (Crawl searching also subdomains, same as `*.target.com`) - - `cat urls | cariddi -rua` (Use a random browser user agent on every request) - - `cat urls | cariddi -proxy http://127.0.0.1:8080` (Set a Proxy, http and socks5 supported) - - `cat urls | cariddi -headers "Cookie: auth=admin;type=2;; X-Custom: customHeader"` - - `cat urls | cariddi -headersfile headers.txt` (Read from an external file custom headers) - - `cat urls | cariddi -err` (Hunt for errors in websites) - - `cat urls | cariddi -info` (Hunt for useful informations in websites) - - `cat urls | cariddi -debug` (Print debug information while crawling) - - `cat urls | cariddi -ua "Custom User Agent"` (Use a custom User Agent) - - `cat urls | cariddi -json` (Print the output as JSON in stdout) - - `cat urls | cariddi -json | jq .` (Pipe the JSON output into jq) +- `cariddi -version` (Print the version) +- `cariddi -h` (Print the help) +- `cariddi -examples` (Print the examples) +- `cat urls | cariddi -s` (Hunt for secrets) +- `cat urls | cariddi -d 2` (2 seconds between a page crawled and another) +- `cat urls | cariddi -c 200` (Set the concurrency level to 200) +- `cat urls | cariddi -e` (Hunt for juicy endpoints) +- `cat urls | cariddi -plain` (Print only results) +- `cat urls | cariddi -ot target_name` (Results in txt file) +- `cat urls | cariddi -oh target_name` (Results in html file) +- `cat urls | cariddi -ext 2` (Hunt for juicy (level 2 out of 7) files) +- `cat urls | cariddi -e -ef endpoints_file` (Hunt for custom endpoints) +- `cat urls | cariddi -s -sf secrets_file` (Hunt for custom secrets) +- `cat urls | cariddi -i forum,blog,community,open` (Ignore urls containing these words) +- `cat urls | cariddi -it ignore_file` (Ignore urls containing at least one line in the input file) +- `cat urls | cariddi -cache` (Use the .cariddi_cache folder as cache) +- `cat urls | cariddi -t 5` (Set the timeout for the requests) +- `cat urls | cariddi -intensive` (Crawl searching also subdomains, same as `*.target.com`) +- `cat urls | cariddi -rua` (Use a random browser user agent on every request) +- `cat urls | cariddi -proxy http://127.0.0.1:8080` (Set a Proxy, http and socks5 supported) +- `cat urls | cariddi -headers "Cookie: auth=admin;type=2;; X-Custom: customHeader"` +- `cat urls | cariddi -headersfile headers.txt` (Read from an external file custom headers) +- `cat urls | cariddi -err` (Hunt for errors in websites) +- `cat urls | cariddi -info` (Hunt for useful informations in websites) +- `cat urls | cariddi -debug` (Print debug information while crawling) +- `cat urls | cariddi -ua "Custom User Agent"` (Use a custom User Agent) +- `cat urls | cariddi -json` (Print the output as JSON in stdout) +- `cat urls | cariddi -sr` (Store HTTP responses) - - For Windows: - - use `powershell.exe -Command "cat urls | .\cariddi.exe"` inside the Command prompt - - or just `cat urls | cariddi.exe` using PowerShell +- For Windows: + - use `powershell.exe -Command "cat urls | .\cariddi.exe"` inside the Command prompt + - or just `cat urls | cariddi.exe` using PowerShell - - To integrate cariddi with Burpsuite [make sure to follow these steps](https://github.com/edoardottt/cariddi/wiki/BurpSuite-Integration). +- To integrate cariddi with Burpsuite [make sure to follow these steps](https://github.com/edoardottt/cariddi/wiki/BurpSuite-Integration). Changelog 📌 ------- + Detailed changes for each release are documented in the [release notes](https://github.com/edoardottt/cariddi/releases). Contributing 🛠 @@ -209,16 +213,17 @@ Contributing 🛠 Just open an [issue](https://github.com/edoardottt/cariddi/issues)/[pull request](https://github.com/edoardottt/cariddi/pulls). Before opening a pull request, download [golangci-lint](https://golangci-lint.run/usage/install/) and run + ```bash golangci-lint run ``` + If there aren't errors, go ahead :) **Help me building this!** Special thanks to: [go-colly](http://go-colly.org/), [zricethezav](https://github.com/zricethezav/gitleaks/blob/master/config/default.go), [projectdiscovery](https://github.com/projectdiscovery/nuclei-templates/tree/master/file/keys), [tomnomnom](https://github.com/tomnomnom/gf/tree/master/examples), [RegexPassive](https://github.com/hahwul/RegexPassive) and [all the contributors](https://github.com/edoardottt/cariddi/wiki/Contributors). - License 📝 ------- From 839ac6af2b1141314b00808ce6afee6e95d84f81 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 6 Jun 2023 08:42:47 +0200 Subject: [PATCH 10/14] update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbfcc8b..9f04995 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ License

-Preview :bar_chart +Preview 📊 ---------- From a4df7b52413ceeaff8201491a5eae75f6bef6013 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Fri, 9 Jun 2023 11:35:10 +0200 Subject: [PATCH 11/14] update func --- cmd/cariddi/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cariddi/main.go b/cmd/cariddi/main.go index 649995b..384b232 100644 --- a/cmd/cariddi/main.go +++ b/cmd/cariddi/main.go @@ -128,7 +128,7 @@ func main() { } if config.StoreResp { - fileUtils.CreateOutputFile("index", "responses", "txt") + fileUtils.CreateIndexOutputFile("index.responses.txt") } // Read headers if needed From e26214df51aeee4f7a4c6b81034434d7bd4538f5 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Fri, 9 Jun 2023 11:35:28 +0200 Subject: [PATCH 12/14] add create index file func --- internal/file/file.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/internal/file/file.go b/internal/file/file.go index bd3b2fc..19c01be 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -65,7 +65,7 @@ func CreateHostOutputFolder(host string) { } } -// CreateOutputFile takes a target (of the attack), a subcommand +// CreateOutputFile takes as input a target (of the attack), a subcommand // (PORT-DNS-DIR-SUBDOMAIN-REPORT) and a format (json-html-txt). // It creates the output folder if needed, then checks if the output file // already exists, if yes asks the user if cariddi has to overwrite it; @@ -76,9 +76,9 @@ func CreateOutputFile(target string, subcommand string, format string) string { var filename string if subcommand != "" { - filename = "output-cariddi" + "/" + target + "." + subcommand + "." + format + filename = filepath.Join("output-cariddi", target+"."+subcommand+"."+format) } else { - filename = "output-cariddi" + "/" + target + "." + format + filename = filepath.Join("output-cariddi", target+"."+format) } _, err := os.Stat(filename) @@ -114,6 +114,29 @@ func CreateOutputFile(target string, subcommand string, format string) string { return filename } +// CreateIndexOutputFile takes as input the name of the index file. +// It creates the output folder if needed, then checks if the index output file +// already exists, if no cariddi creates it. +// Whenever an instruction fails, it exits with an error message. +func CreateIndexOutputFile(filename string) { + _, err := os.Stat(filename) + + if os.IsNotExist(err) { + if _, err := os.Stat("output-cariddi/"); os.IsNotExist(err) { + CreateOutputFolder() + } + // If the file doesn't exist, create it. + filename = filepath.Join("output-cariddi", filename) + f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, Permission0644) + if err != nil { + fmt.Println("Can't create output file.") + os.Exit(1) + } + + f.Close() + } +} + // ReplaceBadCharacterOutput replaces forward-slashes // with dashes (to avoid problems with output folder). func ReplaceBadCharacterOutput(input string) string { From c90430b5ef8d54ad3f7c6da56f8c7b4c7e507c36 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Fri, 9 Jun 2023 11:35:39 +0200 Subject: [PATCH 13/14] fix sr --- pkg/output/responses.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/output/responses.go b/pkg/output/responses.go index d983e28..fc3decb 100644 --- a/pkg/output/responses.go +++ b/pkg/output/responses.go @@ -119,7 +119,9 @@ func getResponseFileName(folder, url string) string { // correct information linking to HTTP responses files. // If it fails returns an error. func UpdateIndex(resp *colly.Response) error { - index, err := os.OpenFile(filepath.Join(CariddiOutputFolder, index), os.O_APPEND|os.O_WRONLY, fileUtils.Permission0644) + index, err := os.OpenFile(filepath.Join(CariddiOutputFolder, index), + os.O_APPEND|os.O_WRONLY, + fileUtils.Permission0644) if err != nil { return err } From 36079c0ebc1486004b7801339ccf4f05a5a8755e Mon Sep 17 00:00:00 2001 From: edoardottt Date: Fri, 9 Jun 2023 11:36:47 +0200 Subject: [PATCH 14/14] v1.3.2 --- pkg/output/banner.go | 2 +- snapcraft.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/output/banner.go b/pkg/output/banner.go index ae5e4d0..ac29b3f 100644 --- a/pkg/output/banner.go +++ b/pkg/output/banner.go @@ -35,7 +35,7 @@ import ( // nolint: checknoglobals const ( - version = "v1.3.1" + version = "v1.3.2" banner = ` _ _ _ _ (_) | | | (_) ___ __ _ _ __ _ __| | __| |_ diff --git a/snapcraft.yaml b/snapcraft.yaml index 24ea194..44b51f0 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -2,7 +2,7 @@ name: cariddi summary: Fast web crawler and scanner description: | Take a list of domains, crawl urls and scan for endpoints, secrets, api keys, file extensions, tokens and more -version: 1.3.1 +version: 1.3.2 grade: stable base: core20