Merge branch 'main' of github.com:edoardottt/cariddi

crawler/colly.go

@@ -41,8 +41,8 @@ import (
 //Crawler it's the actual crawler core
 func Crawler(target string, txt string, html string, delayTime int, concurrency int, ignore string,
 	ignoreTxt string, cache bool, timeout int, intensive bool, rua bool, proxy string, secrets bool,
-	secretsFile []string, plain bool, endpoints bool, endpointsFile []string,
-	fileType int) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched, []scanner.FileTypeMatched) {
+	secretsFile []string, plain bool, endpoints bool, endpointsFile []string, fileType int,
+	headers map[string]string) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched, []scanner.FileTypeMatched) {
 
 	// This is to avoid to insert into the crawler target regular
 	// expression directories passed as input.
@@ -224,6 +224,15 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 		}
 	})
 
+	//Add headers (if needed) on each request
+	if (len(headers)) > 0 {
+		c.OnRequest(func(r *colly.Request) {
+			for header, value := range headers {
+				r.Headers.Set(header, value)
+			}
+		})
+	}
+
 	c.OnResponse(func(r *colly.Response) {
 
 		fmt.Println(r.Request.URL.String())

input/check.go

@@ -38,7 +38,8 @@ func CheckDataPost(input string) (map[string]string, error) {
 	return map[string]string{}, nil
 }
 
-//CheckOutputFile >
+//CheckOutputFile checks if the string provided as input
+//is formatted in correct way.
 func CheckOutputFile(input string) bool {
 	invalid := []string{"\\", "/", "'", "\""}
 	for _, elem := range invalid {
@@ -49,7 +50,7 @@ func CheckOutputFile(input string) bool {
 	return true
 }
 
-//CheckFlags checks the flags inserted
+//CheckFlags checks the flags inputted
 func CheckFlags(flags Input) {
 	if flags.Txt != "" {
 		if !CheckOutputFile(flags.Txt) {
@@ -116,4 +117,12 @@ func CheckFlags(flags Input) {
 		fmt.Println("	- cat urls | cariddi -it ignore.txt")
 		os.Exit(1)
 	}
+
+	if flags.Headers != "" && flags.HeadersFile != "" {
+		fmt.Println("You should use only one among -headers and -headersfile.")
+		fmt.Println("Examples:")
+		fmt.Println("	- cat urls | cariddi -headers \"Cookie: auth=yes;;Client: type=2\"")
+		fmt.Println("	- cat urls | cariddi -headersfile headers.txt")
+		os.Exit(1)
+	}
 }

input/flags.go

@@ -49,6 +49,8 @@ type Input struct {
 	Endpoints     bool
 	EndpointsFile string
 	Extensions    int
+	Headers       string
+	HeadersFile   string
 }
 
 //ScanFlag defines all the switches taken
@@ -79,6 +81,9 @@ func ScanFlag() Input {
 
 	extensionsPtr := flag.Int("ext", 0, "Hunt for juicy file extensions. Integer from 1(juicy) to 7(not juicy).")
 
+	headersPtr := flag.String("headers", "", "Use custom headers for each request E.g. -headers \"Cookie: auth=yes;;Client: type=2\".")
+	headersFilePtr := flag.String("headersfile", "", "Read from an external file custom headers (same format of headers flag).")
+
 	flag.Parse()
 
 	result := Input{
@@ -102,6 +107,8 @@ func ScanFlag() Input {
 		*endpointsPtr,
 		*endpointsFilePtr,
 		*extensionsPtr,
+		*headersPtr,
+		*headersFilePtr,
 	}
 
 	return result

input/input.go

@@ -25,6 +25,7 @@ package input
 
 import (
 	"bufio"
+	"fmt"
 	"os"
 	"strings"
 
@@ -47,3 +48,32 @@ func ScanTargets() []string {
 	}
 	return utils.RemoveDuplicateValues(result)
 }
+
+//GetHeaders returns the headers provided as input
+func GetHeaders(input string) map[string]string {
+	result := make(map[string]string)
+	if input != "" {
+		if !strings.Contains(input, ":") {
+			fmt.Println("The headers provided don't contains the : separator.")
+			os.Exit(1)
+		}
+		headers := strings.Split(input, ";;")
+		for _, header := range headers {
+			var parts []string
+			if strings.Contains(header, ":") {
+				parts = strings.SplitN(header, ":", 2)
+			} else {
+				continue
+			}
+			result[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
+		}
+	} else {
+		fmt.Println("Headers or HeadersFile flag provided, but the content is empty.")
+		os.Exit(1)
+	}
+	if len(result) == 0 {
+		fmt.Println("Headers or HeadersFile flag provided, but the content is empty.")
+		os.Exit(1)
+	}
+	return result
+}

main.go

@@ -99,13 +99,25 @@ func main() {
 		output.HeaderHTML("Results", ResultHtml)
 	}
 
+	//Read headers if needed
+	var headers map[string]string
+	if flags.HeadersFile != "" || flags.Headers != "" {
+		var headersInput string
+		if flags.HeadersFile != "" {
+			headersInput = string(utils.ReadEntireFile(flags.HeadersFile))
+		} else {
+			headersInput = flags.Headers
+		}
+		headers = input.GetHeaders(headersInput)
+	}
+
 	//For each target generate a crawler and collect all the results.
 	for _, inp := range targets {
 
 		results, secrets, endpoints, extensions := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay,
 			flags.Concurrency, flags.Ignore, flags.IgnoreTxt, flags.Cache, flags.Timeout, flags.Intensive,
 			flags.Rua, flags.Proxy, flags.Secrets, secretsFileSlice, flags.Plain, flags.Endpoints, endpointsFileSlice,
-			flags.Extensions)
+			flags.Extensions, headers)
 
 		finalResults = append(finalResults, results...)
 		finalSecret = append(finalSecret, secrets...)

output/examples.go

@@ -67,5 +67,9 @@ func PrintExamples() {
 
 	cat urls | cariddi -rua (Use a random browser user agent on every request)
 
-	cat urls | cariddi -proxy http://127.0.0.1:8080 (Set a Proxy to be used (http and socks5 supported))`)
+	cat urls | cariddi -proxy http://127.0.0.1:8080 (Set a Proxy to be used (http and socks5 supported))
+	
+	cat urls | cariddi -headers "Cookie: auth=admin;type=2;; X-Custom: customHeader"
+	
+	cat urls | cariddi -headersfile headers.txt`)
 }

output/help.go

@@ -43,6 +43,10 @@ func PrintHelp() {
 	-ext int
 		Hunt for juicy file extensions. Integer from 1(juicy) to 7(not juicy).
 	-h	Print the help.
+	-headers string
+    	Use custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2".
+  	-headersfile string
+    	Read from an external file custom headers (same format of headers flag).
 	-i string
 		Ignore the URL containing at least one of the elements of this array.
 	-intensive