update

crawler/colly.go

@@ -5,11 +5,12 @@ import (
 	"time"
 
 	"github.com/edoardottt/cariddi/input"
+	"github.com/edoardottt/cariddi/scanner"
 	"github.com/gocolly/colly"
 )
 
 //Crawler
-func Crawler(target string, delayTime int, concurrency int) []string {
+func Crawler(target string, delayTime int, concurrency int, secrets bool, secretsFile string, dataPost map[string]string) []string {
 
 	//clean target input
 	target = input.RemoveHeaders(target)
@@ -67,10 +68,12 @@ func Crawler(target string, delayTime int, concurrency int) []string {
 		//result = append(result, e.Request.AbsoluteURL(link))
 	})
 
-	// Before making a request print "Visiting ..."
-	// THEN AFTER TESTS COMMENT THIS.
 	c.OnRequest(func(r *colly.Request) {
 		//fmt.Println("Visiting", r.URL.String())
+		// HERE SCAN FOR SECRETS
+		if secrets {
+			huntSecrets(secretsFile, r.URL.String(), dataPost)
+		}
 		result = append(result, r.URL.String())
 	})
 
@@ -80,3 +83,37 @@ func Crawler(target string, delayTime int, concurrency int) []string {
 	c.Wait()
 	return result
 }
+
+//huntSecrets
+func huntSecrets(secretsFile string, target string, data map[string]string) []scanner.Secret {
+	if secretsFile == "" {
+		body := RetrieveBody(target, data)
+		secrets := SecretsMatch(body)
+		return secrets
+	}
+	return scanner.GetRegexes()
+}
+
+//RetrieveBody
+func RetrieveBody(target string, data map[string]string) string {
+	sb, err := GetRequest(target)
+	if err == nil && sb != "" {
+		return sb
+	}
+	sb, err = PostRequest(target, data)
+	if err == nil && sb != "" {
+		return sb
+	}
+	return ""
+}
+
+//SecretsMatch
+func SecretsMatch(body string) []scanner.Secret {
+	var secrets []scanner.Secret
+	for _, secret := range scanner.GetRegexes() {
+		if matched, err := regexp.Match(secret.Regex, []byte(body)); err == nil && matched {
+			secrets = append(secrets, secret)
+		}
+	}
+	return secrets
+}

input/check.go

@@ -0,0 +1,8 @@
+package input
+
+//CheckDataPost
+func CheckDataPost(input string) (map[string]string, error) {
+
+	// ===== TODO =======
+	return map[string]string{}, nil
+}

input/flags.go

@@ -12,6 +12,7 @@ type Input struct {
 	Examples    bool
 	Html        string
 	Txt         string
+	DataPost    string
 	Secrets     bool
 	SecretsFile string
 }
@@ -22,12 +23,13 @@ func ScanFlag() Input {
 
 	verbosePtr := flag.Bool("v", false, "Verbose mode.")
 	versionPtr := flag.Bool("version", false, "Print the version.")
-	delayPtr := flag.Int("delay", 0, "Delay between a page crawled and another.")
+	delayPtr := flag.Int("d", 0, "Delay between a page crawled and another.")
 	concurrencyPtr := flag.Int("c", 20, "Concurrency level (20 is default).")
-	helpPtr := flag.Bool("help", false, "Print the version.")
+	helpPtr := flag.Bool("h", false, "Print the version.")
 	examplesPtr := flag.Bool("examples", false, "Print the version.")
 	outputHtmlPtr := flag.String("oh", "", "Write the output into an HTML file.")
 	outputTxtPtr := flag.String("ot", "", "Write the output into a TXT file.")
+	dataPostPtr := flag.String("post", "", "Set the data to perform the POST requests.")
 	secretsPtr := flag.Bool("s", false, "Hunt for secrets.")
 	secretsFilePtr := flag.String("sf", "", "Use an external file (txt, one per line) to use custom regexes for secrets hunting.")
 
@@ -42,6 +44,7 @@ func ScanFlag() Input {
 		*examplesPtr,
 		*outputHtmlPtr,
 		*outputTxtPtr,
+		*dataPostPtr,
 		*secretsPtr,
 		*secretsFilePtr,
 	}

main.go

@@ -35,10 +35,13 @@ func main() {
 		os.Exit(0)
 	}
 
+	// ----------- TODO: check flags.dataPost --------------
+	data, _ := input.CheckDataPost(flags.DataPost)
+
 	var finalResult []string
 	for _, inp := range targets {
 		var result []string
-		result = crawler.Crawler(inp, flags.Delay, flags.Concurrency)
+		result = crawler.Crawler(inp, flags.Delay, flags.Concurrency, flags.Secrets, flags.SecretsFile, data)
 		finalResult = append(finalResult, result...)
 	}
 	output.PrintSimpleOutput(finalResult)