Merge pull request #27 from edoardottt/devel

Devel

crawler/colly.go

@@ -226,9 +226,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 
 	c.OnResponse(func(r *colly.Response) {
 
-		if utils.SameDomain(protocolTemp+"://"+target, r.Request.URL.String()) {
-			fmt.Println(r.Request.URL.String())
-		}
+		fmt.Println(r.Request.URL.String())
 
 		lengthOk := len(string(r.Body)) > 10
 
@@ -259,6 +257,16 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 	})
 
 	// Start scraping on target
+	path, err := utils.GetPath(protocolTemp + "://" + target)
+	if err == nil {
+		if path == "" {
+			c.Visit(protocolTemp + "://" + target + "/" + "robots.txt")
+			c.Visit(protocolTemp + "://" + target + "/" + "sitemap.xml")
+		} else if path == "/" {
+			c.Visit(protocolTemp + "://" + target + "robots.txt")
+			c.Visit(protocolTemp + "://" + target + "sitemap.xml")
+		}
+	}
 	c.Visit(protocolTemp + "://" + target)
 	c.Wait()
 	if html != "" {

main.go

@@ -1,6 +1,6 @@
 /*
 ==========
-Cariddi v1.1.2
+Cariddi v1.1.3
 ==========
 
 This program is free software: you can redistribute it and/or modify
@@ -33,39 +33,50 @@ import (
 	"github.com/edoardottt/cariddi/utils"
 )
 
-//main
+//main function >
 func main() {
 
+	// Scan flags.
 	flags := input.ScanFlag()
 
+	//Print version and exit.
 	if flags.Version {
 		output.Beautify()
 		os.Exit(0)
 	}
 
+	//Print help and exit.
 	if flags.Help {
 		output.PrintHelp()
 		os.Exit(0)
 	}
 
+	//Print examples and exit.
 	if flags.Examples {
 		output.PrintExamples()
 		os.Exit(0)
 	}
 
+	//If it's possible print the cariddi banner.
 	if !flags.Plain {
 		output.Beautify()
 	}
 
+	//Read the targets from standard input.
 	targets := input.ScanTargets()
 
+	//Check if there are errors in the flags definition.
 	input.CheckFlags(flags)
 
+	//If it is needed, read custom endpoints definition
+	//from the specified file.
 	var endpointsFileSlice []string
 	if flags.EndpointsFile != "" {
 		endpointsFileSlice = utils.ReadFile(flags.EndpointsFile)
 	}
 
+	//If it is needed, read custom secrets definition
+	//from the specified file.
 	var secretsFileSlice []string
 	if flags.SecretsFile != "" {
 		secretsFileSlice = utils.ReadFile(flags.SecretsFile)
@@ -76,7 +87,7 @@ func main() {
 	var finalEndpoints []scanner.EndpointMatched
 	var finalExtensions []scanner.FileTypeMatched
 
-	// output files
+	//Create output files if needed (txt / html).
 	var ResultTxt = ""
 	if flags.Txt != "" {
 		ResultTxt = utils.CreateOutputFile(flags.Txt, "results", "txt")
@@ -88,6 +99,7 @@ func main() {
 		output.HeaderHTML("Results", ResultHtml)
 	}
 
+	//For each target generate a crawler and collect all the results.
 	for _, inp := range targets {
 
 		results, secrets, endpoints, extensions := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay,
@@ -101,29 +113,30 @@ func main() {
 		finalExtensions = append(finalExtensions, extensions...)
 	}
 
+	//Remove duplicates from all the results.
 	finalResults = utils.RemoveDuplicateValues(finalResults)
 	finalSecret = scanner.RemoveDuplicateSecrets(finalSecret)
 	finalEndpoints = scanner.RemovDuplicateEndpoints(finalEndpoints)
 	finalExtensions = scanner.RemoveDuplicateExtensions(finalExtensions)
 
-	// IF TXT OUTPUT
+	// IF TXT OUTPUT >
 	if flags.Txt != "" {
 		output.TxtOutput(flags, finalResults, finalSecret, finalEndpoints, finalExtensions)
 	}
 
-	// IF HTML OUTPUT
+	// IF HTML OUTPUT >
 	if flags.Html != "" {
 		output.HtmlOutput(flags, ResultHtml, finalResults, finalSecret, finalEndpoints, finalExtensions)
 	}
 
-	// if needed print secrets
+	//If needed print secrets.
 	if !flags.Plain && len(finalSecret) != 0 {
 		for _, elem := range finalSecret {
 			output.EncapsulateCustomGreen(elem.Secret.Name, elem.Match+" in "+elem.Url)
 		}
 	}
 
-	// if needed print endpoints
+	//If needed print endpoints.
 	if !flags.Plain && len(finalEndpoints) != 0 {
 		for _, elem := range finalEndpoints {
 			for _, parameter := range elem.Parameters {
@@ -140,7 +153,7 @@ func main() {
 		}
 	}
 
-	// if needed print extensions
+	//If needed print extensions.
 	if !flags.Plain && len(finalExtensions) != 0 {
 		for _, elem := range finalExtensions {
 			output.EncapsulateCustomGreen(elem.Filetype.Extension, elem.Url+" matched!")

output/beautify.go

@@ -35,7 +35,7 @@ func Beautify() {
 	banner2 := "   ___ __ _ _ __(_) __| | __| (_)\n"
 	banner3 := "  / __/ _` | '__| |/ _` |/ _` | |\n"
 	banner4 := " | (_| (_| | |  | | (_| | (_| | |\n"
-	banner5 := "  \\___\\__,_|_|  |_|\\__,_|\\__,_|_| v1.1.2\n"
+	banner5 := "  \\___\\__,_|_|  |_|\\__,_|\\__,_|_| v1.1.3\n"
 	banner6 := ""
 	banner7 := " > github.com/edoardottt/cariddi\n"
 	banner8 := " > edoardoottavianelli.it\n"

scanner/secrets.go

@@ -308,6 +308,13 @@ func GetRegexes() []Secret {
 			[]string{},
 			"?",
 		},
+		{
+			"Bugsnag API Key",
+			"Bugsnag API Key",
+			"(?i)(bs|bugsnag)(.{0,20})?[0-9a-f]{32}",
+			[]string{},
+			"?",
+		},
 		{
 			"S3 Bucket",
 			"S3 Bucket",

utils/files.go

@@ -26,7 +26,9 @@ package utils
 import (
 	"bufio"
 	"fmt"
+	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"strings"
 )
@@ -115,3 +117,41 @@ func ElementExists(path string) (bool, error) {
 	}
 	return false, err
 }
+
+//ReadHTTPRequestFromFile reads from a file an HTTP
+//request and returns a *http.Request object
+func ReadHTTPFromFile(inputFile string) (*http.Request, error) {
+	f, err := os.Open(inputFile)
+	if err != nil {
+		fmt.Println("Cannot open input file.")
+		os.Exit(1)
+	}
+	defer f.Close()
+
+	buf := bufio.NewReader(f)
+	req, err := http.ReadRequest(buf)
+	if err != nil {
+		fmt.Println("Cannot read request from input file.")
+		os.Exit(1)
+	}
+	return req, nil
+
+}
+
+//ReadEntireFile returns the content of the inputted file.
+func ReadEntireFile(inputFile string) []byte {
+	file, err := os.Open(inputFile)
+	if err != nil {
+		fmt.Println("Cannot open input file.")
+		os.Exit(1)
+	}
+	defer func() {
+		if err = file.Close(); err != nil {
+			fmt.Println("Cannot close input file.")
+			os.Exit(1)
+		}
+	}()
+
+	b, err := ioutil.ReadAll(file)
+	return b
+}

utils/urls.go

@@ -130,3 +130,12 @@ func SameDomain(url1 string, url2 string) bool {
 	}
 	return u1.Host == u2.Host
 }
+
+//GetPath returns the path of the input URL
+func GetPath(input string) (string, error) {
+	u, err := url.Parse(input)
+	if err != nil {
+		return "", err
+	}
+	return u.Path, nil
+}