Merge pull request #61 from edoardottt/devel

Devel

README.md

@@ -152,7 +152,7 @@ Usage of cariddi:
   -t int
     	Set timeout for the requests. (default 10)
   -ua string
-    	Use a Custom User Agent.
+    	Use a custom User Agent.
   -version
     	Print the version.
 ```

crawler/colly.go

@@ -29,7 +29,6 @@ package crawler
 import (
 	"fmt"
 	"log"
-	"net/url"
 	"os"
 	"regexp"
 	"strings"
@@ -49,7 +48,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 	ignore string, ignoreTxt string, cache bool, timeout int, intensive bool, rua bool,
 	proxy string, secrets bool, secretsFile []string, plain bool, endpoints bool,
 	endpointsFile []string, fileType int, headers map[string]string,
-	errors bool, info bool, debug bool) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched,
+	errors bool, info bool, debug bool, userAgent string) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched,
 	[]scanner.FileTypeMatched, []scanner.ErrorMatched, []scanner.InfoMatched) {
 
 	// This is to avoid to insert into the crawler target regular
@@ -67,7 +66,12 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 	}
 
 	if intensive {
-		targetTemp = utils.GetRootHost(targetTemp)
+		var err error
+		targetTemp, err = utils.GetRootHost(protocolTemp + "://" + targetTemp)
+		if err != nil {
+			fmt.Println(err.Error())
+			os.Exit(1)
+		}
 	}
 
 	if targetTemp == "" {
@@ -100,7 +104,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 	var FinalInfos []scanner.InfoMatched
 
 	//crawler creation
-	c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy, target)
+	c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy, userAgent, target)
 
 	// On every a element which has href attribute call callback
 	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
@@ -359,13 +363,16 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 //CreateColly takes as input all the settings needed to instantiate
 //a new Colly Collector object and it returns this object.
 func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
-	intensive bool, rua bool, proxy string, target string) *colly.Collector {
+	intensive bool, rua bool, proxy string, userAgent string, target string) *colly.Collector {
 
 	c := colly.NewCollector(
 		colly.Async(true),
 	)
 	c.IgnoreRobotsTxt = true
 	c.AllowURLRevisit = false
+	if userAgent != "" {
+		c.UserAgent = userAgent
+	}
 
 	err := c.Limit(
 		&colly.LimitRule{
@@ -570,9 +577,10 @@ func IgnoreMatch(url string, ignoreSlice []string) bool {
 //in intensive mode (if the 2nd level domain matches with
 //the inputted target).
 func intensiveOk(target string, urlInput string) bool {
-	u, err := url.Parse(urlInput)
+	root, err := utils.GetRootHost(urlInput)
 	if err != nil {
-		return false
+		fmt.Println(err.Error())
+		os.Exit(1)
 	}
-	return utils.GetRootHost(u.Host) == target
+	return root == target
 }

go.mod

@@ -3,7 +3,6 @@ module github.com/edoardottt/cariddi
 go 1.17
 
 require (
-	github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89
 	github.com/fatih/color v1.13.0
 	github.com/gocolly/colly v1.2.0
 )

go.sum

@@ -8,8 +8,7 @@ github.com/antchfx/xmlquery v1.3.10 h1:U2yMwr8U0KmGM2iDG2Ky/3LfxNsiK4uw1bSBkeMO9
 github.com/antchfx/xmlquery v1.3.10/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks=
 github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
 github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
-github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 h1:2pkAuIM8OF1fy4ToFpMnI4oE+VeUNRbGrpSLKshK0oQ=
-github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/09nEjna1UMoasyyQDhOrIn8hi2v2kiJglPWed1idck=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
@@ -28,16 +27,17 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
 github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=

input/flags.go

@@ -58,6 +58,7 @@ type Input struct {
 	Errors        bool
 	Info          bool
 	Debug         bool
+	UserAgent     string
 }
 
 //ScanFlag defines all the options taken
@@ -98,6 +99,8 @@ func ScanFlag() Input {
 
 	debugPtr := flag.Bool("debug", false, "Print debug information while crawling.")
 
+	userAgentPtr := flag.String("ua", "", "Use a custom User Agent.")
+
 	flag.Parse()
 
 	result := Input{
@@ -126,6 +129,7 @@ func ScanFlag() Input {
 		*errorsPtr,
 		*infoPtr,
 		*debugPtr,
+		*userAgentPtr,
 	}
 
 	return result

main.go

@@ -122,7 +122,7 @@ func main() {
 		results, secrets, endpoints, extensions, errors, infos := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay,
 			flags.Concurrency, flags.Ignore, flags.IgnoreTxt, flags.Cache, flags.Timeout, flags.Intensive,
 			flags.Rua, flags.Proxy, flags.Secrets, secretsFileSlice, flags.Plain, flags.Endpoints, endpointsFileSlice,
-			flags.Extensions, headers, flags.Errors, flags.Info, flags.Debug)
+			flags.Extensions, headers, flags.Errors, flags.Info, flags.Debug, flags.UserAgent)
 
 		finalResults = append(finalResults, results...)
 		finalSecret = append(finalSecret, secrets...)

output/help.go

@@ -77,8 +77,8 @@ func PrintHelp() {
 		Use an external file (txt, one per line) to use custom regexes for secrets hunting.
 	-t int
 		Set timeout for the requests. (default 10)
-	-ua string
-		Use a Custom User Agent.	
+	-ua
+		Use a custom User Agent.
 	-version
 		Print the version.`)
 }

utils/urls.go

@@ -27,10 +27,9 @@ along with this program.  If not, see http://www.gnu.org/licenses/.
 package utils
 
 import (
+	"errors"
 	"net/url"
 	"strings"
-
-	"github.com/bobesa/go-domain-util/domainutil"
 )
 
 //GetHost takes as input a string and
@@ -61,12 +60,18 @@ func GetProtocol(input string) string {
 //tries to parse it as url, if it's a
 //well formatted url this function returns
 //the second level domain
-func GetRootHost(input string) string {
-	_, err := url.Parse(input)
+func GetRootHost(input string) (string, error) {
+	u, err := url.Parse(input)
 	if err != nil {
-		return ""
+		return "", err
 	}
-	return domainutil.Domain(input)
+	//divide host and port, then split by dot
+	parts := strings.Split(strings.Split(u.Host, ":")[0], ".")
+	//return the last two parts
+	if len(parts) > 1 {
+		return parts[len(parts)-2] + "." + parts[len(parts)-1], nil
+	}
+	return "", errors.New("domain formatted in a bad way")
 }
 
 //HasProtocol takes as input a string and