From 1655c4c9fa91359d6b7e28004114fc64cd804095 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sun, 29 May 2022 08:56:58 +0200 Subject: [PATCH 1/7] Fix1 Can't parse url list with ports #58 --- crawler/colly.go | 15 ++++++++++----- utils/urls.go | 17 +++++++++++------ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/crawler/colly.go b/crawler/colly.go index cd47f41..ab5b0aa 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -29,7 +29,6 @@ package crawler import ( "fmt" "log" - "net/url" "os" "regexp" "strings" @@ -67,7 +66,12 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency } if intensive { - targetTemp = utils.GetRootHost(targetTemp) + var err error + targetTemp, err = utils.GetRootHost(protocolTemp + "://" + targetTemp) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } } if targetTemp == "" { @@ -570,9 +574,10 @@ func IgnoreMatch(url string, ignoreSlice []string) bool { //in intensive mode (if the 2nd level domain matches with //the inputted target). func intensiveOk(target string, urlInput string) bool { - u, err := url.Parse(urlInput) + root, err := utils.GetRootHost(urlInput) if err != nil { - return false + fmt.Println(err.Error()) + os.Exit(1) } - return utils.GetRootHost(u.Host) == target + return root == target } diff --git a/utils/urls.go b/utils/urls.go index 61a8b6c..d2af3a2 100644 --- a/utils/urls.go +++ b/utils/urls.go @@ -27,10 +27,9 @@ along with this program. If not, see http://www.gnu.org/licenses/. package utils import ( + "errors" "net/url" "strings" - - "github.com/bobesa/go-domain-util/domainutil" ) //GetHost takes as input a string and @@ -61,12 +60,18 @@ func GetProtocol(input string) string { //tries to parse it as url, if it's a //well formatted url this function returns //the second level domain -func GetRootHost(input string) string { - _, err := url.Parse(input) +func GetRootHost(input string) (string, error) { + u, err := url.Parse(input) if err != nil { - return "" + return "", err } - return domainutil.Domain(input) + //divide host and port, then split by dot + parts := strings.Split(strings.Split(u.Host, ":")[0], ".") + //return the last two parts + if len(parts) > 1 { + return parts[len(parts)-2] + "." + parts[len(parts)-1], nil + } + return "", errors.New("domain formatted in a bad way") } //HasProtocol takes as input a string and From 389142213fe08fa79c1575e708cf65bff270f4d6 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sun, 29 May 2022 08:57:20 +0200 Subject: [PATCH 2/7] go mod tidy --- go.mod | 1 - go.sum | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 66d68fa..e274d39 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/edoardottt/cariddi go 1.17 require ( - github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 github.com/fatih/color v1.13.0 github.com/gocolly/colly v1.2.0 ) diff --git a/go.sum b/go.sum index 9534d15..cd76f61 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,7 @@ github.com/antchfx/xmlquery v1.3.10 h1:U2yMwr8U0KmGM2iDG2Ky/3LfxNsiK4uw1bSBkeMO9 github.com/antchfx/xmlquery v1.3.10/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks= github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8= github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 h1:2pkAuIM8OF1fy4ToFpMnI4oE+VeUNRbGrpSLKshK0oQ= -github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/09nEjna1UMoasyyQDhOrIn8hi2v2kiJglPWed1idck= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -28,16 +27,17 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= From d7f14adb7654cb9bfdf54b24f7bc8fadefa92a2f Mon Sep 17 00:00:00 2001 From: edoardottt Date: Thu, 18 Aug 2022 16:57:45 +0200 Subject: [PATCH 3/7] Add ua flag --- input/flags.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/input/flags.go b/input/flags.go index 7d66f51..493b789 100644 --- a/input/flags.go +++ b/input/flags.go @@ -58,6 +58,7 @@ type Input struct { Errors bool Info bool Debug bool + UserAgent string } //ScanFlag defines all the options taken @@ -98,6 +99,8 @@ func ScanFlag() Input { debugPtr := flag.Bool("debug", false, "Print debug information while crawling.") + userAgentPtr := flag.String("ua", "", "Use a custom User Agent.") + flag.Parse() result := Input{ @@ -126,6 +129,7 @@ func ScanFlag() Input { *errorsPtr, *infoPtr, *debugPtr, + *userAgentPtr, } return result From bf04c4b90a7a881e02aac86e0486791d99919cb2 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Thu, 18 Aug 2022 16:59:55 +0200 Subject: [PATCH 4/7] Add Custom User Agent --- crawler/colly.go | 9 ++++++--- main.go | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crawler/colly.go b/crawler/colly.go index ab5b0aa..81401cf 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -48,7 +48,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency ignore string, ignoreTxt string, cache bool, timeout int, intensive bool, rua bool, proxy string, secrets bool, secretsFile []string, plain bool, endpoints bool, endpointsFile []string, fileType int, headers map[string]string, - errors bool, info bool, debug bool) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched, + errors bool, info bool, debug bool, userAgent string) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched, []scanner.FileTypeMatched, []scanner.ErrorMatched, []scanner.InfoMatched) { // This is to avoid to insert into the crawler target regular @@ -104,7 +104,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency var FinalInfos []scanner.InfoMatched //crawler creation - c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy, target) + c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy, userAgent, target) // On every a element which has href attribute call callback c.OnHTML("a[href]", func(e *colly.HTMLElement) { @@ -363,13 +363,16 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency //CreateColly takes as input all the settings needed to instantiate //a new Colly Collector object and it returns this object. func CreateColly(delayTime int, concurrency int, cache bool, timeout int, - intensive bool, rua bool, proxy string, target string) *colly.Collector { + intensive bool, rua bool, proxy string, userAgent string, target string) *colly.Collector { c := colly.NewCollector( colly.Async(true), ) c.IgnoreRobotsTxt = true c.AllowURLRevisit = false + if userAgent != "" { + c.UserAgent = userAgent + } err := c.Limit( &colly.LimitRule{ diff --git a/main.go b/main.go index 141279f..c59727c 100644 --- a/main.go +++ b/main.go @@ -122,7 +122,7 @@ func main() { results, secrets, endpoints, extensions, errors, infos := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay, flags.Concurrency, flags.Ignore, flags.IgnoreTxt, flags.Cache, flags.Timeout, flags.Intensive, flags.Rua, flags.Proxy, flags.Secrets, secretsFileSlice, flags.Plain, flags.Endpoints, endpointsFileSlice, - flags.Extensions, headers, flags.Errors, flags.Info, flags.Debug) + flags.Extensions, headers, flags.Errors, flags.Info, flags.Debug, flags.UserAgent) finalResults = append(finalResults, results...) finalSecret = append(finalSecret, secrets...) From 80e427b439e121f01e514a72a607c30cec675e67 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Thu, 18 Aug 2022 17:07:42 +0200 Subject: [PATCH 5/7] Add Custom User Agent --- output/examples.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/output/examples.go b/output/examples.go index 8932f51..94c1c38 100644 --- a/output/examples.go +++ b/output/examples.go @@ -80,5 +80,7 @@ func PrintExamples() { cat urls | cariddi -info - cat urls | cariddi -debug`) + cat urls | cariddi -debug + + cat urls | cariddi -ua "Custom User Agent"`) } From e7f0071db9121a2d7a9d14409ee3ff3bb27d42c9 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Thu, 18 Aug 2022 17:08:28 +0200 Subject: [PATCH 6/7] Add Custom User Agent --- output/help.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/output/help.go b/output/help.go index 93a404a..c175e78 100644 --- a/output/help.go +++ b/output/help.go @@ -77,6 +77,8 @@ func PrintHelp() { Use an external file (txt, one per line) to use custom regexes for secrets hunting. -t int Set timeout for the requests. (default 10) + -rua + Use a custom User Agent. -version Print the version.`) } From d94d4767c14d5e28444377ff3eed0d8213a89c56 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Thu, 18 Aug 2022 17:10:37 +0200 Subject: [PATCH 7/7] Add Custom User Agent --- README.md | 3 +++ output/help.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c754e9..2d3e480 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,8 @@ Usage of cariddi: Use an external file (txt, one per line) to use custom regexes for secrets hunting. -t int Set timeout for the requests. (default 10) + -ua string + Use a custom User Agent. -version Print the version. ``` @@ -184,6 +186,7 @@ Examples 💡 - `cat urls | cariddi -err` (Hunt for errors in websites.) - `cat urls | cariddi -info` (Hunt for useful informations in websites.) - `cat urls | cariddi -debug` (Print debug information while crawling.) + - `cat urls | cariddi -ua "Custom User Agent"` (Use a custom User Agent.) - For Windows: - use `powershell.exe -Command "cat urls | .\cariddi.exe"` inside the Command prompt diff --git a/output/help.go b/output/help.go index c175e78..b31505a 100644 --- a/output/help.go +++ b/output/help.go @@ -77,7 +77,7 @@ func PrintHelp() { Use an external file (txt, one per line) to use custom regexes for secrets hunting. -t int Set timeout for the requests. (default 10) - -rua + -ua Use a custom User Agent. -version Print the version.`)