update

crawler/colly.go

@@ -30,7 +30,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/edoardottt/cariddi/input"
 	"github.com/edoardottt/cariddi/output"
 	"github.com/edoardottt/cariddi/scanner"
 	"github.com/edoardottt/cariddi/utils"
@@ -42,8 +41,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 	secrets bool, secretsFile []string, plain bool, endpoints bool, endpointsFile []string,
 	fileType int) ([]scanner.SecretMatched, []scanner.EndpointMatched, []scanner.FileTypeMatched) {
 
+	// This is to avoid to insert into the crawler target regular
+	// expression directories passed as input.
+	var targetTemp string
+	if !utils.HasScheme(target) {
+		targetTemp = utils.GetHost("http://" + target)
+	} else {
+		targetTemp = utils.GetHost(target)
+	}
+
 	//clean target input
-	target = input.RemoveProtocol(target)
+	target = utils.RemoveProtocol(target)
 
 	var ignoreSlice []string
 	ignoreBool := false
@@ -59,8 +67,8 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 		ignoreSlice = utils.ReadFile(ignoreTxt)
 	}
 
-	var Finalsecrets []scanner.SecretMatched
-	var Finalendpoints []scanner.EndpointMatched
+	var FinalSecrets []scanner.SecretMatched
+	var FinalEndpoints []scanner.EndpointMatched
 	var FinalExtensions []scanner.FileTypeMatched
 
 	// Instantiate  collector
@@ -68,7 +76,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 		colly.AllowedDomains(target),
 		colly.Async(true),
 		colly.URLFilters(
-			regexp.MustCompile(target+"*"),
+			regexp.MustCompile(targetTemp+"*"),
 		),
 	)
 
@@ -157,14 +165,14 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 			// HERE SCAN FOR SECRETS
 			if secrets {
 				secretsSlice := huntSecrets(secretsFile, r.Request.URL.String(), string(r.Body))
-				Finalsecrets = append(Finalsecrets, secretsSlice...)
+				FinalSecrets = append(FinalSecrets, secretsSlice...)
 			}
 			// HERE SCAN FOR ENDPOINTS
 			if endpoints {
 				endpointsSlice := huntEndpoints(endpointsFile, r.Request.URL.String())
 				for _, elem := range endpointsSlice {
 					if len(elem.Parameters) != 0 {
-						Finalendpoints = append(Finalendpoints, elem)
+						FinalEndpoints = append(FinalEndpoints, elem)
 					}
 				}
 			}
@@ -185,7 +193,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 	if html != "" {
 		output.FooterHTML(html)
 	}
-	return Finalsecrets, Finalendpoints, FinalExtensions
+	return FinalSecrets, FinalEndpoints, FinalExtensions
 }
 
 //huntSecrets hunts for secrets

input/input.go

@@ -42,26 +42,8 @@ func ScanTargets() []string {
 	for sc.Scan() {
 		domain := strings.ToLower(sc.Text())
 		if len(domain) > 2 {
-			result = append(result, RemoveProtocol(domain))
+			result = append(result, utils.RemoveProtocol(domain))
 		}
 	}
 	return utils.RemoveDuplicateValues(result)
 }
-
-//RemovePort removes port from target (:80...)
-func RemovePort(input string) string {
-	res := strings.Index(input, ":")
-	if res >= 0 {
-		return input[:res-1]
-	}
-	return input
-}
-
-//RemoveProtocol removes protocol from target (something://...)
-func RemoveProtocol(input string) string {
-	res := strings.Index(input, "://")
-	if res >= 0 {
-		return input[res+3:]
-	}
-	return input
-}

utils/urls.go

@@ -0,0 +1,71 @@
+/*
+==========
+Cariddi
+==========
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see http://www.gnu.org/licenses/.
+
+	@Repository:  https://github.com/edoardottt/cariddi
+
+	@Author:      edoardottt, https://www.edoardoottavianelli.it
+*/
+
+package utils
+
+import (
+	"net/url"
+	"strings"
+)
+
+//GetHost >
+func GetHost(input string) string {
+	u, err := url.Parse(input)
+	if err != nil {
+		return ""
+	}
+	return u.Host
+}
+
+//GetScheme >
+func GetScheme(input string) string {
+	u, err := url.Parse(input)
+	if err != nil {
+		return ""
+	}
+	return u.Scheme
+}
+
+//HasScheme >
+func HasScheme(input string) bool {
+	res := strings.Index(input, "://")
+	return res >= 0
+}
+
+//RemoveProtocol removes protocol from target (something://...)
+func RemoveProtocol(input string) string {
+	res := strings.Index(input, "://")
+	if res >= 0 {
+		return input[res+3:]
+	}
+	return input
+}
+
+//RemovePort removes port from target (:80...)
+func RemovePort(input string) string {
+	res := strings.Index(input, ":")
+	if res >= 0 {
+		return input[:res-1]
+	}
+	return input
+}