From 999fcdc60ffd2f4ba4db128055c7c5744dc6bc14 Mon Sep 17 00:00:00 2001
From: edoardottt <edoardott@gmail.com>
Date: Tue, 25 May 2021 10:05:32 +0200
Subject: [PATCH] update

---
 crawler/colly.go | 63 +++++++++++++++++++++++++++---------------------
 input/input.go   |  4 +--
 2 files changed, 37 insertions(+), 30 deletions(-)

diff --git a/crawler/colly.go b/crawler/colly.go
index 7154a04..4687a14 100644
--- a/crawler/colly.go
+++ b/crawler/colly.go
@@ -34,11 +34,13 @@ import (
 )
 
 //Crawler
-func Crawler(target string, delayTime int, concurrency int, secrets bool, secretsFile []string, plain bool, endpoints bool, endpointsFile []string, fileType int) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched, []scanner.FileTypeMatched) {
+func Crawler(target string, delayTime int, concurrency int, secrets bool, secretsFile []string, plain bool, endpoints bool, endpointsFile []string,
+	fileType int) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched, []scanner.FileTypeMatched) {
 
 	//clean target input
-	target = input.RemoveHeaders(target)
+	target = input.RemoveProtocol(target)
 
+	results := make(map[string]bool)
 	var Finalresult []string
 	var Finalsecrets []scanner.SecretMatched
 	var Finalendpoints []scanner.EndpointMatched
@@ -79,36 +81,41 @@ func Crawler(target string, delayTime int, concurrency int, secrets bool, secret
 	})
 
 	c.OnResponse(func(r *colly.Response) {
-		// HERE SCAN FOR SECRETS
-		if secrets {
-			// DON'T SCAN THE URLS SCANNED BEFORE
-			secretsSlice := huntSecrets(secretsFile, r.Request.URL.String(), string(r.Body))
-			for _, elem := range secretsSlice {
+		//if not in map:
+		if _, ok := results[r.Request.URL.String()]; !ok {
 
-				secretFound := scanner.SecretMatched{Secret: elem, Url: r.Request.URL.String()}
-				Finalsecrets = append(Finalsecrets, secretFound)
-			}
-		}
-		// HERE SCAN FOR ENDPOINTS
-		if endpoints {
-			// DON'T SCAN THE URLS SCANNED BEFORE
-			endpointsSlice := huntEndpoints(endpointsFile, r.Request.URL.String())
-			for _, elem := range endpointsSlice {
-				if len(elem.Parameters) != 0 {
-					Finalendpoints = append(Finalendpoints, elem)
+			Finalresult = append(Finalresult, r.Request.URL.String())
+			results[r.Request.URL.String()] = true
+
+			//if endpoints or secrets or filetype: scan
+			if endpoints || secrets || (1 <= fileType && fileType <= 7) {
+				// HERE SCAN FOR SECRETS
+				if secrets {
+					secretsSlice := huntSecrets(secretsFile, r.Request.URL.String(), string(r.Body))
+					for _, elem := range secretsSlice {
+
+						secretFound := scanner.SecretMatched{Secret: elem, Url: r.Request.URL.String()}
+						Finalsecrets = append(Finalsecrets, secretFound)
+					}
+				}
+				// HERE SCAN FOR ENDPOINTS
+				if endpoints {
+					endpointsSlice := huntEndpoints(endpointsFile, r.Request.URL.String())
+					for _, elem := range endpointsSlice {
+						if len(elem.Parameters) != 0 {
+							Finalendpoints = append(Finalendpoints, elem)
+						}
+					}
+				}
+				// HERE SCAN FOR EXTENSIONS
+				if 1 <= fileType && fileType <= 7 {
+					extension := huntExtensions(r.Request.URL.String(), fileType)
+					if extension.Url != "" {
+						FinalExtensions = append(FinalExtensions, extension)
+					}
 				}
 			}
 		}
-		// HERE SCAN FOR EXTENSIONS
-		if 1 <= fileType && fileType <= 7 {
-			// DON'T SCAN THE URLS SCANNED BEFORE
-			extension := huntExtensions(r.Request.URL.String(), fileType)
-			if extension.Url != "" {
-				FinalExtensions = append(FinalExtensions, extension)
-			}
-		}
-
-		Finalresult = append(Finalresult, r.Request.URL.String())
 	})
 
 	// Start scraping on target
diff --git a/input/input.go b/input/input.go
index c36b4cf..d5f27c0 100644
--- a/input/input.go
+++ b/input/input.go
@@ -52,8 +52,8 @@ func RemovePort(input string) string {
 	return input
 }
 
-//RemoveHeaders
-func RemoveHeaders(input string) string {
+//RemoveProtocol
+func RemoveProtocol(input string) string {
 	res := strings.Index(input, "://")
 	if res >= 0 {
 		return input[res+3:]