diff --git a/crawler/colly.go b/crawler/colly.go index 90aef99..6a0251f 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -79,6 +79,7 @@ func Crawler(target string, delayTime int, concurrency int, secrets bool, secret c.OnRequest(func(r *colly.Request) { // HERE SCAN FOR SECRETS if secrets { + // DON'T SCAN THE URLS SCANNED BEFORE secretsSlice := huntSecrets(secretsFile, r.URL.String()) for _, elem := range secretsSlice { @@ -88,8 +89,13 @@ func Crawler(target string, delayTime int, concurrency int, secrets bool, secret } // HERE SCAN FOR ENDPOINTS if endpoints { + // DON'T SCAN THE URLS SCANNED BEFORE endpointsSlice := huntEndpoints(endpointsFile, r.URL.String()) - Finalendpoints = append(Finalendpoints, endpointsSlice...) + for _, elem := range endpointsSlice { + if len(elem.Parameters) != 0 { + Finalendpoints = append(Finalendpoints, elem) + } + } } Finalresult = append(Finalresult, r.URL.String()) }) @@ -128,8 +134,7 @@ func SecretsMatch(body string) []scanner.Secret { //huntEndpoints func huntEndpoints(endpointsFile string, target string) []scanner.EndpointMatched { if endpointsFile == "" { - body := RetrieveBody(target) - endpoints := EndpointsMatch(body, target) + endpoints := EndpointsMatch(target) return endpoints } @@ -139,11 +144,11 @@ func huntEndpoints(endpointsFile string, target string) []scanner.EndpointMatche } //EndpointsMatch -func EndpointsMatch(body string, target string) []scanner.EndpointMatched { +func EndpointsMatch(target string) []scanner.EndpointMatched { var endpoints []scanner.EndpointMatched matched := []string{} for _, parameter := range scanner.GetJuicyParameters() { - if strings.Contains(body, parameter) { + if strings.Contains(target, parameter) { matched = append(matched, parameter) } endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, Url: target}) diff --git a/utils/slice.go b/utils/slice.go index fa30883..59cd9f6 100644 --- a/utils/slice.go +++ b/utils/slice.go @@ -22,8 +22,8 @@ along with this program. If not, see http://www.gnu.org/licenses/. package utils -//removeDuplicateValues -func removeDuplicateValues(intSlice []string) []string { +//RemoveDuplicateValues +func RemoveDuplicateValues(intSlice []string) []string { keys := make(map[string]bool) list := []string{} for _, entry := range intSlice {