Add -insecure flag to ignore invalid HTTPS certificates

crawler/colly.go

@@ -27,8 +27,10 @@ along with this program.  If not, see http://www.gnu.org/licenses/.
 package crawler
 
 import (
+  "crypto/tls"
 	"fmt"
 	"log"
+  "net/http"
 	"os"
 	"regexp"
 	"strings"
@@ -46,7 +48,7 @@ import (
 //(event handlers, secrets, errors, extensions and endpoints scanning)
 func Crawler(target string, txt string, html string, delayTime int, concurrency int,
 	ignore string, ignoreTxt string, cache bool, timeout int, intensive bool, rua bool,
-	proxy string, secrets bool, secretsFile []string, plain bool, endpoints bool,
+	proxy string, insecure bool, secrets bool, secretsFile []string, plain bool, endpoints bool,
 	endpointsFile []string, fileType int, headers map[string]string,
 	errors bool, info bool, debug bool, userAgent string) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched,
 	[]scanner.FileTypeMatched, []scanner.ErrorMatched, []scanner.InfoMatched) {
@@ -104,7 +106,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 	var FinalInfos []scanner.InfoMatched
 
 	//crawler creation
-	c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy, userAgent, target)
+	c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy, insecure, userAgent, target)
 
 	// On every a element which has href attribute call callback
 	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
@@ -363,7 +365,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 //CreateColly takes as input all the settings needed to instantiate
 //a new Colly Collector object and it returns this object.
 func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
-	intensive bool, rua bool, proxy string, userAgent string, target string) *colly.Collector {
+	intensive bool, rua bool, proxy string, insecure bool, userAgent string, target string) *colly.Collector {
 
 	c := colly.NewCollector(
 		colly.Async(true),
@@ -412,6 +414,12 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
 		}
 	}
 
+  if insecure {
+    c.WithTransport(&http.Transport{
+      TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+    })
+  }
+
 	return c
 }
 

input/flags.go

@@ -48,6 +48,7 @@ type Input struct {
 	Intensive     bool
 	Rua           bool
 	Proxy         string
+	Insecure      bool
 	Secrets       bool
 	SecretsFile   string
 	Endpoints     bool
@@ -81,6 +82,7 @@ func ScanFlag() Input {
 	intensivePtr := flag.Bool("intensive", false, "Crawl searching for resources matching 2nd level domain.")
 	ruaPtr := flag.Bool("rua", false, "Use a random browser user agent on every request.")
 	proxyPtr := flag.String("proxy", "", "Set a Proxy to be used (http and socks5 supported).")
+	insecurePtr := flag.Bool("insecure", false, "Ignore invalid HTTPS certificates")
 
 	secretsPtr := flag.Bool("s", false, "Hunt for secrets.")
 	secretsFilePtr := flag.String("sf", "", "Use an external file (txt, one per line) to use custom regexes for secrets hunting.")
@@ -119,6 +121,7 @@ func ScanFlag() Input {
 		*intensivePtr,
 		*ruaPtr,
 		*proxyPtr,
+		*insecurePtr,
 		*secretsPtr,
 		*secretsFilePtr,
 		*endpointsPtr,

main.go

@@ -121,7 +121,7 @@ func main() {
 
 		results, secrets, endpoints, extensions, errors, infos := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay,
 			flags.Concurrency, flags.Ignore, flags.IgnoreTxt, flags.Cache, flags.Timeout, flags.Intensive,
-			flags.Rua, flags.Proxy, flags.Secrets, secretsFileSlice, flags.Plain, flags.Endpoints, endpointsFileSlice,
+			flags.Rua, flags.Proxy, flags.Insecure, flags.Secrets, secretsFileSlice, flags.Plain, flags.Endpoints, endpointsFileSlice,
 			flags.Extensions, headers, flags.Errors, flags.Info, flags.Debug, flags.UserAgent)
 
 		finalResults = append(finalResults, results...)