Merge pull request #30 from edoardottt/devel

Devel

crawler/colly.go

@@ -235,7 +235,11 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 
 	c.OnResponse(func(r *colly.Response) {
 
-		fmt.Println(r.Request.URL.String())
+		if ignoreBool && !IgnoreMatch(r.Request.URL.String(), ignoreSlice) {
+			fmt.Println(r.Request.URL.String())
+		} else {
+			fmt.Println(r.Request.URL.String())
+		}
 
 		lengthOk := len(string(r.Body)) > 10
 
@@ -292,6 +296,7 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
 	c := colly.NewCollector(
 		colly.Async(true),
 	)
+	c.IgnoreRobotsTxt = false
 
 	c.Limit(
 		&colly.LimitRule{

output/help.go

@@ -44,9 +44,9 @@ func PrintHelp() {
 		Hunt for juicy file extensions. Integer from 1(juicy) to 7(not juicy).
 	-h	Print the help.
 	-headers string
-    	Use custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2".
+		Use custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2".
   	-headersfile string
-    	Read from an external file custom headers (same format of headers flag).
+	  	Read from an external file custom headers (same format of headers flag).
 	-i string
 		Ignore the URL containing at least one of the elements of this array.
 	-intensive
@@ -60,7 +60,7 @@ func PrintHelp() {
 	-plain
 		Print only the results.
 	-proxy string
-    	Set a Proxy to be used (http and socks5 supported).
+		Set a Proxy to be used (http and socks5 supported).
 	-rua
 		Use a random browser user agent on every request.
 	-s	Hunt for secrets.