Merge pull request #21 from edoardottt/devel

Devel - v1.1

.github/workflows/go.yml

@@ -2,9 +2,9 @@ name: Go
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main,devel ]
   pull_request:
-    branches: [ main ]
+    branches: [ main,devel ]
 
 jobs:
 

crawler/colly.go

@@ -113,6 +113,9 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 	// Use a Random User Agent for each request if needed
 	if rua {
 		extensions.RandomUserAgent(c)
+	} else {
+		// Avoid using the default colly user agent
+		c.UserAgent = GenerateRandomUserAgent()
 	}
 
 	// Use a Proxy if needed
@@ -212,9 +215,56 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 		}
 	})
 
+	// On every from element which has action attribute call callback
+	c.OnHTML("form[action]", func(e *colly.HTMLElement) {
+		link := e.Attr("action")
+		if len(link) != 0 {
+			absoluteUrl := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
+			// Visit link found on page
+			// Only those links are visited which are in AllowedDomains
+			if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
+				(intensive && intensiveOk(targetTemp, absoluteUrl)) {
+				if ignoreBool {
+					if !IgnoreMatch(link, ignoreSlice) {
+						FinalResults = append(FinalResults, absoluteUrl)
+						c.Visit(absoluteUrl)
+					}
+				} else {
+					FinalResults = append(FinalResults, absoluteUrl)
+					c.Visit(absoluteUrl)
+				}
+			}
+		}
+	})
+
+	// Create a callback on the XPath query searching for the URLs
+	c.OnXML("//urlset/url/loc", func(e *colly.XMLElement) {
+		link := e.Text
+		output.EncapsulateCustomRed("XML URL", link)
+		if len(link) != 0 {
+			absoluteUrl := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
+			// Visit link found on page
+			// Only those links are visited which are in AllowedDomains
+			if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) ||
+				(intensive && intensiveOk(targetTemp, absoluteUrl)) {
+				if ignoreBool {
+					if !IgnoreMatch(link, ignoreSlice) {
+						FinalResults = append(FinalResults, absoluteUrl)
+						c.Visit(absoluteUrl)
+					}
+				} else {
+					FinalResults = append(FinalResults, absoluteUrl)
+					c.Visit(absoluteUrl)
+				}
+			}
+		}
+	})
+
 	c.OnResponse(func(r *colly.Response) {
 
-		fmt.Println(r.Request.URL.String())
+		if utils.SameDomain(protocolTemp+"://"+target, r.Request.URL.String()) {
+			fmt.Println(r.Request.URL.String())
+		}
 
 		lengthOk := len(string(r.Body)) > 10
 

crawler/useragents.go

@@ -0,0 +1,92 @@
+/*
+==========
+Cariddi
+==========
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see http://www.gnu.org/licenses/.
+
+	@Repository:  https://github.com/edoardottt/cariddi
+
+	@Author:      edoardottt, https://www.edoardoottavianelli.it
+
+
+* Disclaimer *: Code partially taken from:
+https://github.com/gocolly/colly/blob/v1.2.0/extensions/random_user_agent.go
+
+*/
+
+package crawler
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+)
+
+var ffVersions = []float32{
+	58.0,
+	57.0,
+	56.0,
+	52.0,
+	48.0,
+	40.0,
+	35.0,
+}
+
+var chromeVersions = []string{
+	"65.0.3325.146",
+	"64.0.3282.0",
+	"41.0.2228.0",
+	"40.0.2214.93",
+	"37.0.2062.124",
+}
+
+var osStrings = []string{
+	"Macintosh; Intel Mac OS X 10_10",
+	"Windows NT 10.0",
+	"Windows NT 5.1",
+	"Windows NT 6.1; WOW64",
+	"Windows NT 6.1; Win64; x64",
+	"X11; Linux x86_64",
+}
+
+//genFirefoxUA generates a random Firefox User Agent
+func genFirefoxUA() string {
+	rand.Seed(time.Now().UnixNano())
+	version := ffVersions[rand.Intn(len(ffVersions))]
+	os := osStrings[rand.Intn(len(osStrings))]
+	return fmt.Sprintf("Mozilla/5.0 (%s; rv:%.1f) Gecko/20100101 Firefox/%.1f", os, version, version)
+}
+
+//genChromeUA generates a random Chrome User Agent
+func genChromeUA() string {
+	rand.Seed(time.Now().UnixNano())
+	version := chromeVersions[rand.Intn(len(chromeVersions))]
+	os := osStrings[rand.Intn(len(osStrings))]
+	return fmt.Sprintf("Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", os, version)
+}
+
+//GenerateRandomUserAgent generates a random user agent
+//(can be Chrome or Firefox).
+func GenerateRandomUserAgent() string {
+	rand.Seed(time.Now().UnixNano())
+	var ua string
+	decision := rand.Intn(100)
+	if decision%2 == 0 {
+		ua = genChromeUA()
+	} else {
+		ua = genFirefoxUA()
+	}
+	return ua
+}

main.go

@@ -1,6 +1,6 @@
 /*
 ==========
-Cariddi v0.dev
+Cariddi v1.1
 ==========
 
 This program is free software: you can redistribute it and/or modify

output/beautify.go

@@ -35,7 +35,7 @@ func Beautify() {
 	banner2 := "   ___ __ _ _ __(_) __| | __| (_)\n"
 	banner3 := "  / __/ _` | '__| |/ _` |/ _` | |\n"
 	banner4 := " | (_| (_| | |  | | (_| | (_| | |\n"
-	banner5 := "  \\___\\__,_|_|  |_|\\__,_|\\__,_|_| v1.0\n"
+	banner5 := "  \\___\\__,_|_|  |_|\\__,_|\\__,_|_| v1.1\n"
 	banner6 := ""
 	banner7 := " > github.com/edoardottt/cariddi\n"
 	banner8 := " > edoardoottavianelli.it\n"