Add user agent creation to avoid the colly default one

crawler/colly.go

@@ -113,6 +113,9 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
 	// Use a Random User Agent for each request if needed
 	if rua {
 		extensions.RandomUserAgent(c)
+	} else {
+		// Avoid using the default colly user agent
+		c.UserAgent = GenerateRandomUserAgent()
 	}
 
 	// Use a Proxy if needed

crawler/useragents.go

@@ -0,0 +1,92 @@
+/*
+==========
+Cariddi
+==========
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see http://www.gnu.org/licenses/.
+
+	@Repository:  https://github.com/edoardottt/cariddi
+
+	@Author:      edoardottt, https://www.edoardoottavianelli.it
+
+
+* Disclaimer *: Code partially taken from:
+https://github.com/gocolly/colly/blob/v1.2.0/extensions/random_user_agent.go
+
+*/
+
+package crawler
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+)
+
+var ffVersions = []float32{
+	58.0,
+	57.0,
+	56.0,
+	52.0,
+	48.0,
+	40.0,
+	35.0,
+}
+
+var chromeVersions = []string{
+	"65.0.3325.146",
+	"64.0.3282.0",
+	"41.0.2228.0",
+	"40.0.2214.93",
+	"37.0.2062.124",
+}
+
+var osStrings = []string{
+	"Macintosh; Intel Mac OS X 10_10",
+	"Windows NT 10.0",
+	"Windows NT 5.1",
+	"Windows NT 6.1; WOW64",
+	"Windows NT 6.1; Win64; x64",
+	"X11; Linux x86_64",
+}
+
+//genFirefoxUA generates a random Firefox User Agent
+func genFirefoxUA() string {
+	rand.Seed(time.Now().UnixNano())
+	version := ffVersions[rand.Intn(len(ffVersions))]
+	os := osStrings[rand.Intn(len(osStrings))]
+	return fmt.Sprintf("Mozilla/5.0 (%s; rv:%.1f) Gecko/20100101 Firefox/%.1f", os, version, version)
+}
+
+//genChromeUA generates a random Chrome User Agent
+func genChromeUA() string {
+	rand.Seed(time.Now().UnixNano())
+	version := chromeVersions[rand.Intn(len(chromeVersions))]
+	os := osStrings[rand.Intn(len(osStrings))]
+	return fmt.Sprintf("Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", os, version)
+}
+
+//GenerateRandomUserAgent generates a random user agent
+//(can be Chrome or Firefox).
+func GenerateRandomUserAgent() string {
+	rand.Seed(time.Now().UnixNano())
+	var ua string
+	decision := rand.Intn(100)
+	if decision%2 == 0 {
+		ua = genChromeUA()
+	} else {
+		ua = genFirefoxUA()
+	}
+	return ua
+}