diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b069ce9..299e1d3 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,9 +2,9 @@ name: Go on: push: - branches: [ main ] + branches: [ main,devel ] pull_request: - branches: [ main ] + branches: [ main,devel ] jobs: diff --git a/crawler/colly.go b/crawler/colly.go index 012f2b4..c058618 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -113,6 +113,9 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Use a Random User Agent for each request if needed if rua { extensions.RandomUserAgent(c) + } else { + // Avoid using the default colly user agent + c.UserAgent = GenerateRandomUserAgent() } // Use a Proxy if needed @@ -212,9 +215,56 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency } }) + // On every from element which has action attribute call callback + c.OnHTML("form[action]", func(e *colly.HTMLElement) { + link := e.Attr("action") + if len(link) != 0 { + absoluteUrl := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)) + // Visit link found on page + // Only those links are visited which are in AllowedDomains + if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || + (intensive && intensiveOk(targetTemp, absoluteUrl)) { + if ignoreBool { + if !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } + } else { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } + } + } + }) + + // Create a callback on the XPath query searching for the URLs + c.OnXML("//urlset/url/loc", func(e *colly.XMLElement) { + link := e.Text + output.EncapsulateCustomRed("XML URL", link) + if len(link) != 0 { + absoluteUrl := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)) + // Visit link found on page + // Only those links are visited which are in AllowedDomains + if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || + (intensive && intensiveOk(targetTemp, absoluteUrl)) { + if ignoreBool { + if !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } + } else { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } + } + } + }) + c.OnResponse(func(r *colly.Response) { - fmt.Println(r.Request.URL.String()) + if utils.SameDomain(protocolTemp+"://"+target, r.Request.URL.String()) { + fmt.Println(r.Request.URL.String()) + } lengthOk := len(string(r.Body)) > 10 diff --git a/crawler/useragents.go b/crawler/useragents.go new file mode 100644 index 0000000..e21efa8 --- /dev/null +++ b/crawler/useragents.go @@ -0,0 +1,92 @@ +/* +========== +Cariddi +========== + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see http://www.gnu.org/licenses/. + + @Repository: https://github.com/edoardottt/cariddi + + @Author: edoardottt, https://www.edoardoottavianelli.it + + +* Disclaimer *: Code partially taken from: +https://github.com/gocolly/colly/blob/v1.2.0/extensions/random_user_agent.go + +*/ + +package crawler + +import ( + "fmt" + "math/rand" + "time" +) + +var ffVersions = []float32{ + 58.0, + 57.0, + 56.0, + 52.0, + 48.0, + 40.0, + 35.0, +} + +var chromeVersions = []string{ + "65.0.3325.146", + "64.0.3282.0", + "41.0.2228.0", + "40.0.2214.93", + "37.0.2062.124", +} + +var osStrings = []string{ + "Macintosh; Intel Mac OS X 10_10", + "Windows NT 10.0", + "Windows NT 5.1", + "Windows NT 6.1; WOW64", + "Windows NT 6.1; Win64; x64", + "X11; Linux x86_64", +} + +//genFirefoxUA generates a random Firefox User Agent +func genFirefoxUA() string { + rand.Seed(time.Now().UnixNano()) + version := ffVersions[rand.Intn(len(ffVersions))] + os := osStrings[rand.Intn(len(osStrings))] + return fmt.Sprintf("Mozilla/5.0 (%s; rv:%.1f) Gecko/20100101 Firefox/%.1f", os, version, version) +} + +//genChromeUA generates a random Chrome User Agent +func genChromeUA() string { + rand.Seed(time.Now().UnixNano()) + version := chromeVersions[rand.Intn(len(chromeVersions))] + os := osStrings[rand.Intn(len(osStrings))] + return fmt.Sprintf("Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", os, version) +} + +//GenerateRandomUserAgent generates a random user agent +//(can be Chrome or Firefox). +func GenerateRandomUserAgent() string { + rand.Seed(time.Now().UnixNano()) + var ua string + decision := rand.Intn(100) + if decision%2 == 0 { + ua = genChromeUA() + } else { + ua = genFirefoxUA() + } + return ua +} diff --git a/main.go b/main.go index 1b91feb..c2aa7ec 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ /* ========== -Cariddi v0.dev +Cariddi v1.1 ========== This program is free software: you can redistribute it and/or modify diff --git a/output/beautify.go b/output/beautify.go index 282f395..8b58a98 100644 --- a/output/beautify.go +++ b/output/beautify.go @@ -35,7 +35,7 @@ func Beautify() { banner2 := " ___ __ _ _ __(_) __| | __| (_)\n" banner3 := " / __/ _` | '__| |/ _` |/ _` | |\n" banner4 := " | (_| (_| | | | | (_| | (_| | |\n" - banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.0\n" + banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.1\n" banner6 := "" banner7 := " > github.com/edoardottt/cariddi\n" banner8 := " > edoardoottavianelli.it\n"