From c1ae2036de81991df06b112958fb164a17d19216 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 28 Sep 2021 21:47:52 +0200 Subject: [PATCH 1/6] Add event on action attribute of form HTML tag --- crawler/colly.go | 22 ++++++++++++++++++++++ main.go | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/crawler/colly.go b/crawler/colly.go index 012f2b4..491e77e 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -212,6 +212,28 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency } }) + // On every from element which has action attribute call callback + c.OnHTML("form[action]", func(e *colly.HTMLElement) { + link := e.Attr("action") + if len(link) != 0 { + absoluteUrl := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)) + // Visit link found on page + // Only those links are visited which are in AllowedDomains + if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || + (intensive && intensiveOk(targetTemp, absoluteUrl)) { + if ignoreBool { + if !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } + } else { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } + } + } + }) + c.OnResponse(func(r *colly.Response) { fmt.Println(r.Request.URL.String()) diff --git a/main.go b/main.go index 1b91feb..bd329d5 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ /* ========== -Cariddi v0.dev +Cariddi v1.0-devel ========== This program is free software: you can redistribute it and/or modify From 55fdca038452844bb9463bd91b7bcb246dde5452 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 28 Sep 2021 22:12:47 +0200 Subject: [PATCH 2/6] Add user agent creation to avoid the colly default one --- crawler/colly.go | 3 ++ crawler/useragents.go | 92 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 crawler/useragents.go diff --git a/crawler/colly.go b/crawler/colly.go index 491e77e..8789511 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -113,6 +113,9 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // Use a Random User Agent for each request if needed if rua { extensions.RandomUserAgent(c) + } else { + // Avoid using the default colly user agent + c.UserAgent = GenerateRandomUserAgent() } // Use a Proxy if needed diff --git a/crawler/useragents.go b/crawler/useragents.go new file mode 100644 index 0000000..e21efa8 --- /dev/null +++ b/crawler/useragents.go @@ -0,0 +1,92 @@ +/* +========== +Cariddi +========== + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see http://www.gnu.org/licenses/. + + @Repository: https://github.com/edoardottt/cariddi + + @Author: edoardottt, https://www.edoardoottavianelli.it + + +* Disclaimer *: Code partially taken from: +https://github.com/gocolly/colly/blob/v1.2.0/extensions/random_user_agent.go + +*/ + +package crawler + +import ( + "fmt" + "math/rand" + "time" +) + +var ffVersions = []float32{ + 58.0, + 57.0, + 56.0, + 52.0, + 48.0, + 40.0, + 35.0, +} + +var chromeVersions = []string{ + "65.0.3325.146", + "64.0.3282.0", + "41.0.2228.0", + "40.0.2214.93", + "37.0.2062.124", +} + +var osStrings = []string{ + "Macintosh; Intel Mac OS X 10_10", + "Windows NT 10.0", + "Windows NT 5.1", + "Windows NT 6.1; WOW64", + "Windows NT 6.1; Win64; x64", + "X11; Linux x86_64", +} + +//genFirefoxUA generates a random Firefox User Agent +func genFirefoxUA() string { + rand.Seed(time.Now().UnixNano()) + version := ffVersions[rand.Intn(len(ffVersions))] + os := osStrings[rand.Intn(len(osStrings))] + return fmt.Sprintf("Mozilla/5.0 (%s; rv:%.1f) Gecko/20100101 Firefox/%.1f", os, version, version) +} + +//genChromeUA generates a random Chrome User Agent +func genChromeUA() string { + rand.Seed(time.Now().UnixNano()) + version := chromeVersions[rand.Intn(len(chromeVersions))] + os := osStrings[rand.Intn(len(osStrings))] + return fmt.Sprintf("Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", os, version) +} + +//GenerateRandomUserAgent generates a random user agent +//(can be Chrome or Firefox). +func GenerateRandomUserAgent() string { + rand.Seed(time.Now().UnixNano()) + var ua string + decision := rand.Intn(100) + if decision%2 == 0 { + ua = genChromeUA() + } else { + ua = genFirefoxUA() + } + return ua +} From 910d9fc92680d4cfddae8e0cdcf53b59f0774771 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 28 Sep 2021 22:14:47 +0200 Subject: [PATCH 3/6] v1.0-devel --- output/beautify.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/output/beautify.go b/output/beautify.go index 282f395..711e8a6 100644 --- a/output/beautify.go +++ b/output/beautify.go @@ -35,7 +35,7 @@ func Beautify() { banner2 := " ___ __ _ _ __(_) __| | __| (_)\n" banner3 := " / __/ _` | '__| |/ _` |/ _` | |\n" banner4 := " | (_| (_| | | | | (_| | (_| | |\n" - banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.0\n" + banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.0-devel\n" banner6 := "" banner7 := " > github.com/edoardottt/cariddi\n" banner8 := " > edoardoottavianelli.it\n" From 41c3c38b5a01dfd5faff7ae6fc43cf4de31b4f11 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 28 Sep 2021 22:36:45 +0200 Subject: [PATCH 4/6] Add onXML event to catch sitemaps --- crawler/colly.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/crawler/colly.go b/crawler/colly.go index 8789511..c058618 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -237,9 +237,34 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency } }) + // Create a callback on the XPath query searching for the URLs + c.OnXML("//urlset/url/loc", func(e *colly.XMLElement) { + link := e.Text + output.EncapsulateCustomRed("XML URL", link) + if len(link) != 0 { + absoluteUrl := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)) + // Visit link found on page + // Only those links are visited which are in AllowedDomains + if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteUrl)) || + (intensive && intensiveOk(targetTemp, absoluteUrl)) { + if ignoreBool { + if !IgnoreMatch(link, ignoreSlice) { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } + } else { + FinalResults = append(FinalResults, absoluteUrl) + c.Visit(absoluteUrl) + } + } + } + }) + c.OnResponse(func(r *colly.Response) { - fmt.Println(r.Request.URL.String()) + if utils.SameDomain(protocolTemp+"://"+target, r.Request.URL.String()) { + fmt.Println(r.Request.URL.String()) + } lengthOk := len(string(r.Body)) > 10 From f7925a35d92ee09f3cccf3e76378d83e8cc51057 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 28 Sep 2021 22:37:42 +0200 Subject: [PATCH 5/6] v1.1 --- main.go | 2 +- output/beautify.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index bd329d5..c2aa7ec 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ /* ========== -Cariddi v1.0-devel +Cariddi v1.1 ========== This program is free software: you can redistribute it and/or modify diff --git a/output/beautify.go b/output/beautify.go index 711e8a6..8b58a98 100644 --- a/output/beautify.go +++ b/output/beautify.go @@ -35,7 +35,7 @@ func Beautify() { banner2 := " ___ __ _ _ __(_) __| | __| (_)\n" banner3 := " / __/ _` | '__| |/ _` |/ _` | |\n" banner4 := " | (_| (_| | | | | (_| | (_| | |\n" - banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.0-devel\n" + banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.1\n" banner6 := "" banner7 := " > github.com/edoardottt/cariddi\n" banner8 := " > edoardoottavianelli.it\n" From ab2b1d8e62ee06c44aacdec0b5cfa8435af75e4c Mon Sep 17 00:00:00 2001 From: gilfoyle97 Date: Tue, 28 Sep 2021 22:38:35 +0200 Subject: [PATCH 6/6] Update go.yml --- .github/workflows/go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b069ce9..299e1d3 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,9 +2,9 @@ name: Go on: push: - branches: [ main ] + branches: [ main,devel ] pull_request: - branches: [ main ] + branches: [ main,devel ] jobs: