From 9573835f90bba6d573680085cd90827b9c2e8078 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 3 May 2022 15:33:47 +0200 Subject: [PATCH] Add golangci-lint action --- .github/workflows/golangci-lint.yml | 46 +++++++++++ README.md | 3 + crawler/colly.go | 121 +++++++++++++++++++--------- input/check.go | 5 ++ input/flags.go | 4 + main.go | 4 +- output/beautify.go | 2 +- output/examples.go | 4 +- output/help.go | 2 + output/html.go | 21 +++-- scanner/scanner.go | 47 ----------- 11 files changed, 160 insertions(+), 99 deletions(-) create mode 100644 .github/workflows/golangci-lint.yml delete mode 100644 scanner/scanner.go diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 0000000..bc3b86c --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,46 @@ +name: golangci-lint +on: + push: + tags: + - v* + branches: + - devel + - main + pull_request: +permissions: + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + # pull-requests: read +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v3 + with: + go-version: 1.17 + - uses: actions/checkout@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.29 + + # Optional: working directory, useful for monorepos + # working-directory: somedir + + # Optional: golangci-lint command line arguments. + # args: --issues-exit-code=0 + + # Optional: show only new issues if it's a pull request. The default value is `false`. + # only-new-issues: true + + # Optional: if set to true then the all caching functionality will be complete disabled, + # takes precedence over all other caching options. + # skip-cache: true + + # Optional: if set to true then the action don't cache or restore ~/go/pkg. + # skip-pkg-cache: true + + # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. + # skip-build-cache: true \ No newline at end of file diff --git a/README.md b/README.md index 0dea4ac..f68a783 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ Usage of cariddi: Use the .cariddi_cache folder as cache. -d int Delay between a page crawled and another. + -debug + Print debug information while crawling. -e Hunt for juicy endpoints. -ef string Use an external file (txt, one per line) to use custom parameters for endpoints hunting. @@ -180,6 +182,7 @@ Examples 💡 - `cat urls | cariddi -headersfile headers.txt` (Read from an external file custom headers) - `cat urls | cariddi -err` (Hunt for errors in websites.) - `cat urls | cariddi -info` (Hunt for useful informations in websites.) + - `cat urls | cariddi -debug` (Print debug information while crawling.) - For Windows: - use `powershell.exe -Command "cat urls | .\cariddi.exe"` inside the Command prompt diff --git a/crawler/colly.go b/crawler/colly.go index 287c60b..8f0e64e 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -28,6 +28,7 @@ package crawler import ( "fmt" + "log" "net/url" "os" "regexp" @@ -48,7 +49,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency ignore string, ignoreTxt string, cache bool, timeout int, intensive bool, rua bool, proxy string, secrets bool, secretsFile []string, plain bool, endpoints bool, endpointsFile []string, fileType int, headers map[string]string, - errors bool, info bool) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched, + errors bool, info bool, debug bool) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched, []scanner.FileTypeMatched, []scanner.ErrorMatched, []scanner.InfoMatched) { // This is to avoid to insert into the crawler target regular @@ -99,7 +100,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency var FinalInfos []scanner.InfoMatched //crawler creation - c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy) + c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy, target) // On every a element which has href attribute call callback c.OnHTML("a[href]", func(e *colly.HTMLElement) { @@ -113,11 +114,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency if ignoreBool { if !IgnoreMatch(link, ignoreSlice) { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } else { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } } @@ -135,11 +142,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency if ignoreBool { if !IgnoreMatch(link, ignoreSlice) { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } else { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } } @@ -157,11 +170,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency if ignoreBool { if !IgnoreMatch(link, ignoreSlice) { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } else { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } } @@ -179,11 +198,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency if ignoreBool { if !IgnoreMatch(link, ignoreSlice) { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } else { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } } @@ -201,11 +226,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency if ignoreBool { if !IgnoreMatch(link, ignoreSlice) { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } else { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } } @@ -223,11 +254,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency if ignoreBool { if !IgnoreMatch(link, ignoreSlice) { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } else { FinalResults = append(FinalResults, absoluteUrl) - c.Visit(absoluteUrl) + err := c.Visit(absoluteUrl) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } } @@ -253,10 +290,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // HERE SCAN FOR SECRETS if secrets && lengthOk { secretsSlice := huntSecrets(secretsFile, r.Request.URL.String(), string(r.Body)) - //FinalSecrets = append(FinalSecrets, secretsSlice...) - for _, elem := range secretsSlice { - FinalSecrets = append(FinalSecrets, elem) - } + FinalSecrets = append(FinalSecrets, secretsSlice...) } // HERE SCAN FOR ENDPOINTS if endpoints { @@ -277,19 +311,13 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency // HERE SCAN FOR ERRORS if errors { errorsSlice := huntErrors(r.Request.URL.String(), string(r.Body)) - //FinalErrors = append(FinalErrors, errorsSlice...) - for _, elem := range errorsSlice { - FinalErrors = append(FinalErrors, elem) - } + FinalErrors = append(FinalErrors, errorsSlice...) } // HERE SCAN FOR INFOS if info { infosSlice := huntInfos(r.Request.URL.String(), string(r.Body)) - //FinalInfos = append(FinalInfos, infosSlice...) - for _, elem := range infosSlice { - FinalInfos = append(FinalInfos, elem) - } + FinalInfos = append(FinalInfos, infosSlice...) } } }) @@ -298,14 +326,29 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency path, err := utils.GetPath(protocolTemp + "://" + target) if err == nil { if path == "" { - c.Visit(protocolTemp + "://" + target + "/" + "robots.txt") - c.Visit(protocolTemp + "://" + target + "/" + "sitemap.xml") + err = c.Visit(protocolTemp + "://" + target + "/" + "robots.txt") + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } + err = c.Visit(protocolTemp + "://" + target + "/" + "sitemap.xml") + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } else if path == "/" { - c.Visit(protocolTemp + "://" + target + "robots.txt") - c.Visit(protocolTemp + "://" + target + "sitemap.xml") + err = c.Visit(protocolTemp + "://" + target + "robots.txt") + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } + err = c.Visit(protocolTemp + "://" + target + "sitemap.xml") + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } } } - c.Visit(protocolTemp + "://" + target) + err = c.Visit(protocolTemp + "://" + target) + if err != nil && debug && err != colly.ErrAlreadyVisited { + log.Println(err) + } c.Wait() if html != "" { output.FooterHTML(html) @@ -316,20 +359,24 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency //CreateColly takes as input all the settings needed to instantiate //a new Colly Collector object and it returns this object. func CreateColly(delayTime int, concurrency int, cache bool, timeout int, - intensive bool, rua bool, proxy string) *colly.Collector { + intensive bool, rua bool, proxy string, target string) *colly.Collector { c := colly.NewCollector( colly.Async(true), ) c.IgnoreRobotsTxt = false + c.AllowURLRevisit = false - c.Limit( + err := c.Limit( &colly.LimitRule{ Parallelism: concurrency, Delay: time.Duration(delayTime) * time.Second, + DomainGlob: "*" + target, }, ) - c.AllowURLRevisit = false + if err != nil { + log.Fatal(err) + } // Using timeout if needed if timeout != 10 { @@ -509,12 +556,6 @@ func RetrieveBody(target string) string { return "" } -//isLinkOkay checks if a link is built in a proper way -func isLinkOkay(input string) bool { - _, err := url.Parse(input) - return err == nil -} - //IgnoreMatch checks if the URL should be ignored or not. func IgnoreMatch(url string, ignoreSlice []string) bool { for _, ignore := range ignoreSlice { diff --git a/input/check.go b/input/check.go index c1ea7f0..5ed1973 100644 --- a/input/check.go +++ b/input/check.go @@ -106,6 +106,11 @@ func CheckFlags(flags Input) { } } + if flags.Plain && flags.Debug { + fmt.Println("You cannot use both plain and debug mode.") + os.Exit(1) + } + if flags.IgnoreTxt != "" { _ = utils.ReadFile(flags.IgnoreTxt) } diff --git a/input/flags.go b/input/flags.go index 8d0b62c..7d66f51 100644 --- a/input/flags.go +++ b/input/flags.go @@ -57,6 +57,7 @@ type Input struct { HeadersFile string Errors bool Info bool + Debug bool } //ScanFlag defines all the options taken @@ -95,6 +96,8 @@ func ScanFlag() Input { infoPtr := flag.Bool("info", false, "Hunt for useful informations in websites.") + debugPtr := flag.Bool("debug", false, "Print debug information while crawling.") + flag.Parse() result := Input{ @@ -122,6 +125,7 @@ func ScanFlag() Input { *headersFilePtr, *errorsPtr, *infoPtr, + *debugPtr, } return result diff --git a/main.go b/main.go index 5954aad..141279f 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ /* ========== -Cariddi v1.1.6 +Cariddi v1.1.7 ========== This program is free software: you can redistribute it and/or modify @@ -122,7 +122,7 @@ func main() { results, secrets, endpoints, extensions, errors, infos := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay, flags.Concurrency, flags.Ignore, flags.IgnoreTxt, flags.Cache, flags.Timeout, flags.Intensive, flags.Rua, flags.Proxy, flags.Secrets, secretsFileSlice, flags.Plain, flags.Endpoints, endpointsFileSlice, - flags.Extensions, headers, flags.Errors, flags.Info) + flags.Extensions, headers, flags.Errors, flags.Info, flags.Debug) finalResults = append(finalResults, results...) finalSecret = append(finalSecret, secrets...) diff --git a/output/beautify.go b/output/beautify.go index 0f643ce..c4f68ac 100644 --- a/output/beautify.go +++ b/output/beautify.go @@ -38,7 +38,7 @@ func Beautify() { banner2 := " ___ __ _ _ __(_) __| | __| (_)\n" banner3 := " / __/ _` | '__| |/ _` |/ _` | |\n" banner4 := " | (_| (_| | | | | (_| | (_| | |\n" - banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.1.6\n" + banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.1.7\n" banner6 := "" banner7 := " > github.com/edoardottt/cariddi\n" banner8 := " > edoardoottavianelli.it\n" diff --git a/output/examples.go b/output/examples.go index 1e0b25f..8932f51 100644 --- a/output/examples.go +++ b/output/examples.go @@ -78,5 +78,7 @@ func PrintExamples() { cat urls | cariddi -err - cat urls | cariddi -info`) + cat urls | cariddi -info + + cat urls | cariddi -debug`) } diff --git a/output/help.go b/output/help.go index c060f60..93a404a 100644 --- a/output/help.go +++ b/output/help.go @@ -38,6 +38,8 @@ func PrintHelp() { Use the .cariddi_cache folder as cache. -d int Delay between a page crawled and another. + -debug + Print debug information while crawling. -e Hunt for juicy endpoints. -ef string Use an external file (txt, one per line) to use custom parameters for endpoints hunting. diff --git a/output/html.go b/output/html.go index 8313a6b..534f50a 100644 --- a/output/html.go +++ b/output/html.go @@ -38,11 +38,13 @@ func BannerHTML(filename string) { log.Println(err) os.Exit(1) } - file.WriteString("

Cariddi

") - file.WriteString("
") + _, err = file.WriteString(`

Cariddi

+
`) + if err != nil { + log.Fatal(err) + } file.Close() } @@ -108,8 +110,11 @@ func BannerFooterHTML(filename string) { if err != nil { log.Println(err) } - file.WriteString("
") - file.WriteString("
") + _, err = file.WriteString(`
+
`) + if err != nil { + log.Fatal(err) + } file.Close() } diff --git a/scanner/scanner.go b/scanner/scanner.go deleted file mode 100644 index f253713..0000000 --- a/scanner/scanner.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -========== -Cariddi -========== - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses/. - - @Repository: https://github.com/edoardottt/cariddi - - @Author: edoardottt, https://www.edoardoottavianelli.it - - @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE - -*/ - -package scanner - -//isEmailUrl checks if the url is a link to a mail -func isEmailUrl(inp string) bool { - return inp[:7] == "mailto:" -} - -//isFtpUrl checks if the protocol is ftp -func isFtpUrl(inp string) bool { - return inp[:4] == "ftp:" -} - -//isHttpUrl checks if the protocol is http -func isHttpUrl(inp string) bool { - return inp[:5] == "http:" -} - -//isHttpsUrl checks if the protocol is https -func isHttpsUrl(inp string) bool { - return inp[:6] == "https:" -}