WEBDIENSTE/cariddi
1
0
Fork 0
mirror of https://github.com/edoardottt/cariddi.git synced 2026-09-09 19:27:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #53 from edoardottt/devel

Browse Source 
Add golangci-lint action
This commit is contained in:
vrenzolaverace
2022-05-03 15:34:08 +02:00
committed by GitHub
parent 50bccc2677 9573835f90
commit cfcd63c5cd
11 changed files with 160 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/golangci-lint.yml
+46
View File
@@ -0,0 +1,46 @@
name: golangci-lint
on:
push:
tags:
- v*
branches:
- devel
- main
pull_request:
permissions:
contents: read
# Optional: allow read access to pull request. Use with `only-new-issues` option.
# pull-requests: read
jobs:
golangci:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v3
with:
go-version: 1.17
- uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
# Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
version: v1.29
# Optional: working directory, useful for monorepos
# working-directory: somedir
# Optional: golangci-lint command line arguments.
# args: --issues-exit-code=0
# Optional: show only new issues if it's a pull request. The default value is `false`.
# only-new-issues: true
# Optional: if set to true then the all caching functionality will be complete disabled,
# takes precedence over all other caching options.
# skip-cache: true
# Optional: if set to true then the action don't cache or restore ~/go/pkg.
# skip-pkg-cache: true
# Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
# skip-build-cache: true
README.md
+3
View File
@@ -111,6 +111,8 @@ Usage of cariddi:
Use the .cariddi_cache folder as cache.
-d int
Delay between a page crawled and another.
-debug
Print debug information while crawling.
-e Hunt for juicy endpoints.
-ef string
Use an external file (txt, one per line) to use custom parameters for endpoints hunting.
@@ -180,6 +182,7 @@ Examples 💡
- `cat urls | cariddi -headersfile headers.txt` (Read from an external file custom headers)
- `cat urls | cariddi -err` (Hunt for errors in websites.)
- `cat urls | cariddi -info` (Hunt for useful informations in websites.)
- `cat urls | cariddi -debug` (Print debug information while crawling.)
- For Windows:
- use `powershell.exe -Command "cat urls | .\cariddi.exe"` inside the Command prompt
crawler/colly.go
+81 -40
View File
@@ -28,6 +28,7 @@ package crawler
import (
"fmt"
"log"
"net/url"
"os"
"regexp"
@@ -48,7 +49,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
ignore string, ignoreTxt string, cache bool, timeout int, intensive bool, rua bool,
proxy string, secrets bool, secretsFile []string, plain bool, endpoints bool,
endpointsFile []string, fileType int, headers map[string]string,
errors bool, info bool) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched,
errors bool, info bool, debug bool) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched,
[]scanner.FileTypeMatched, []scanner.ErrorMatched, []scanner.InfoMatched) {
// This is to avoid to insert into the crawler target regular
@@ -99,7 +100,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
var FinalInfos []scanner.InfoMatched
//crawler creation
c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy)
c := CreateColly(delayTime, concurrency, cache, timeout, intensive, rua, proxy, target)
// On every a element which has href attribute call callback
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
@@ -113,11 +114,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
}
}
@@ -135,11 +142,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
}
}
@@ -157,11 +170,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
}
}
@@ -179,11 +198,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
}
}
@@ -201,11 +226,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
}
}
@@ -223,11 +254,17 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
} else {
FinalResults = append(FinalResults, absoluteUrl)
c.Visit(absoluteUrl)
err := c.Visit(absoluteUrl)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
}
}
@@ -253,10 +290,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
// HERE SCAN FOR SECRETS
if secrets && lengthOk {
secretsSlice := huntSecrets(secretsFile, r.Request.URL.String(), string(r.Body))
//FinalSecrets = append(FinalSecrets, secretsSlice...)
for _, elem := range secretsSlice {
FinalSecrets = append(FinalSecrets, elem)
}
FinalSecrets = append(FinalSecrets, secretsSlice...)
}
// HERE SCAN FOR ENDPOINTS
if endpoints {
@@ -277,19 +311,13 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
// HERE SCAN FOR ERRORS
if errors {
errorsSlice := huntErrors(r.Request.URL.String(), string(r.Body))
//FinalErrors = append(FinalErrors, errorsSlice...)
for _, elem := range errorsSlice {
FinalErrors = append(FinalErrors, elem)
}
FinalErrors = append(FinalErrors, errorsSlice...)
}
// HERE SCAN FOR INFOS
if info {
infosSlice := huntInfos(r.Request.URL.String(), string(r.Body))
//FinalInfos = append(FinalInfos, infosSlice...)
for _, elem := range infosSlice {
FinalInfos = append(FinalInfos, elem)
}
FinalInfos = append(FinalInfos, infosSlice...)
}
}
})
@@ -298,14 +326,29 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
path, err := utils.GetPath(protocolTemp + "://" + target)
if err == nil {
if path == "" {
c.Visit(protocolTemp + "://" + target + "/" + "robots.txt")
c.Visit(protocolTemp + "://" + target + "/" + "sitemap.xml")
err = c.Visit(protocolTemp + "://" + target + "/" + "robots.txt")
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
err = c.Visit(protocolTemp + "://" + target + "/" + "sitemap.xml")
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
} else if path == "/" {
c.Visit(protocolTemp + "://" + target + "robots.txt")
c.Visit(protocolTemp + "://" + target + "sitemap.xml")
err = c.Visit(protocolTemp + "://" + target + "robots.txt")
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
err = c.Visit(protocolTemp + "://" + target + "sitemap.xml")
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
}
}
c.Visit(protocolTemp + "://" + target)
err = c.Visit(protocolTemp + "://" + target)
if err != nil && debug && err != colly.ErrAlreadyVisited {
log.Println(err)
}
c.Wait()
if html != "" {
output.FooterHTML(html)
@@ -316,20 +359,24 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency
//CreateColly takes as input all the settings needed to instantiate
//a new Colly Collector object and it returns this object.
func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
intensive bool, rua bool, proxy string) *colly.Collector {
intensive bool, rua bool, proxy string, target string) *colly.Collector {
c := colly.NewCollector(
colly.Async(true),
)
c.IgnoreRobotsTxt = false
c.AllowURLRevisit = false
c.Limit(
err := c.Limit(
&colly.LimitRule{
Parallelism: concurrency,
Delay: time.Duration(delayTime) * time.Second,
DomainGlob: "*" + target,
},
)
c.AllowURLRevisit = false
if err != nil {
log.Fatal(err)
}
// Using timeout if needed
if timeout != 10 {
@@ -509,12 +556,6 @@ func RetrieveBody(target string) string {
return ""
}
//isLinkOkay checks if a link is built in a proper way
func isLinkOkay(input string) bool {
_, err := url.Parse(input)
return err == nil
}
//IgnoreMatch checks if the URL should be ignored or not.
func IgnoreMatch(url string, ignoreSlice []string) bool {
for _, ignore := range ignoreSlice {
input/check.go
+5
View File
@@ -106,6 +106,11 @@ func CheckFlags(flags Input) {
}
}
if flags.Plain && flags.Debug {
fmt.Println("You cannot use both plain and debug mode.")
os.Exit(1)
}
if flags.IgnoreTxt != "" {
_ = utils.ReadFile(flags.IgnoreTxt)
}
input/flags.go
+4
View File
@@ -57,6 +57,7 @@ type Input struct {
HeadersFile string
Errors bool
Info bool
Debug bool
}
//ScanFlag defines all the options taken
@@ -95,6 +96,8 @@ func ScanFlag() Input {
infoPtr := flag.Bool("info", false, "Hunt for useful informations in websites.")
debugPtr := flag.Bool("debug", false, "Print debug information while crawling.")
flag.Parse()
result := Input{
@@ -122,6 +125,7 @@ func ScanFlag() Input {
*headersFilePtr,
*errorsPtr,
*infoPtr,
*debugPtr,
}
return result
main.go
+2 -2
View File
@@ -1,6 +1,6 @@
/*
==========
Cariddi v1.1.6
Cariddi v1.1.7
==========
This program is free software: you can redistribute it and/or modify
@@ -122,7 +122,7 @@ func main() {
results, secrets, endpoints, extensions, errors, infos := crawler.Crawler(inp, ResultTxt, ResultHtml, flags.Delay,
flags.Concurrency, flags.Ignore, flags.IgnoreTxt, flags.Cache, flags.Timeout, flags.Intensive,
flags.Rua, flags.Proxy, flags.Secrets, secretsFileSlice, flags.Plain, flags.Endpoints, endpointsFileSlice,
flags.Extensions, headers, flags.Errors, flags.Info)
flags.Extensions, headers, flags.Errors, flags.Info, flags.Debug)
finalResults = append(finalResults, results...)
finalSecret = append(finalSecret, secrets...)
output/beautify.go
+1 -1
View File
@@ -38,7 +38,7 @@ func Beautify() {
banner2 := " ___ __ _ _ __(_) __| | __| (_)\n"
banner3 := " / __/ _` | '__| |/ _` |/ _` | |\n"
banner4 := " | (_| (_| | | | | (_| | (_| | |\n"
banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.1.6\n"
banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.1.7\n"
banner6 := ""
banner7 := " > github.com/edoardottt/cariddi\n"
banner8 := " > edoardoottavianelli.it\n"
output/examples.go
+3 -1
View File
@@ -78,5 +78,7 @@ func PrintExamples() {
cat urls | cariddi -err
cat urls | cariddi -info`)
cat urls | cariddi -info
cat urls | cariddi -debug`)
}
output/help.go
+2
View File
@@ -38,6 +38,8 @@ func PrintHelp() {
Use the .cariddi_cache folder as cache.
-d int
Delay between a page crawled and another.
-debug
Print debug information while crawling.
-e Hunt for juicy endpoints.
-ef string
Use an external file (txt, one per line) to use custom parameters for endpoints hunting.
output/html.go
+13 -8
View File
@@ -38,11 +38,13 @@ func BannerHTML(filename string) {
log.Println(err)
os.Exit(1)
}
file.WriteString("<html><body><div style='" + "background-color:#4adeff;color:white" + "'><h1>Cariddi</h1>")
file.WriteString("<ul>")
file.WriteString("<li><a href='" + "https://github.com/edoardottt/cariddi'" + ">github.com/edoardottt/cariddi</a></li>")
file.WriteString("<li>edoardottt, <a href='" + "https://www.edoardoottavianelli.it'" + ">edoardoottavianelli.it</a></li>")
file.WriteString("<li>Released under <a href='" + "http://www.gnu.org/licenses/gpl-3.0.html'" + ">GPLv3 License</a></li></ul></div>")
_, err = file.WriteString(`<html><body><div style='background-color:#4adeff;color:white'><h1>Cariddi</h1>
<ul><li><a href='https://github.com/edoardottt/cariddi'>github.com/edoardottt/cariddi</a></li>
<li>edoardottt, <a href='https://www.edoardoottavianelli.it'>edoardoottavianelli.it</a></li>
<li>Released under <a href='http://www.gnu.org/licenses/gpl-3.0.html'>GPLv3 License</a></li></ul></div>`)
if err != nil {
log.Fatal(err)
}
file.Close()
}
@@ -108,8 +110,11 @@ func BannerFooterHTML(filename string) {
if err != nil {
log.Println(err)
}
file.WriteString("<div style='" + "background-color:#4adeff;color:white" + "'>")
file.WriteString("<ul><li><a href='" + "https://github.com/edoardottt/cariddi'" + ">Contribute to cariddi</a></li>")
file.WriteString("<li>Released under <a href='" + "http://www.gnu.org/licenses/gpl-3.0.html'" + ">GPLv3 License</a></li></ul></div>")
_, err = file.WriteString(`<div style='background-color:#4adeff;color:white'>
<ul><li><a href='https://github.com/edoardottt/cariddi'>Contribute to cariddi</a></li>
<li>Released under <a href='http://www.gnu.org/licenses/gpl-3.0.html'>GPLv3 License</a></li></ul></div>`)
if err != nil {
log.Fatal(err)
}
file.Close()
}
scanner/scanner.go
-47
View File
@@ -1,47 +0,0 @@
/*
==========
Cariddi
==========
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see http://www.gnu.org/licenses/.
@Repository: https://github.com/edoardottt/cariddi
@Author: edoardottt, https://www.edoardoottavianelli.it
@License: https://github.com/edoardottt/cariddi/blob/main/LICENSE
*/
package scanner
//isEmailUrl checks if the url is a link to a mail
func isEmailUrl(inp string) bool {
return inp[:7] == "mailto:"
}
//isFtpUrl checks if the protocol is ftp
func isFtpUrl(inp string) bool {
return inp[:4] == "ftp:"
}
//isHttpUrl checks if the protocol is http
func isHttpUrl(inp string) bool {
return inp[:5] == "http:"
}
//isHttpsUrl checks if the protocol is https
func isHttpsUrl(inp string) bool {
return inp[:6] == "https:"
}