WEBDIENSTE/cariddi
1
0
Fork 0
mirror of https://github.com/edoardottt/cariddi.git synced 2026-09-12 04:37:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #125 from edoardottt/devel

Browse Source 
v1.3.2
This commit is contained in:
vrenzolaverace
2023-06-09 11:59:26 +02:00
committed by GitHub
parent cf84ea2cc3 d9ec9c4199
commit c2f158aeac
21 changed files with 778 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.golangci.yml
+1 -2
View File
@@ -5,7 +5,6 @@ linters:
enable:
- asciicheck
- bodyclose
- depguard
- dogsled
- dupl
- errcheck
@@ -59,5 +58,5 @@ issues:
exclude-rules:
- path: crawler/colly.go
text: "should replace loop with `(FinalInfos|FinalErrors|FinalSecrets)"
- path: pkg/output/beautify.go
- path: pkg/output/banner.go
text: "should be written without leading space as"
Makefile
+4 -4
View File
@@ -1,8 +1,8 @@
REPO=github.com/edoardottt/cariddi
fmt:
@gofmt -s ./*
@echo "Done."
tidy:
@go get -u ./...
@go mod tidy -v
remod:
@rm -rf go.*
@@ -11,7 +11,7 @@ remod:
@echo "Done."
update:
@go get -u
@go get -u ./...
@go mod tidy -v
@make unlinux
@git pull
README.md
+70 -76
View File
@@ -33,10 +33,6 @@
<a href="https://edoardoottavianelli.it">
<img src="https://github.com/edoardottt/images/blob/main/cariddi/ask-me-anything.svg" alt="ask me anything" />
</a>
<!-- gobadge -->
<a href="https://edoardoottavianelli.it">
<img src="https://github.com/edoardottt/images/blob/main/cariddi/gobadge" alt="gobadge" />
</a>
<!-- license GPLv3.0 -->
<a href="https://github.com/edoardottt/cariddi/blob/master/LICENSE">
<img src="https://github.com/edoardottt/images/blob/main/cariddi/license-GPL3.svg" alt="license-GPL3" />
@@ -60,7 +56,7 @@
<a href="#license-">License</a>
</p>
Preview :bar_chart:
Preview 📊
----------
<!--[![asciicast](https://asciinema.org/a/415989.svg)](https://asciinema.org/a/415989)-->
@@ -73,11 +69,13 @@ Installation 📡
----------
### Using Snap
```bash
sudo snap install cariddi
```
### Using Go
```bash
go install -v github.com/edoardottt/cariddi/cmd/cariddi@latest
```
@@ -101,112 +99,112 @@ You need [Go](https://golang.org/).
- `git clone https://github.com/edoardottt/cariddi.git`
- `cd cariddi`
- `go get ./...`
- `.\make.bat windows` (to install)
- `.\make.bat windows` (to install)
- `.\make.bat unwindows` (to uninstall)
Get Started 🎉
----------
`cariddi -h` prints the help in the command line.
*Note*: Don't rely on the CLI output, use always `-ot/-oh` to save the output.
`cariddi -h` prints the help.
```
Usage of cariddi:
-c int
Concurrency level. (default 20)
Concurrency level. (default 20)
-cache
Use the .cariddi_cache folder as cache.
Use the .cariddi_cache folder as cache.
-d int
Delay between a page crawled and another.
Delay between a page crawled and another.
-debug
Print debug information while crawling.
-e Hunt for juicy endpoints.
Print debug information while crawling.
-e Hunt for juicy endpoints.
-ef string
Use an external file (txt, one per line) to use custom parameters for endpoints hunting.
Use an external file (txt, one per line) to use custom parameters for endpoints hunting.
-err
Hunt for errors in websites.
Hunt for errors in websites.
-examples
Print the examples.
Print the examples.
-ext int
Hunt for juicy file extensions. Integer from 1(juicy) to 7(not juicy).
-h Print the help.
Hunt for juicy file extensions. Integer from 1(juicy) to 7(not juicy).
-h Print the help.
-headers string
Use custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2".
Use custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2".
-headersfile string
Read from an external file custom headers (same format of headers flag).
Read from an external file custom headers (same format of headers flag).
-json
Print the output as JSON in stdout.
Print the output as JSON in stdout.
-i string
Ignore the URL containing at least one of the elements of this array.
Ignore the URL containing at least one of the elements of this array.
-info
Hunt for useful informations in websites.
Hunt for useful informations in websites.
-intensive
Crawl searching for resources matching 2nd level domain.
Crawl searching for resources matching 2nd level domain.
-it string
Ignore the URL containing at least one of the lines of this file.
Ignore the URL containing at least one of the lines of this file.
-oh string
Write the output into an HTML file.
Write the output into an HTML file.
-ot string
Write the output into a TXT file.
Write the output into a TXT file.
-plain
Print only the results.
Print only the results.
-proxy string
Set a Proxy to be used (http and socks5 supported).
Set a Proxy to be used (http and socks5 supported).
-rua
Use a random browser user agent on every request.
-s Hunt for secrets.
Use a random browser user agent on every request.
-s Hunt for secrets.
-sf string
Use an external file (txt, one per line) to use custom regexes for secrets hunting.
Use an external file (txt, one per line) to use custom regexes for secrets hunting.
-sr
Store HTTP responses.
-t int
Set timeout for the requests. (default 10)
Set timeout for the requests. (default 10)
-ua string
Use a custom User Agent.
Use a custom User Agent.
-version
Print the version.
Print the version.
```
Examples 💡
----------
- `cariddi -version` (Print the version)
- `cariddi -h` (Print the help)
- `cariddi -examples` (Print the examples)
- `cat urls | cariddi -s` (Hunt for secrets)
- `cat urls | cariddi -d 2` (2 seconds between a page crawled and another)
- `cat urls | cariddi -c 200` (Set the concurrency level to 200)
- `cat urls | cariddi -e` (Hunt for juicy endpoints)
- `cat urls | cariddi -plain` (Print only useful things)
- `cat urls | cariddi -ot target_name` (Results in txt file)
- `cat urls | cariddi -oh target_name` (Results in html file)
- `cat urls | cariddi -ext 2` (Hunt for juicy (level 2 out of 7) files)
- `cat urls | cariddi -e -ef endpoints_file` (Hunt for custom endpoints)
- `cat urls | cariddi -s -sf secrets_file` (Hunt for custom secrets)
- `cat urls | cariddi -i forum,blog,community,open` (Ignore urls containing these words)
- `cat urls | cariddi -it ignore_file` (Ignore urls containing at least one line in the input file)
- `cat urls | cariddi -cache` (Use the .cariddi_cache folder as cache)
- `cat urls | cariddi -t 5` (Set the timeout for the requests)
- `cat urls | cariddi -intensive` (Crawl searching also subdomains, same as `*.target.com`)
- `cat urls | cariddi -rua` (Use a random browser user agent on every request)
- `cat urls | cariddi -proxy http://127.0.0.1:8080` (Set a Proxy (http and socks5 supported))
- `cat urls | cariddi -headers "Cookie: auth=admin;type=2;; X-Custom: customHeader"`
- `cat urls | cariddi -headersfile headers.txt` (Read from an external file custom headers)
- `cat urls | cariddi -err` (Hunt for errors in websites)
- `cat urls | cariddi -info` (Hunt for useful informations in websites)
- `cat urls | cariddi -debug` (Print debug information while crawling)
- `cat urls | cariddi -ua "Custom User Agent"` (Use a custom User Agent)
- `cat urls | cariddi -json` (Print the output as JSON in stdout)
- `cat urls | cariddi -json | jq .` (Pipe the JSON output into jq)
- `cariddi -version` (Print the version)
- `cariddi -h` (Print the help)
- `cariddi -examples` (Print the examples)
- `cat urls | cariddi -s` (Hunt for secrets)
- `cat urls | cariddi -d 2` (2 seconds between a page crawled and another)
- `cat urls | cariddi -c 200` (Set the concurrency level to 200)
- `cat urls | cariddi -e` (Hunt for juicy endpoints)
- `cat urls | cariddi -plain` (Print only results)
- `cat urls | cariddi -ot target_name` (Results in txt file)
- `cat urls | cariddi -oh target_name` (Results in html file)
- `cat urls | cariddi -ext 2` (Hunt for juicy (level 2 out of 7) files)
- `cat urls | cariddi -e -ef endpoints_file` (Hunt for custom endpoints)
- `cat urls | cariddi -s -sf secrets_file` (Hunt for custom secrets)
- `cat urls | cariddi -i forum,blog,community,open` (Ignore urls containing these words)
- `cat urls | cariddi -it ignore_file` (Ignore urls containing at least one line in the input file)
- `cat urls | cariddi -cache` (Use the .cariddi_cache folder as cache)
- `cat urls | cariddi -t 5` (Set the timeout for the requests)
- `cat urls | cariddi -intensive` (Crawl searching also subdomains, same as `*.target.com`)
- `cat urls | cariddi -rua` (Use a random browser user agent on every request)
- `cat urls | cariddi -proxy http://127.0.0.1:8080` (Set a Proxy, http and socks5 supported)
- `cat urls | cariddi -headers "Cookie: auth=admin;type=2;; X-Custom: customHeader"`
- `cat urls | cariddi -headersfile headers.txt` (Read from an external file custom headers)
- `cat urls | cariddi -err` (Hunt for errors in websites)
- `cat urls | cariddi -info` (Hunt for useful informations in websites)
- `cat urls | cariddi -debug` (Print debug information while crawling)
- `cat urls | cariddi -ua "Custom User Agent"` (Use a custom User Agent)
- `cat urls | cariddi -json` (Print the output as JSON in stdout)
- `cat urls | cariddi -sr` (Store HTTP responses)
- For Windows:
- use `powershell.exe -Command "cat urls | .\cariddi.exe"` inside the Command prompt
- or just `cat urls | cariddi.exe` using PowerShell
- For Windows:
- use `powershell.exe -Command "cat urls | .\cariddi.exe"` inside the Command prompt
- or just `cat urls | cariddi.exe` using PowerShell
- To integrate cariddi with Burpsuite [make sure to follow these steps](https://github.com/edoardottt/cariddi/wiki/BurpSuite-Integration).
- To integrate cariddi with Burpsuite [make sure to follow these steps](https://github.com/edoardottt/cariddi/wiki/BurpSuite-Integration).
Changelog 📌
-------
Detailed changes for each release are documented in the [release notes](https://github.com/edoardottt/cariddi/releases).
Contributing 🛠
@@ -215,21 +213,17 @@ Contributing 🛠
Just open an [issue](https://github.com/edoardottt/cariddi/issues)/[pull request](https://github.com/edoardottt/cariddi/pulls).
Before opening a pull request, download [golangci-lint](https://golangci-lint.run/usage/install/) and run
```bash
golangci-lint run
```
If there aren't errors, go ahead :)
**Help me building this!**
Special thanks to: [go-colly](http://go-colly.org/), [zricethezav](https://github.com/zricethezav/gitleaks/blob/master/config/default.go), [projectdiscovery](https://github.com/projectdiscovery/nuclei-templates/tree/master/file/keys), [tomnomnom](https://github.com/tomnomnom/gf/tree/master/examples) and [RegexPassive](https://github.com/hahwul/RegexPassive).
Special thanks to: [go-colly](http://go-colly.org/), [zricethezav](https://github.com/zricethezav/gitleaks/blob/master/config/default.go), [projectdiscovery](https://github.com/projectdiscovery/nuclei-templates/tree/master/file/keys), [tomnomnom](https://github.com/tomnomnom/gf/tree/master/examples), [RegexPassive](https://github.com/hahwul/RegexPassive) and [all the contributors](https://github.com/edoardottt/cariddi/wiki/Contributors).
**To do:**
- [ ] Add more Tests
- [ ] Tor support
License 📝
-------
cmd/cariddi/main.go
+7 -2
View File
@@ -44,7 +44,7 @@ func main() {
// Print version and exit.
if flags.Version {
output.Beautify()
output.Banner()
os.Exit(0)
}
@@ -62,7 +62,7 @@ func main() {
// If it's possible print the cariddi banner.
if !flags.Plain {
output.Beautify()
output.Banner()
}
// Setup the config according to the flags that were
@@ -86,6 +86,7 @@ func main() {
InfoFlag: flags.Info,
Debug: flags.Debug,
UserAgent: flags.UserAgent,
StoreResp: flags.StoreResp,
}
// Read the targets from standard input.
@@ -126,6 +127,10 @@ func main() {
output.HeaderHTML("Results", ResultHTML)
}
if config.StoreResp {
fileUtils.CreateIndexOutputFile("index.responses.txt")
}
// Read headers if needed
if flags.HeadersFile != "" || flags.Headers != "" {
var headersInput string
go.mod
+2 -2
View File
@@ -18,11 +18,11 @@ require (
github.com/golang/protobuf v1.5.3 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-isatty v0.0.18 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
github.com/temoto/robotstxt v1.1.2 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/text v0.8.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.30.0 // indirect
go.sum
+4 -3
View File
@@ -30,8 +30,8 @@ github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
@@ -65,8 +65,9 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
internal/file/file.go
+42 -5
View File
@@ -33,6 +33,7 @@ import (
"log"
"net/http"
"os"
"path/filepath"
"strings"
)
@@ -52,20 +53,32 @@ func CreateOutputFolder() {
}
}
// CreateOutputFile takes a target (of the attack), a subcommand
// CreateHostOutputFolder creates the host output folder
// for the HTTP responses.
// If it fails exits with an error message.
func CreateHostOutputFolder(host string) {
// Create a folder/directory at a full qualified path
err := os.MkdirAll(filepath.Join("output-cariddi", host), Permission0755)
if err != nil {
fmt.Println("Can't create host output folder.")
os.Exit(1)
}
}
// CreateOutputFile takes as input a target (of the attack), a subcommand
// (PORT-DNS-DIR-SUBDOMAIN-REPORT) and a format (json-html-txt).
// It creates the output folder if needed, then checks if the output file
// already exists, if yes asks the user if scilla has to overwrite it;
// if no scilla creates it.
// already exists, if yes asks the user if cariddi has to overwrite it;
// if no cariddi creates it.
// Whenever an instruction fails, it exits with an error message.
func CreateOutputFile(target string, subcommand string, format string) string {
target = ReplaceBadCharacterOutput(target)
var filename string
if subcommand != "" {
filename = "output-cariddi" + "/" + target + "." + subcommand + "." + format
filename = filepath.Join("output-cariddi", target+"."+subcommand+"."+format)
} else {
filename = "output-cariddi" + "/" + target + "." + format
filename = filepath.Join("output-cariddi", target+"."+format)
}
_, err := os.Stat(filename)
@@ -101,6 +114,30 @@ func CreateOutputFile(target string, subcommand string, format string) string {
return filename
}
// CreateIndexOutputFile takes as input the name of the index file.
// It creates the output folder if needed, then checks if the index output file
// already exists, if no cariddi creates it.
// Whenever an instruction fails, it exits with an error message.
func CreateIndexOutputFile(filename string) {
_, err := os.Stat(filename)
if os.IsNotExist(err) {
if _, err := os.Stat("output-cariddi/"); os.IsNotExist(err) {
CreateOutputFolder()
}
// If the file doesn't exist, create it.
filename = filepath.Join("output-cariddi", filename)
f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, Permission0644)
if err != nil {
fmt.Println("Can't create output file.")
os.Exit(1)
}
f.Close()
}
}
// ReplaceBadCharacterOutput replaces forward-slashes
// with dashes (to avoid problems with output folder).
func ReplaceBadCharacterOutput(input string) string {
make.bat
+6 -7
View File
@@ -17,8 +17,8 @@ IF "%ARG%"=="update" (
GOTO Done
)
IF "%ARG%"=="fmt" (
CALL :Fmt
IF "%ARG%"=="tidy" (
CALL :Tidy
GOTO Done
)
@@ -50,10 +50,9 @@ go test -v ./...
echo Done
EXIT /B 0
:Fmt
set GO111MODULE=on
echo Formatting ...
go fmt ./...
:Tidy
go get -u ./...
go mod tidy -v
echo Done.
EXIT /B 0
@@ -64,7 +63,7 @@ EXIT /B 0
:Update
set GO111MODULE=on
echo Updating ...
go get -u
go get -u ./...
go mod tidy -v
CALL :Unwindows
git pull
pkg/crawler/colly.go
+52 -49
View File
@@ -35,6 +35,7 @@ import (
"net/http"
"os"
"os/signal"
"strings"
"time"
fileUtils "github.com/edoardottt/cariddi/internal/file"
@@ -47,47 +48,6 @@ import (
"github.com/gocolly/colly/extensions"
)
type Results struct {
URLs []string
Secrets []scanner.SecretMatched
Endpoints []scanner.EndpointMatched
Extensions []scanner.FileTypeMatched
Errors []scanner.ErrorMatched
Infos []scanner.InfoMatched
}
type Scan struct {
// Flags
Cache bool
Debug bool
EndpointsFlag bool
ErrorsFlag bool
InfoFlag bool
Intensive bool
Plain bool
Rua bool
SecretsFlag bool
Ignore string
IgnoreTxt string
JSON bool
HTML string
Proxy string
Target string
Txt string
UserAgent string
FileType int
Headers map[string]string
// Settings
Concurrency int
Delay int
Timeout int
// Storage
SecretsSlice []string
EndpointsSlice []string
}
// New it's the actual crawler engine.
// It controls all the behaviours of a scan
// (event handlers, secrets, errors, extensions and endpoints scanning).
@@ -151,7 +111,7 @@ func New(scan *Scan) *Results {
Intensive: scan.Intensive,
Ignore: ignoreBool,
Debug: scan.Debug,
JSON: scan.JSON,
JSON: scan.JSON,
IgnoreSlice: ignoreSlice,
URLs: &results.URLs,
}
@@ -169,6 +129,17 @@ func New(scan *Scan) *Results {
}
c.OnResponse(func(r *colly.Response) {
if !scan.JSON {
fmt.Println(r.Request.URL)
}
if scan.StoreResp {
err := output.StoreHTTPResponse(r)
if err != nil {
log.Println(err)
}
}
minBodyLentgh := 10
lengthOk := len(string(r.Body)) > minBodyLentgh
secrets := []scanner.SecretMatched{}
@@ -218,10 +189,12 @@ func New(scan *Scan) *Results {
infos = append(infos, infosSlice...)
}
}
if scan.JSON {
jsonOutput, err := output.GetJSONString(
r, secrets, parameters, filetype, errors, infos,
)
if err == nil {
fmt.Println(string(jsonOutput))
} else {
@@ -361,13 +334,6 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
// registerHTMLEvents registers the associated functions for each
// HTML event triggering an action.
func registerHTMLEvents(c *colly.Collector, event *Event) {
// On every request that Colly is making, print the URL it's currently visiting
c.OnRequest(func(e *colly.Request) {
if (!event.JSON){
fmt.Println(e.URL.String())
}
})
// On every a element which has href attribute call callback
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
link := e.Attr("href")
@@ -435,3 +401,40 @@ func registerXMLEvents(c *colly.Collector, event *Event) {
visitXMLLink(e.Text, event, e, c)
})
}
// visitHTMLLink checks if the collector should visit a link or not.
func visitHTMLLink(link string, event *Event, e *colly.HTMLElement, c *colly.Collector) {
if len(link) != 0 && !strings.HasPrefix(link, "data:image") {
absoluteURL := urlUtils.AbsoluteURL(event.ProtocolTemp, event.TargetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
visitLink(event, c, absoluteURL)
}
}
// visitXMLLink checks if the collector should visit a link or not.
func visitXMLLink(link string, event *Event, e *colly.XMLElement, c *colly.Collector) {
if len(link) != 0 && !strings.HasPrefix(link, "data:image") {
absoluteURL := urlUtils.AbsoluteURL(event.ProtocolTemp, event.TargetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
visitLink(event, c, absoluteURL)
}
}
// visitLink is a protocol agnostic wrapper to visit a link.
func visitLink(event *Event, c *colly.Collector, absoluteURL string) {
if (!event.Intensive && urlUtils.SameDomain(event.ProtocolTemp+"://"+event.Target, absoluteURL)) ||
(event.Intensive && intensiveOk(event.TargetTemp, absoluteURL, event.Debug)) {
if !event.Ignore || (event.Ignore && !IgnoreMatch(absoluteURL, &event.IgnoreSlice)) {
err := c.Visit(absoluteURL)
if !errors.Is(err, colly.ErrAlreadyVisited) {
*event.URLs = append(*event.URLs, absoluteURL)
if err != nil && event.Debug {
log.Println(err)
}
}
}
}
}
pkg/crawler/options.go
+83
View File
@@ -0,0 +1,83 @@
/*
==========
Cariddi
==========
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see http://www.gnu.org/licenses/.
@Repository: https://github.com/edoardottt/cariddi
@Author: edoardottt, https://www.edoardoottavianelli.it
@License: https://github.com/edoardottt/cariddi/blob/main/LICENSE
*/
package crawler
import "github.com/edoardottt/cariddi/pkg/scanner"
type Results struct {
URLs []string
Secrets []scanner.SecretMatched
Endpoints []scanner.EndpointMatched
Extensions []scanner.FileTypeMatched
Errors []scanner.ErrorMatched
Infos []scanner.InfoMatched
}
type Scan struct {
// Flags
Cache bool
Debug bool
EndpointsFlag bool
ErrorsFlag bool
InfoFlag bool
Intensive bool
Plain bool
Rua bool
SecretsFlag bool
Ignore string
IgnoreTxt string
JSON bool
HTML string
Proxy string
Target string
Txt string
UserAgent string
FileType int
Headers map[string]string
StoreResp bool
// Settings
Concurrency int
Delay int
Timeout int
// Storage
SecretsSlice []string
EndpointsSlice []string
}
type Event struct {
ProtocolTemp string
TargetTemp string
Target string
Intensive bool
Ignore bool
Debug bool
JSON bool
IgnoreSlice []string
URLs *[]string
}
pkg/crawler/scan.go
+194
View File
@@ -0,0 +1,194 @@
/*
==========
Cariddi
==========
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see http://www.gnu.org/licenses/.
@Repository: https://github.com/edoardottt/cariddi
@Author: edoardottt, https://www.edoardoottavianelli.it
@License: https://github.com/edoardottt/cariddi/blob/main/LICENSE
*/
package crawler
import (
"regexp"
"strings"
urlUtils "github.com/edoardottt/cariddi/internal/url"
"github.com/edoardottt/cariddi/pkg/scanner"
)
// huntSecrets hunts for secrets.
func huntSecrets(target, body string, secretsFile *[]string) []scanner.SecretMatched {
secrets := SecretsMatch(target, body, secretsFile)
return secrets
}
// SecretsMatch checks if a body matches some secrets.
func SecretsMatch(url, body string, secretsFile *[]string) []scanner.SecretMatched {
var secrets []scanner.SecretMatched
if len(*secretsFile) == 0 {
for _, secret := range scanner.GetSecretRegexes() {
if matched, err := regexp.Match(secret.Regex, []byte(body)); err == nil && matched {
re := regexp.MustCompile(secret.Regex)
matches := re.FindAllStringSubmatch(body, -1)
// Avoiding false positives
var isFalsePositive = false
for _, match := range matches {
for _, falsePositive := range secret.FalsePositives {
if strings.Contains(strings.ToLower(match[0]), falsePositive) {
isFalsePositive = true
break
}
}
if !isFalsePositive {
secretFound := scanner.SecretMatched{Secret: secret, URL: url, Match: match[0]}
secrets = append(secrets, secretFound)
}
}
}
}
} else {
for _, secret := range *secretsFile {
if matched, err := regexp.Match(secret, []byte(body)); err == nil && matched {
re := regexp.MustCompile(secret)
matches := re.FindAllStringSubmatch(body, -1)
for _, match := range matches {
secretScanned := scanner.Secret{Name: "CustomFromFile", Description: "", Regex: secret, Poc: ""}
secretFound := scanner.SecretMatched{Secret: secretScanned, URL: url, Match: match[0]}
secrets = append(secrets, secretFound)
}
}
}
}
return scanner.RemoveDuplicateSecrets(secrets)
}
// huntEndpoints hunts for juicy endpoints.
func huntEndpoints(target string, endpointsFile *[]string) []scanner.EndpointMatched {
endpoints := EndpointsMatch(target, endpointsFile)
return endpoints
}
// EndpointsMatch check if an endpoint matches a juicy parameter.
func EndpointsMatch(target string, endpointsFile *[]string) []scanner.EndpointMatched {
endpoints := []scanner.EndpointMatched{}
matched := []scanner.Parameter{}
parameters := urlUtils.RetrieveParameters(target)
if len(*endpointsFile) == 0 {
for _, parameter := range scanner.GetJuicyParameters() {
for _, param := range parameters {
if strings.ToLower(param) == parameter.Parameter {
matched = append(matched, parameter)
}
}
}
endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target})
} else {
for _, parameter := range *endpointsFile {
for _, param := range parameters {
if param == parameter {
matched = append(matched, scanner.Parameter{Parameter: parameter, Attacks: []string{}})
}
}
}
endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target})
}
return endpoints
}
// huntExtensions hunts for extensions.
func huntExtensions(target string, severity int) scanner.FileTypeMatched {
extension := scanner.FileTypeMatched{}
copyTarget := target
for _, ext := range scanner.GetExtensions() {
if ext.Severity <= severity {
firstIndex := strings.Index(target, "?")
if firstIndex > -1 {
target = target[:firstIndex]
}
if strings.ToLower(target[len(target)-len("."+ext.Extension):]) == "."+ext.Extension {
extension = scanner.FileTypeMatched{Filetype: ext, URL: copyTarget}
}
}
}
return extension
}
// huntErrors hunts for errors.
func huntErrors(target, body string) []scanner.ErrorMatched {
errorsSlice := ErrorsMatch(target, body)
return errorsSlice
}
// ErrorsMatch checks the patterns for errors.
func ErrorsMatch(url, body string) []scanner.ErrorMatched {
errors := []scanner.ErrorMatched{}
for _, errorItem := range scanner.GetErrorRegexes() {
for _, errorRegex := range errorItem.Regex {
if matched, err := regexp.Match(errorRegex, []byte(body)); err == nil && matched {
re := regexp.MustCompile(errorRegex)
matches := re.FindAllStringSubmatch(body, -1)
for _, match := range matches {
errorFound := scanner.ErrorMatched{Error: errorItem, URL: url, Match: match[0]}
errors = append(errors, errorFound)
}
}
}
}
return scanner.RemoveDuplicateErrors(errors)
}
// huntInfos hunts for infos.
func huntInfos(target, body string) []scanner.InfoMatched {
infosSlice := InfoMatch(target, body)
return infosSlice
}
// InfoMatch checks the patterns for infos.
func InfoMatch(url, body string) []scanner.InfoMatched {
infos := []scanner.InfoMatched{}
for _, infoItem := range scanner.GetInfoRegexes() {
if matched, err := regexp.Match(infoItem.Regex, []byte(body)); err == nil && matched {
re := regexp.MustCompile(infoItem.Regex)
matches := re.FindAllStringSubmatch(body, -1)
for _, match := range matches {
infoFound := scanner.InfoMatched{Info: infoItem, URL: url, Match: match[0]}
infos = append(infos, infoFound)
}
}
}
return scanner.RemoveDuplicateInfos(infos)
}
pkg/crawler/utils.go
+26 -213
View File
@@ -1,225 +1,38 @@
/*
==========
Cariddi
==========
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see http://www.gnu.org/licenses/.
@Repository: https://github.com/edoardottt/cariddi
@Author: edoardottt, https://www.edoardoottavianelli.it
@License: https://github.com/edoardottt/cariddi/blob/main/LICENSE
*/
package crawler
import (
"errors"
"fmt"
"log"
"regexp"
"strings"
urlUtils "github.com/edoardottt/cariddi/internal/url"
"github.com/edoardottt/cariddi/pkg/scanner"
"github.com/gocolly/colly"
)
type Event struct {
ProtocolTemp string
TargetTemp string
Target string
Intensive bool
Ignore bool
Debug bool
JSON bool
IgnoreSlice []string
URLs *[]string
}
// visitHTMLLink checks if the collector should visit a link or not.
func visitHTMLLink(link string, event *Event, e *colly.HTMLElement, c *colly.Collector) {
if len(link) != 0 && !strings.HasPrefix(link, "data:image") {
absoluteURL := urlUtils.AbsoluteURL(event.ProtocolTemp, event.TargetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
visitLink(event, c, absoluteURL)
}
}
// visitXMLLink checks if the collector should visit a link or not.
func visitXMLLink(link string, event *Event, e *colly.XMLElement, c *colly.Collector) {
if len(link) != 0 && !strings.HasPrefix(link, "data:image") {
absoluteURL := urlUtils.AbsoluteURL(event.ProtocolTemp, event.TargetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
visitLink(event, c, absoluteURL)
}
}
// visitLink is a protocol agnostic wrapper to visit a link.
func visitLink(event *Event, c *colly.Collector, absoluteURL string) {
if (!event.Intensive && urlUtils.SameDomain(event.ProtocolTemp+"://"+event.Target, absoluteURL)) ||
(event.Intensive && intensiveOk(event.TargetTemp, absoluteURL, event.Debug)) {
if !event.Ignore || (event.Ignore && !IgnoreMatch(absoluteURL, &event.IgnoreSlice)) {
err := c.Visit(absoluteURL)
if !errors.Is(err, colly.ErrAlreadyVisited) {
*event.URLs = append(*event.URLs, absoluteURL)
if err != nil && event.Debug {
log.Println(err)
}
}
}
}
}
// huntSecrets hunts for secrets.
func huntSecrets(target, body string, secretsFile *[]string) []scanner.SecretMatched {
secrets := SecretsMatch(target, body, secretsFile)
return secrets
}
// SecretsMatch checks if a body matches some secrets.
func SecretsMatch(url, body string, secretsFile *[]string) []scanner.SecretMatched {
var secrets []scanner.SecretMatched
if len(*secretsFile) == 0 {
for _, secret := range scanner.GetSecretRegexes() {
if matched, err := regexp.Match(secret.Regex, []byte(body)); err == nil && matched {
re := regexp.MustCompile(secret.Regex)
matches := re.FindAllStringSubmatch(body, -1)
// Avoiding false positives
var isFalsePositive = false
for _, match := range matches {
for _, falsePositive := range secret.FalsePositives {
if strings.Contains(strings.ToLower(match[0]), falsePositive) {
isFalsePositive = true
break
}
}
if !isFalsePositive {
secretFound := scanner.SecretMatched{Secret: secret, URL: url, Match: match[0]}
secrets = append(secrets, secretFound)
}
}
}
}
} else {
for _, secret := range *secretsFile {
if matched, err := regexp.Match(secret, []byte(body)); err == nil && matched {
re := regexp.MustCompile(secret)
matches := re.FindAllStringSubmatch(body, -1)
for _, match := range matches {
secretScanned := scanner.Secret{Name: "CustomFromFile", Description: "", Regex: secret, Poc: ""}
secretFound := scanner.SecretMatched{Secret: secretScanned, URL: url, Match: match[0]}
secrets = append(secrets, secretFound)
}
}
}
}
return scanner.RemoveDuplicateSecrets(secrets)
}
// huntEndpoints hunts for juicy endpoints.
func huntEndpoints(target string, endpointsFile *[]string) []scanner.EndpointMatched {
endpoints := EndpointsMatch(target, endpointsFile)
return endpoints
}
// EndpointsMatch check if an endpoint matches a juicy parameter.
func EndpointsMatch(target string, endpointsFile *[]string) []scanner.EndpointMatched {
endpoints := []scanner.EndpointMatched{}
matched := []scanner.Parameter{}
parameters := urlUtils.RetrieveParameters(target)
if len(*endpointsFile) == 0 {
for _, parameter := range scanner.GetJuicyParameters() {
for _, param := range parameters {
if strings.ToLower(param) == parameter.Parameter {
matched = append(matched, parameter)
}
}
}
endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target})
} else {
for _, parameter := range *endpointsFile {
for _, param := range parameters {
if param == parameter {
matched = append(matched, scanner.Parameter{Parameter: parameter, Attacks: []string{}})
}
}
}
endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target})
}
return endpoints
}
// huntExtensions hunts for extensions.
func huntExtensions(target string, severity int) scanner.FileTypeMatched {
extension := scanner.FileTypeMatched{}
copyTarget := target
for _, ext := range scanner.GetExtensions() {
if ext.Severity <= severity {
firstIndex := strings.Index(target, "?")
if firstIndex > -1 {
target = target[:firstIndex]
}
if strings.ToLower(target[len(target)-len("."+ext.Extension):]) == "."+ext.Extension {
extension = scanner.FileTypeMatched{Filetype: ext, URL: copyTarget}
}
}
}
return extension
}
// huntErrors hunts for errors.
func huntErrors(target, body string) []scanner.ErrorMatched {
errorsSlice := ErrorsMatch(target, body)
return errorsSlice
}
// ErrorsMatch checks the patterns for errors.
func ErrorsMatch(url, body string) []scanner.ErrorMatched {
errors := []scanner.ErrorMatched{}
for _, errorItem := range scanner.GetErrorRegexes() {
for _, errorRegex := range errorItem.Regex {
if matched, err := regexp.Match(errorRegex, []byte(body)); err == nil && matched {
re := regexp.MustCompile(errorRegex)
matches := re.FindAllStringSubmatch(body, -1)
for _, match := range matches {
errorFound := scanner.ErrorMatched{Error: errorItem, URL: url, Match: match[0]}
errors = append(errors, errorFound)
}
}
}
}
return scanner.RemoveDuplicateErrors(errors)
}
// huntInfos hunts for infos.
func huntInfos(target, body string) []scanner.InfoMatched {
infosSlice := InfoMatch(target, body)
return infosSlice
}
// InfoMatch checks the patterns for infos.
func InfoMatch(url, body string) []scanner.InfoMatched {
infos := []scanner.InfoMatched{}
for _, infoItem := range scanner.GetInfoRegexes() {
if matched, err := regexp.Match(infoItem.Regex, []byte(body)); err == nil && matched {
re := regexp.MustCompile(infoItem.Regex)
matches := re.FindAllStringSubmatch(body, -1)
for _, match := range matches {
infoFound := scanner.InfoMatched{Info: infoItem, URL: url, Match: match[0]}
infos = append(infos, infoFound)
}
}
}
return scanner.RemoveDuplicateInfos(infos)
}
// RetrieveBody retrieves the body (in the response) of a url.
func RetrieveBody(target *string) string {
sb, err := GetRequest(*target)
pkg/input/flags.go
+60 -28
View File
@@ -38,33 +38,62 @@ const (
// Input struct.
// It contains all the possible options.
type Input struct {
Version bool
Delay int
Concurrency int
Help bool
Examples bool
Plain bool
JSON bool
HTMLout string
TXTout string
Ignore string
IgnoreTXT string
Cache bool
Timeout int
Intensive bool
Rua bool
Proxy string
Secrets bool
SecretsFile string
Endpoints bool
// Version prints the version banner.
Version bool
// Delay between a page crawled and another.
Delay int
// Concurrency level.
Concurrency int
// Help prints the help banner.
Help bool
// Examples prints the examples banner.
Examples bool
// Plain prints only the results.
Plain bool
// JSON prints the output as JSON in stdout.
JSON bool
// HTMLout writes the output into an HTML file.
HTMLout string
// TXTout writes the output into an TXT file.
TXTout string
// Ignore ignores the URL containing at least one of the elements of this array.
Ignore string
// IgnoreTXT ignores the URL containing at least one of the lines of this file.
IgnoreTXT string
// Cache uses the .cariddi_cache folder as cache.
Cache bool
// Timeout set timeout for the requests. (default 10)
Timeout int
// Intensive crawls searching for resources matching 2nd level domain.
Intensive bool
// Rua uses a random browser user agent on every request.
Rua bool
// Proxy set a Proxy to be used (http and socks5 supported).
Proxy string
// Secrets hunts for secrets.
Secrets bool
// SecretsFile uses an external file (txt, one per line) to use custom regexes for secrets hunting.
SecretsFile string
// Endpoints hunts for juicy endpoints.
Endpoints bool
// EndpointsFile uses an external file (txt, one per line) to use custom parameters for endpoints hunting.
EndpointsFile string
Extensions int
Headers string
HeadersFile string
Errors bool
Info bool
Debug bool
UserAgent string
// Extensions hunts for juicy file extensions. Integer from 1(juicy) to 7(not juicy).
Extensions int
// Headers uses custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2".
Headers string
// HeadersFile reads from an external file custom headers (same format of headers flag).
HeadersFile string
// Errors hunts for errors in websites.
Errors bool
// Info hunts for useful informations in websites.
Info bool
// Debug prints debug information while crawling.
Debug bool
// UserAgent uses a custom User Agent.
UserAgent string
// StoreResp stores HTTP responses.
StoreResp bool
}
// ScanFlag defines all the options taken
@@ -76,7 +105,7 @@ func ScanFlag() Input {
concurrencyPtr := flag.Int("c", DefaultConcurrency, "Concurrency level.")
helpPtr := flag.Bool("h", false, "Print the help.")
examplesPtr := flag.Bool("examples", false, "Print the examples.")
plainPtr := flag.Bool("plain", false, "Print only the results.")
plainPtr := flag.Bool("plain", false, "Print only results.")
JSONPtr := flag.Bool("json", false, "Print the output as JSON in stdout.")
outputHTMLPtr := flag.String("oh", "", "Write the output into an HTML file.")
outputTXTPtr := flag.String("ot", "", "Write the output into a TXT file.")
@@ -86,7 +115,7 @@ func ScanFlag() Input {
timeoutPtr := flag.Int("t", TimeoutRequest, "Set timeout for the requests.")
intensivePtr := flag.Bool("intensive", false, "Crawl searching for resources matching 2nd level domain.")
ruaPtr := flag.Bool("rua", false, "Use a random browser user agent on every request.")
proxyPtr := flag.String("proxy", "", "Set a Proxy to be used (http and socks5 supported).")
proxyPtr := flag.String("proxy", "", "Set a Proxy, http and socks5 supported.")
secretsPtr := flag.Bool("s", false, "Hunt for secrets.")
secretsFilePtr := flag.String("sf", "", "Use an external file (txt, one per line)"+
@@ -111,6 +140,8 @@ func ScanFlag() Input {
userAgentPtr := flag.String("ua", "", "Use a custom User Agent.")
storeRespPtr := flag.Bool("sr", false, "Store HTTP responses.")
flag.Parse()
result := Input{
@@ -141,6 +172,7 @@ func ScanFlag() Input {
*infoPtr,
*debugPtr,
*userAgentPtr,
*storeRespPtr,
}
return result
pkg/output/beautify.go → pkg/output/banner.go
+3 -3
View File
@@ -35,7 +35,7 @@ import (
// nolint: checknoglobals
const (
version = "v1.3.1"
version = "v1.3.2"
banner = ` _ _ _ _
(_) | | | (_)
___ __ _ _ __ _ __| | __| |_
@@ -44,8 +44,8 @@ const (
\___\__,_|_| |_|\__,_|\__,_|_| `
)
// Beautify prints the banner + version.
func Beautify() {
// Banner prints the banner + version.
func Banner() {
links := " > github.com/edoardottt/cariddi\n > edoardoottavianelli.it\n"
sepLine := "========================================\n"
pkg/output/examples.go
+13 -11
View File
@@ -30,7 +30,7 @@ import "fmt"
// PrintExamples prints some examples.
func PrintExamples() {
Beautify()
Banner()
fmt.Println(`
cariddi -version (Print the version)
@@ -38,15 +38,15 @@ func PrintExamples() {
cariddi -examples (Print the examples)
cat urls | cariddi -e (Hunt for secrets)
cat urls | cariddi -s (Hunt for secrets)
cat urls | cariddi -d 2 (2 seconds between a page crawled and another)
cat urls | cariddi -c 200 (Set the concurrency level to 200)
cat urls | cariddi -s (Hunt for juicy endpoints)
cat urls | cariddi -e (Hunt for juicy endpoints)
cat urls | cariddi -plain (Print only useful things)
cat urls | cariddi -plain (Print only results)
cat urls | cariddi -ot target_name (Results in txt file)
@@ -72,17 +72,19 @@ func PrintExamples() {
cat urls | cariddi -proxy http://127.0.0.1:8080 (Set a Proxy to be used (http and socks5 supported))
cat urls | cariddi -headers "Cookie: auth=admin;type=2;; X-Custom: customHeader"
cat urls | cariddi -headers "Cookie: auth=admin;type=2;; X-Custom: customHeader (Use custom headers)"
cat urls | cariddi -headersfile headers.txt
cat urls | cariddi -headersfile headers.txt (Read from an external file custom headers)
cat urls | cariddi -err
cat urls | cariddi -err (Hunt for errors)
cat urls | cariddi -info
cat urls | cariddi -info (Hunt for useful information)
cat urls | cariddi -debug
cat urls | cariddi -debug (Print debug information)
cat urls | cariddi -ua "Custom User Agent"
cat urls | cariddi -ua "Custom User Agent" (Use a custom User Agent)
cat urls | cariddi -insecure`)
cat urls | cariddi -json (Print the output as JSON)
cat urls | cariddi -sr (Store HTTP responses)`)
}
pkg/output/help.go
+4 -2
View File
@@ -30,7 +30,7 @@ import "fmt"
// PrintHelp prints the help.
func PrintHelp() {
Beautify()
Banner()
fmt.Println(`Usage of cariddi:
-c int
Concurrency level. (default 20)
@@ -59,7 +59,7 @@ func PrintHelp() {
-i string
Ignore the URL containing at least one of the elements of this array.
-info
Hunt for useful informations in websites.
Hunt for useful information in websites.
-intensive
Crawl searching for resources matching 2nd level domain.
-it string
@@ -77,6 +77,8 @@ func PrintHelp() {
-s Hunt for secrets.
-sf string
Use an external file (txt, one per line) to use custom regexes for secrets hunting.
-sr
Store HTTP responses.
-t int
Set timeout for the requests. (default 10)
-ua
pkg/output/json.go → pkg/output/jsonl.go
+13 -13
View File
@@ -36,13 +36,13 @@ import (
)
type JSONData struct {
URL string `json:"url"`
Method string `json:"method"`
StatusCode int `json:"status_code"`
Words int `json:"words"`
Lines int `json:"lines"`
ContentType string `json:"content_type,omitempty"`
ContentLength int `json:"content_length,omitempty"`
URL string `json:"url"`
Method string `json:"method"`
StatusCode int `json:"status_code"`
Words int `json:"words"`
Lines int `json:"lines"`
ContentType string `json:"content_type,omitempty"`
ContentLength int `json:"content_length,omitempty"`
Matches *MatcherResults `json:"matches,omitempty"`
// Host string `json:"host"` # TODO: Available in Colly 2.x
}
@@ -141,19 +141,19 @@ func GetJSONString(
// Set empty data if no matches to bridge the omitempty gap for empty structs
var (
isFileTypeNill = false
isFileTypeNill = false
isParametersEmpty = len(parameters) == 0
isErrorsEmpty = len(errorList) == 0
isInfoEmpty = len(infoList) == 0
isSecretsEmpty = len(secretList) == 0
isErrorsEmpty = len(errorList) == 0
isInfoEmpty = len(infoList) == 0
isSecretsEmpty = len(secretList) == 0
)
if (*filetype == scanner.FileType{}){
if (*filetype == scanner.FileType{}) {
matcherResults.FileType = nil
isFileTypeNill = true
}
if (isFileTypeNill && isParametersEmpty && isErrorsEmpty && isInfoEmpty && isSecretsEmpty){
if isFileTypeNill && isParametersEmpty && isErrorsEmpty && isInfoEmpty && isSecretsEmpty {
resp.Matches = nil
}
pkg/output/json_test.go → pkg/output/jsonl_test.go
View File
pkg/output/output.go
+6 -2
View File
@@ -36,6 +36,10 @@ import (
"github.com/edoardottt/cariddi/pkg/scanner"
)
const (
CariddiOutputFolder = "output-cariddi"
)
// PrintSimpleOutput prints line by line.
func PrintSimpleOutput(out []string) {
for _, elem := range out {
@@ -48,7 +52,7 @@ func PrintSimpleOutput(out []string) {
func TxtOutput(flags input.Input, finalResults []string, finalSecret []scanner.SecretMatched,
finalEndpoints []scanner.EndpointMatched, finalExtensions []scanner.FileTypeMatched,
finalErrors []scanner.ErrorMatched, finalInfos []scanner.InfoMatched) {
exists, err := fileUtils.ElementExists("output-cariddi")
exists, err := fileUtils.ElementExists(CariddiOutputFolder)
if err != nil {
fmt.Println("Error while creating the output directory.")
os.Exit(1)
@@ -120,7 +124,7 @@ func TxtOutput(flags input.Input, finalResults []string, finalSecret []scanner.S
func HTMLOutput(flags input.Input, resultFilename string, finalResults []string, finalSecret []scanner.SecretMatched,
finalEndpoints []scanner.EndpointMatched, finalExtensions []scanner.FileTypeMatched,
finalErrors []scanner.ErrorMatched, finalInfos []scanner.InfoMatched) {
exists, err := fileUtils.ElementExists("output-cariddi")
exists, err := fileUtils.ElementExists(CariddiOutputFolder)
if err != nil {
fmt.Println("Error while creating the output directory.")
pkg/output/responses.go
+187
View File
@@ -0,0 +1,187 @@
/*
==========
Cariddi
==========
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see http://www.gnu.org/licenses/.
@Repository: https://github.com/edoardottt/cariddi
@Author: edoardottt, https://www.edoardoottavianelli.it
@License: https://github.com/edoardottt/cariddi/blob/main/LICENSE
*/
package output
import (
"bytes"
"crypto/sha1"
"encoding/hex"
"errors"
"fmt"
"io"
"log"
"net/url"
"os"
"path/filepath"
"strings"
"github.com/gocolly/colly"
fileUtils "github.com/edoardottt/cariddi/internal/file"
)
const (
index = "index.responses.txt"
)
var (
ErrHTTPResp = errors.New("cannot store HTTP response")
)
func getResponseHash(url string) string {
hash := sha1.Sum([]byte(url))
return hex.EncodeToString(hash[:])
}
// FormatResponse formats an HTTP response ready to be written in a file.
func FormatResponse(resp *colly.Response) ([]byte, error) {
builder := &bytes.Buffer{}
builder.WriteString(resp.Request.URL.String())
builder.WriteString("\n\n\n")
builder.WriteString(resp.Request.Method)
builder.WriteString(" ")
path := resp.Request.URL.Path
if resp.Request.URL.Fragment != "" {
path = path + "#" + resp.Request.URL.Fragment
}
builder.WriteString(path)
builder.WriteString(" ")
builder.WriteString("HTTP/1.1")
builder.WriteString("\n")
builder.WriteString("Host: " + resp.Request.URL.Host)
builder.WriteRune('\n')
for k, v := range *resp.Request.Headers {
builder.WriteString(k + ": " + strings.Join(v, "; ") + "\n")
}
if resp.Request.Body != nil {
bodyResp, _ := io.ReadAll(resp.Request.Body)
if string(bodyResp) != "" {
builder.WriteString("\n")
builder.WriteString(string(bodyResp))
}
}
builder.WriteString("\n\n")
builder.WriteString("HTTP/1.1")
builder.WriteString(" ")
builder.WriteString(fmt.Sprint(resp.StatusCode))
builder.WriteString("\n")
for k, v := range *resp.Headers {
builder.WriteString(k + ": " + strings.Join(v, "; ") + "\n")
}
builder.WriteString("\n")
body, _ := io.ReadAll(bytes.NewReader(resp.Body))
builder.WriteString(string(body))
return builder.Bytes(), nil
}
func getResponseFileName(folder, url string) string {
file := getResponseHash(url) + ".txt"
return filepath.Join(folder, file)
}
// UpdateIndex updates the index file with the
// correct information linking to HTTP responses files.
// If it fails returns an error.
func UpdateIndex(resp *colly.Response) error {
index, err := os.OpenFile(filepath.Join(CariddiOutputFolder, index),
os.O_APPEND|os.O_WRONLY,
fileUtils.Permission0644)
if err != nil {
return err
}
defer index.Close()
builder := &bytes.Buffer{}
builder.WriteString(getResponseFileName(filepath.Join(CariddiOutputFolder, resp.Request.URL.Host),
resp.Request.URL.String()))
builder.WriteRune(' ')
builder.WriteString(resp.Request.URL.String())
builder.WriteRune(' ')
builder.WriteString("(" + fmt.Sprint(resp.StatusCode) + ")")
builder.WriteRune('\n')
if _, writeErr := index.Write(builder.Bytes()); writeErr != nil {
return fmt.Errorf("%w %s", err, "could not update index")
}
return nil
}
// WriteHTTPResponse creates an HTTP response output file and
// writes the HTTP response inside it.
// If it fails returns an error.
func WriteHTTPResponse(inputURL *url.URL, response []byte) error {
file := getResponseFileName(filepath.Join(CariddiOutputFolder, inputURL.Host), inputURL.String())
outFile, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY, fileUtils.Permission0644)
if err != nil {
return err
}
if _, writeErr := outFile.Write(response); writeErr != nil {
return ErrHTTPResp
}
return nil
}
// StoreHTTPResponse stores an HTTP response in a file.
// If it fails returns an error.
func StoreHTTPResponse(r *colly.Response) error {
fileUtils.CreateHostOutputFolder(r.Request.URL.Host)
err := UpdateIndex(r)
if err != nil {
log.Println(err)
}
response, err := FormatResponse(r)
if err != nil {
log.Println(err)
}
err = WriteHTTPResponse(r.Request.URL, response)
if err != nil {
log.Println(err)
}
return nil
}
snapcraft.yaml
+1 -1
View File
@@ -2,7 +2,7 @@ name: cariddi
summary: Fast web crawler and scanner
description: |
Take a list of domains, crawl urls and scan for endpoints, secrets, api keys, file extensions, tokens and more
version: 1.3.1
version: 1.3.2
grade: stable
base: core20