From 603e2f072b9cdbc5f8577eeba2b0738a721a5e89 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sat, 11 Feb 2023 16:53:57 +0100 Subject: [PATCH 01/23] update main --- cmd/cariddi/main.go | 216 -------------------------------------------- go.mod | 2 +- go.sum | 15 --- 3 files changed, 1 insertion(+), 232 deletions(-) delete mode 100644 cmd/cariddi/main.go diff --git a/cmd/cariddi/main.go b/cmd/cariddi/main.go deleted file mode 100644 index a43891a..0000000 --- a/cmd/cariddi/main.go +++ /dev/null @@ -1,216 +0,0 @@ -/* -========== -Cariddi -========== - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses/. - - @Repository: https://github.com/edoardottt/cariddi - - @Author: edoardottt, https://www.edoardoottavianelli.it - - @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE - -*/ - -package main - -import ( - "os" - - fileUtils "github.com/edoardottt/cariddi/internal/file" - sliceUtils "github.com/edoardottt/cariddi/internal/slice" - "github.com/edoardottt/cariddi/pkg/crawler" - "github.com/edoardottt/cariddi/pkg/input" - "github.com/edoardottt/cariddi/pkg/output" - "github.com/edoardottt/cariddi/pkg/scanner" -) - -// main function. -func main() { - // Scan flags. - flags := input.ScanFlag() - - // Print version and exit. - if flags.Version { - output.Beautify() - os.Exit(0) - } - - // Print help and exit. - if flags.Help { - output.PrintHelp() - os.Exit(0) - } - - // Print examples and exit. - if flags.Examples { - output.PrintExamples() - os.Exit(0) - } - - // If it's possible print the cariddi banner. - if !flags.Plain { - output.Beautify() - } - - // Setup the config according to the flags that were - // passed via the CLI - config := &crawler.Scan{ - Delay: flags.Delay, - Concurrency: flags.Concurrency, - Ignore: flags.Ignore, - IgnoreTxt: flags.IgnoreTXT, - Cache: flags.Cache, - Timeout: flags.Timeout, - Intensive: flags.Intensive, - Rua: flags.Rua, - Proxy: flags.Proxy, - SecretsFlag: flags.Secrets, - Plain: flags.Plain, - EndpointsFlag: flags.Endpoints, - FileType: flags.Extensions, - ErrorsFlag: flags.Errors, - InfoFlag: flags.Info, - Debug: flags.Debug, - UserAgent: flags.UserAgent, - } - - // Read the targets from standard input. - targets := input.ScanTargets() - - // Check if there are errors in the flags definition. - input.CheckFlags(flags) - - // If it is needed, read custom endpoints definition - // from the specified file. - if flags.EndpointsFile != "" { - config.EndpointsSlice = fileUtils.ReadFile(flags.EndpointsFile) - } - - // If it is needed, read custom secrets definition - // from the specified file. - if flags.SecretsFile != "" { - config.SecretsSlice = fileUtils.ReadFile(flags.SecretsFile) - } - - finalResults := []string{} - finalSecret := []scanner.SecretMatched{} - finalEndpoints := []scanner.EndpointMatched{} - finalExtensions := []scanner.FileTypeMatched{} - finalErrors := []scanner.ErrorMatched{} - finalInfos := []scanner.InfoMatched{} - - // Create output files if needed (txt / html). - config.Txt = "" - if flags.TXT != "" { - config.Txt = fileUtils.CreateOutputFile(flags.TXT, "results", "txt") - } - - var ResultHTML = "" - if flags.HTML != "" { - ResultHTML = fileUtils.CreateOutputFile(flags.HTML, "", "html") - output.BannerHTML(ResultHTML) - output.HeaderHTML("Results", ResultHTML) - } - - // Read headers if needed - if flags.HeadersFile != "" || flags.Headers != "" { - var headersInput string - if flags.HeadersFile != "" { - headersInput = string(fileUtils.ReadEntireFile(flags.HeadersFile)) - } else { - headersInput = flags.Headers - } - - config.Headers = input.GetHeaders(headersInput) - } - - // For each target generate a crawler and collect all the results. - for _, target := range targets { - config.Target = target - results := crawler.New(config) - finalResults = append(finalResults, results.URLs...) - finalSecret = append(finalSecret, results.Secrets...) - finalEndpoints = append(finalEndpoints, results.Endpoints...) - finalExtensions = append(finalExtensions, results.Extensions...) - finalErrors = append(finalErrors, results.Errors...) - finalInfos = append(finalInfos, results.Infos...) - } - - // Remove duplicates from all the results. - finalResults = sliceUtils.RemoveDuplicateValues(finalResults) - finalSecret = scanner.RemoveDuplicateSecrets(finalSecret) - finalEndpoints = scanner.RemovDuplicateEndpoints(finalEndpoints) - finalExtensions = scanner.RemoveDuplicateExtensions(finalExtensions) - finalErrors = scanner.RemoveDuplicateErrors(finalErrors) - finalInfos = scanner.RemoveDuplicateInfos(finalInfos) - - // IF TXT OUTPUT > - if flags.TXT != "" { - output.TxtOutput(flags, finalResults, finalSecret, finalEndpoints, - finalExtensions, finalErrors, finalInfos) - } - - // IF HTML OUTPUT > - if flags.HTML != "" { - output.HTMLOutput(flags, ResultHTML, finalResults, finalSecret, - finalEndpoints, finalExtensions, finalErrors, finalInfos) - } - - // If needed print secrets. - if !flags.Plain && len(finalSecret) != 0 { - for _, elem := range finalSecret { - output.EncapsulateCustomGreen(elem.Secret.Name, elem.Match+" in "+elem.URL) - } - } - - // If needed print endpoints. - if !flags.Plain && len(finalEndpoints) != 0 { - for _, elem := range finalEndpoints { - for _, parameter := range elem.Parameters { - finalString := "" + parameter.Parameter - if len(parameter.Attacks) != 0 { - finalString += " -" - for _, attack := range parameter.Attacks { - finalString += " " + attack - } - } - - output.EncapsulateCustomGreen(finalString, " in "+elem.URL) - } - } - } - - // If needed print extensions. - if !flags.Plain && len(finalExtensions) != 0 { - for _, elem := range finalExtensions { - output.EncapsulateCustomGreen(elem.Filetype.Extension, elem.URL+" matched!") - } - } - - // If needed print errors. - if !flags.Plain && len(finalErrors) != 0 { - for _, elem := range finalErrors { - output.EncapsulateCustomGreen(elem.Error.ErrorName, elem.Match+" in "+elem.URL) - } - } - - // If needed print infos. - if !flags.Plain && len(finalInfos) != 0 { - for _, elem := range finalInfos { - output.EncapsulateCustomGreen(elem.Info.Name, elem.Match+" in "+elem.URL) - } - } -} diff --git a/go.mod b/go.mod index 16f47c7..54986e8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/edoardottt/cariddi -go 1.17 +go 1.18 require ( github.com/fatih/color v1.14.1 diff --git a/go.sum b/go.sum index acb71b0..6fb100d 100644 --- a/go.sum +++ b/go.sum @@ -41,46 +41,31 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= From f706165955093e5dfc5c784fbcfd9d271b27dc7e Mon Sep 17 00:00:00 2001 From: vrenzolaverace Date: Sat, 11 Feb 2023 16:56:28 +0100 Subject: [PATCH 02/23] Create main.go --- cmd/cariddi/main.go | 216 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 cmd/cariddi/main.go diff --git a/cmd/cariddi/main.go b/cmd/cariddi/main.go new file mode 100644 index 0000000..a43891a --- /dev/null +++ b/cmd/cariddi/main.go @@ -0,0 +1,216 @@ +/* +========== +Cariddi +========== + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see http://www.gnu.org/licenses/. + + @Repository: https://github.com/edoardottt/cariddi + + @Author: edoardottt, https://www.edoardoottavianelli.it + + @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE + +*/ + +package main + +import ( + "os" + + fileUtils "github.com/edoardottt/cariddi/internal/file" + sliceUtils "github.com/edoardottt/cariddi/internal/slice" + "github.com/edoardottt/cariddi/pkg/crawler" + "github.com/edoardottt/cariddi/pkg/input" + "github.com/edoardottt/cariddi/pkg/output" + "github.com/edoardottt/cariddi/pkg/scanner" +) + +// main function. +func main() { + // Scan flags. + flags := input.ScanFlag() + + // Print version and exit. + if flags.Version { + output.Beautify() + os.Exit(0) + } + + // Print help and exit. + if flags.Help { + output.PrintHelp() + os.Exit(0) + } + + // Print examples and exit. + if flags.Examples { + output.PrintExamples() + os.Exit(0) + } + + // If it's possible print the cariddi banner. + if !flags.Plain { + output.Beautify() + } + + // Setup the config according to the flags that were + // passed via the CLI + config := &crawler.Scan{ + Delay: flags.Delay, + Concurrency: flags.Concurrency, + Ignore: flags.Ignore, + IgnoreTxt: flags.IgnoreTXT, + Cache: flags.Cache, + Timeout: flags.Timeout, + Intensive: flags.Intensive, + Rua: flags.Rua, + Proxy: flags.Proxy, + SecretsFlag: flags.Secrets, + Plain: flags.Plain, + EndpointsFlag: flags.Endpoints, + FileType: flags.Extensions, + ErrorsFlag: flags.Errors, + InfoFlag: flags.Info, + Debug: flags.Debug, + UserAgent: flags.UserAgent, + } + + // Read the targets from standard input. + targets := input.ScanTargets() + + // Check if there are errors in the flags definition. + input.CheckFlags(flags) + + // If it is needed, read custom endpoints definition + // from the specified file. + if flags.EndpointsFile != "" { + config.EndpointsSlice = fileUtils.ReadFile(flags.EndpointsFile) + } + + // If it is needed, read custom secrets definition + // from the specified file. + if flags.SecretsFile != "" { + config.SecretsSlice = fileUtils.ReadFile(flags.SecretsFile) + } + + finalResults := []string{} + finalSecret := []scanner.SecretMatched{} + finalEndpoints := []scanner.EndpointMatched{} + finalExtensions := []scanner.FileTypeMatched{} + finalErrors := []scanner.ErrorMatched{} + finalInfos := []scanner.InfoMatched{} + + // Create output files if needed (txt / html). + config.Txt = "" + if flags.TXT != "" { + config.Txt = fileUtils.CreateOutputFile(flags.TXT, "results", "txt") + } + + var ResultHTML = "" + if flags.HTML != "" { + ResultHTML = fileUtils.CreateOutputFile(flags.HTML, "", "html") + output.BannerHTML(ResultHTML) + output.HeaderHTML("Results", ResultHTML) + } + + // Read headers if needed + if flags.HeadersFile != "" || flags.Headers != "" { + var headersInput string + if flags.HeadersFile != "" { + headersInput = string(fileUtils.ReadEntireFile(flags.HeadersFile)) + } else { + headersInput = flags.Headers + } + + config.Headers = input.GetHeaders(headersInput) + } + + // For each target generate a crawler and collect all the results. + for _, target := range targets { + config.Target = target + results := crawler.New(config) + finalResults = append(finalResults, results.URLs...) + finalSecret = append(finalSecret, results.Secrets...) + finalEndpoints = append(finalEndpoints, results.Endpoints...) + finalExtensions = append(finalExtensions, results.Extensions...) + finalErrors = append(finalErrors, results.Errors...) + finalInfos = append(finalInfos, results.Infos...) + } + + // Remove duplicates from all the results. + finalResults = sliceUtils.RemoveDuplicateValues(finalResults) + finalSecret = scanner.RemoveDuplicateSecrets(finalSecret) + finalEndpoints = scanner.RemovDuplicateEndpoints(finalEndpoints) + finalExtensions = scanner.RemoveDuplicateExtensions(finalExtensions) + finalErrors = scanner.RemoveDuplicateErrors(finalErrors) + finalInfos = scanner.RemoveDuplicateInfos(finalInfos) + + // IF TXT OUTPUT > + if flags.TXT != "" { + output.TxtOutput(flags, finalResults, finalSecret, finalEndpoints, + finalExtensions, finalErrors, finalInfos) + } + + // IF HTML OUTPUT > + if flags.HTML != "" { + output.HTMLOutput(flags, ResultHTML, finalResults, finalSecret, + finalEndpoints, finalExtensions, finalErrors, finalInfos) + } + + // If needed print secrets. + if !flags.Plain && len(finalSecret) != 0 { + for _, elem := range finalSecret { + output.EncapsulateCustomGreen(elem.Secret.Name, elem.Match+" in "+elem.URL) + } + } + + // If needed print endpoints. + if !flags.Plain && len(finalEndpoints) != 0 { + for _, elem := range finalEndpoints { + for _, parameter := range elem.Parameters { + finalString := "" + parameter.Parameter + if len(parameter.Attacks) != 0 { + finalString += " -" + for _, attack := range parameter.Attacks { + finalString += " " + attack + } + } + + output.EncapsulateCustomGreen(finalString, " in "+elem.URL) + } + } + } + + // If needed print extensions. + if !flags.Plain && len(finalExtensions) != 0 { + for _, elem := range finalExtensions { + output.EncapsulateCustomGreen(elem.Filetype.Extension, elem.URL+" matched!") + } + } + + // If needed print errors. + if !flags.Plain && len(finalErrors) != 0 { + for _, elem := range finalErrors { + output.EncapsulateCustomGreen(elem.Error.ErrorName, elem.Match+" in "+elem.URL) + } + } + + // If needed print infos. + if !flags.Plain && len(finalInfos) != 0 { + for _, elem := range finalInfos { + output.EncapsulateCustomGreen(elem.Info.Name, elem.Match+" in "+elem.URL) + } + } +} From c8b843f6dcfaad8b1c0c735e87fe96fa12909933 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sat, 11 Feb 2023 17:08:52 +0100 Subject: [PATCH 03/23] fix s3 rule --- pkg/scanner/secrets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scanner/secrets.go b/pkg/scanner/secrets.go index 81b49a9..bad7bf7 100644 --- a/pkg/scanner/secrets.go +++ b/pkg/scanner/secrets.go @@ -333,7 +333,7 @@ func GetSecretRegexes() []Secret { "S3 Bucket", `(?:[a-zA-Z0-9_-]+s3\.amazonaws\.com|[a-zA-Z0-9_.-]+amazonaws\.com|` + `[a-zA-Z0-9-\.\_]+\.s3\.amazonaws\.com|s3\:\/\/[a-zA-Z0-9-\.\_]+|` + - `s3-[a-zA-Z0-9-\.\_\/]+|s3\.amazonaws\.com/[a-zA-Z0-9-\.\_]+)`, + `s3\.amazonaws\.com/[a-zA-Z0-9-\.\_]+)`, []string{}, "?", }, From 339cd7b8960be5766cd9aa1eb107ae9608a2e5fe Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sun, 12 Feb 2023 11:59:36 +0100 Subject: [PATCH 04/23] refactor collector --- pkg/crawler/colly.go | 185 +++++++++++++++++++++---------------------- pkg/crawler/utils.go | 90 +++++++++++---------- 2 files changed, 138 insertions(+), 137 deletions(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index 1153bc9..fba1ce6 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -143,96 +143,19 @@ func New(scan *Scan) *Results { c := CreateColly(scan.Delay, scan.Concurrency, scan.Cache, scan.Timeout, scan.Intensive, scan.Rua, scan.Proxy, scan.UserAgent, scan.Target) - // On every request that Colly is making, print the URL it's currently visiting - c.OnRequest(func(e *colly.Request) { - fmt.Println(e.URL.String()) - }) + event := &Event{ + ProtocolTemp: protocolTemp, + TargetTemp: targetTemp, + Target: scan.Target, + Intensive: scan.Intensive, + Ignore: ignoreBool, + Debug: scan.Debug, + IgnoreSlice: ignoreSlice, + URLs: &results.URLs, + } - // On every a element which has href attribute call callback - c.OnHTML("a[href]", func(e *colly.HTMLElement) { - link := e.Attr("href") - if len(link) != 0 && link[0] != '#' { - visitHTMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - } - }) - - // On every script element which has src attribute call callback - c.OnHTML("script[src]", func(e *colly.HTMLElement) { - link := e.Attr("src") - visitHTMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - }) - - // On every link element which has href attribute call callback - c.OnHTML("link[href]", func(e *colly.HTMLElement) { - link := e.Attr("href") - visitHTMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - }) - - // On every iframe element which has src attribute call callback - c.OnHTML("iframe[src]", func(e *colly.HTMLElement) { - link := e.Attr("src") - visitHTMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - }) - - // On every svg element which has src attribute call callback - c.OnHTML("svg[src]", func(e *colly.HTMLElement) { - link := e.Attr("src") - visitHTMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - }) - - // On every img element which has src attribute call callback - c.OnHTML("img[src]", func(e *colly.HTMLElement) { - link := e.Attr("src") - visitHTMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - }) - - // On every from element which has action attribute call callback - c.OnHTML("form[action]", func(e *colly.HTMLElement) { - link := e.Attr("action") - visitHTMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - }) - - // Create a callback on the XPath query searching for the URLs - c.OnXML("//url", func(e *colly.XMLElement) { - link := e.Text - visitXMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - }) - - // Create a callback on the XPath query searching for the URLs - c.OnXML("//link", func(e *colly.XMLElement) { - link := e.Text - visitXMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - }) - - // Create a callback on the XPath query searching for the URLs - c.OnXML("//href", func(e *colly.XMLElement) { - link := e.Text - visitXMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - }) - - // Create a callback on the XPath query searching for the URLs - c.OnXML("//loc", func(e *colly.XMLElement) { - link := e.Text - visitXMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - }) - - // Create a callback on the XPath query searching for the URLs - c.OnXML("//fileurl", func(e *colly.XMLElement) { - link := e.Text - visitXMLLink(link, protocolTemp, targetTemp, scan.Target, scan.Intensive, - ignoreBool, scan.Debug, ignoreSlice, &results.URLs, e, c) - }) + registerHTMLEvents(c, event) + registerXMLEvents(c, event) // Add headers (if needed) on each request if (len(scan.Headers)) > 0 { @@ -253,12 +176,12 @@ func New(scan *Scan) *Results { (1 <= scan.FileType && scan.FileType <= 7) || scan.ErrorsFlag || scan.InfoFlag { // HERE SCAN FOR SECRETS if scan.SecretsFlag && lengthOk { - secretsSlice := huntSecrets(scan.SecretsSlice, r.Request.URL.String(), string(r.Body)) + secretsSlice := huntSecrets(r.Request.URL.String(), string(r.Body), &scan.SecretsSlice) results.Secrets = append(results.Secrets, secretsSlice...) } // HERE SCAN FOR ENDPOINTS if scan.EndpointsFlag { - endpointsSlice := huntEndpoints(scan.EndpointsSlice, r.Request.URL.String()) + endpointsSlice := huntEndpoints(r.Request.URL.String(), &scan.EndpointsSlice) for _, elem := range endpointsSlice { if len(elem.Parameters) != 0 { results.Endpoints = append(results.Endpoints, elem) @@ -300,7 +223,7 @@ func New(scan *Scan) *Results { if path == "" || path == "/" { absoluteURL = protocolTemp + "://" + scan.Target + addPath + "robots.txt" - if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) { + if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, &ignoreSlice)) { err = c.Visit(absoluteURL) if err != nil && scan.Debug && !errors.Is(err, colly.ErrAlreadyVisited) { log.Println(err) @@ -308,7 +231,7 @@ func New(scan *Scan) *Results { } absoluteURL = protocolTemp + "://" + scan.Target + addPath + "sitemap.xml" - if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) { + if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, &ignoreSlice)) { err = c.Visit(absoluteURL) if err != nil && scan.Debug && !errors.Is(err, colly.ErrAlreadyVisited) { log.Println(err) @@ -413,3 +336,79 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int, return c } + +// registerHTMLEvents registers the associated functions for each +// HTML event triggering an action. +func registerHTMLEvents(c *colly.Collector, event *Event) { + // On every request that Colly is making, print the URL it's currently visiting + c.OnRequest(func(e *colly.Request) { + fmt.Println(e.URL.String()) + }) + + // On every a element which has href attribute call callback + c.OnHTML("a[href]", func(e *colly.HTMLElement) { + link := e.Attr("href") + if len(link) != 0 && link[0] != '#' { + visitHTMLLink(link, event, e, c) + } + }) + + // On every script element which has src attribute call callback + c.OnHTML("script[src]", func(e *colly.HTMLElement) { + visitHTMLLink(e.Attr("src"), event, e, c) + }) + + // On every link element which has href attribute call callback + c.OnHTML("link[href]", func(e *colly.HTMLElement) { + visitHTMLLink(e.Attr("href"), event, e, c) + }) + + // On every iframe element which has src attribute call callback + c.OnHTML("iframe[src]", func(e *colly.HTMLElement) { + visitHTMLLink(e.Attr("src"), event, e, c) + }) + + // On every svg element which has src attribute call callback + c.OnHTML("svg[src]", func(e *colly.HTMLElement) { + visitHTMLLink(e.Attr("src"), event, e, c) + }) + + // On every img element which has src attribute call callback + c.OnHTML("img[src]", func(e *colly.HTMLElement) { + visitHTMLLink(e.Attr("src"), event, e, c) + }) + + // On every from element which has action attribute call callback + c.OnHTML("form[action]", func(e *colly.HTMLElement) { + visitHTMLLink(e.Attr("action"), event, e, c) + }) +} + +// registerXMLEvents registers the associated functions for each +// XML event triggering an action. +func registerXMLEvents(c *colly.Collector, event *Event) { + // Create a callback on the XPath query searching for the URLs + c.OnXML("//url", func(e *colly.XMLElement) { + visitXMLLink(e.Text, event, e, c) + }) + + // Create a callback on the XPath query searching for the URLs + c.OnXML("//link", func(e *colly.XMLElement) { + visitXMLLink(e.Text, event, e, c) + }) + + // Create a callback on the XPath query searching for the URLs + c.OnXML("//href", func(e *colly.XMLElement) { + visitXMLLink(e.Text, event, e, c) + }) + + // Create a callback on the XPath query searching for the URLs + c.OnXML("//loc", func(e *colly.XMLElement) { + visitXMLLink(e.Text, event, e, c) + }) + + // Create a callback on the XPath query searching for the URLs + c.OnXML("//fileurl", func(e *colly.XMLElement) { + visitXMLLink(e.Text, event, e, c) + }) +} diff --git a/pkg/crawler/utils.go b/pkg/crawler/utils.go index 988bc6d..ac20afa 100644 --- a/pkg/crawler/utils.go +++ b/pkg/crawler/utils.go @@ -12,46 +12,48 @@ import ( "github.com/gocolly/colly" ) +type Event struct { + ProtocolTemp string + TargetTemp string + Target string + Intensive bool + Ignore bool + Debug bool + IgnoreSlice []string + URLs *[]string +} + // visitHTMLLink checks if the collector should visit a link or not. -func visitHTMLLink(link, protocolTemp, targetTemp, target string, intensive, ignoreBool, debug bool, - ignoreSlice []string, finalResults *[]string, e *colly.HTMLElement, c *colly.Collector) { +func visitHTMLLink(link string, event *Event, e *colly.HTMLElement, c *colly.Collector) { if len(link) != 0 { - absoluteURL := urlUtils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)) + absoluteURL := urlUtils.AbsoluteURL(event.ProtocolTemp, event.TargetTemp, e.Request.AbsoluteURL(link)) // Visit link found on page // Only those links are visited which are in AllowedDomains - if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) || - (intensive && intensiveOk(targetTemp, absoluteURL, debug)) { - if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) { - err := c.Visit(absoluteURL) - if !errors.Is(err, colly.ErrAlreadyVisited) { - *finalResults = append(*finalResults, absoluteURL) - - if err != nil && debug { - log.Println(err) - } - } - } - } + visitLink(event, c, absoluteURL) } } // visitXMLLink checks if the collector should visit a link or not. -func visitXMLLink(link, protocolTemp, targetTemp, target string, intensive, ignoreBool, debug bool, - ignoreSlice []string, finalResults *[]string, e *colly.XMLElement, c *colly.Collector) { +func visitXMLLink(link string, event *Event, e *colly.XMLElement, c *colly.Collector) { if len(link) != 0 { - absoluteURL := urlUtils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link)) + absoluteURL := urlUtils.AbsoluteURL(event.ProtocolTemp, event.TargetTemp, e.Request.AbsoluteURL(link)) // Visit link found on page // Only those links are visited which are in AllowedDomains - if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) || - (intensive && intensiveOk(targetTemp, absoluteURL, debug)) { - if !ignoreBool || (ignoreBool && !IgnoreMatch(absoluteURL, ignoreSlice)) { - err := c.Visit(absoluteURL) - if !errors.Is(err, colly.ErrAlreadyVisited) { - *finalResults = append(*finalResults, absoluteURL) + visitLink(event, c, absoluteURL) + } +} - if err != nil && debug { - log.Println(err) - } +// visitLink is a protocol agnostic wrapper to visit a link. +func visitLink(event *Event, c *colly.Collector, absoluteURL string) { + if (!event.Intensive && urlUtils.SameDomain(event.ProtocolTemp+"://"+event.Target, absoluteURL)) || + (event.Intensive && intensiveOk(event.TargetTemp, absoluteURL, event.Debug)) { + if !event.Ignore || (event.Ignore && !IgnoreMatch(absoluteURL, &event.IgnoreSlice)) { + err := c.Visit(absoluteURL) + if !errors.Is(err, colly.ErrAlreadyVisited) { + *event.URLs = append(*event.URLs, absoluteURL) + + if err != nil && event.Debug { + log.Println(err) } } } @@ -59,16 +61,16 @@ func visitXMLLink(link, protocolTemp, targetTemp, target string, intensive, igno } // huntSecrets hunts for secrets. -func huntSecrets(secretsFile []string, target string, body string) []scanner.SecretMatched { +func huntSecrets(target, body string, secretsFile *[]string) []scanner.SecretMatched { secrets := SecretsMatch(target, body, secretsFile) return secrets } // SecretsMatch checks if a body matches some secrets. -func SecretsMatch(url string, body string, secretsFile []string) []scanner.SecretMatched { +func SecretsMatch(url, body string, secretsFile *[]string) []scanner.SecretMatched { var secrets []scanner.SecretMatched - if len(secretsFile) == 0 { + if len(*secretsFile) == 0 { for _, secret := range scanner.GetSecretRegexes() { if matched, err := regexp.Match(secret.Regex, []byte(body)); err == nil && matched { re := regexp.MustCompile(secret.Regex) @@ -91,7 +93,7 @@ func SecretsMatch(url string, body string, secretsFile []string) []scanner.Secre } } } else { - for _, secret := range secretsFile { + for _, secret := range *secretsFile { if matched, err := regexp.Match(secret, []byte(body)); err == nil && matched { re := regexp.MustCompile(secret) match := re.FindStringSubmatch(body) @@ -106,18 +108,18 @@ func SecretsMatch(url string, body string, secretsFile []string) []scanner.Secre } // huntEndpoints hunts for juicy endpoints. -func huntEndpoints(endpointsFile []string, target string) []scanner.EndpointMatched { +func huntEndpoints(target string, endpointsFile *[]string) []scanner.EndpointMatched { endpoints := EndpointsMatch(target, endpointsFile) return endpoints } // EndpointsMatch check if an endpoint matches a juicy parameter. -func EndpointsMatch(target string, endpointsFile []string) []scanner.EndpointMatched { +func EndpointsMatch(target string, endpointsFile *[]string) []scanner.EndpointMatched { endpoints := []scanner.EndpointMatched{} matched := []scanner.Parameter{} parameters := urlUtils.RetrieveParameters(target) - if len(endpointsFile) == 0 { + if len(*endpointsFile) == 0 { for _, parameter := range scanner.GetJuicyParameters() { for _, param := range parameters { if strings.ToLower(param) == parameter.Parameter { @@ -127,7 +129,7 @@ func EndpointsMatch(target string, endpointsFile []string) []scanner.EndpointMat } } } else { - for _, parameter := range endpointsFile { + for _, parameter := range *endpointsFile { for _, param := range parameters { if param == parameter { matched = append(matched, scanner.Parameter{Parameter: parameter, Attacks: []string{}}) @@ -162,13 +164,13 @@ func huntExtensions(target string, severity int) scanner.FileTypeMatched { } // huntErrors hunts for errors. -func huntErrors(target string, body string) []scanner.ErrorMatched { +func huntErrors(target, body string) []scanner.ErrorMatched { errorsSlice := ErrorsMatch(target, body) return errorsSlice } // ErrorsMatch checks the patterns for errors. -func ErrorsMatch(url string, body string) []scanner.ErrorMatched { +func ErrorsMatch(url, body string) []scanner.ErrorMatched { errors := []scanner.ErrorMatched{} for _, errorItem := range scanner.GetErrorRegexes() { @@ -186,13 +188,13 @@ func ErrorsMatch(url string, body string) []scanner.ErrorMatched { } // huntInfos hunts for infos. -func huntInfos(target string, body string) []scanner.InfoMatched { +func huntInfos(target, body string) []scanner.InfoMatched { infosSlice := InfoMatch(target, body) return infosSlice } // InfoMatch checks the patterns for infos. -func InfoMatch(url string, body string) []scanner.InfoMatched { +func InfoMatch(url, body string) []scanner.InfoMatched { infos := []scanner.InfoMatched{} for _, infoItem := range scanner.GetInfoRegexes() { @@ -210,8 +212,8 @@ func InfoMatch(url string, body string) []scanner.InfoMatched { } // RetrieveBody retrieves the body (in the response) of a url. -func RetrieveBody(target string) string { - sb, err := GetRequest(target) +func RetrieveBody(target *string) string { + sb, err := GetRequest(*target) if err == nil && sb != "" { return sb } @@ -220,8 +222,8 @@ func RetrieveBody(target string) string { } // IgnoreMatch checks if the URL should be ignored or not. -func IgnoreMatch(url string, ignoreSlice []string) bool { - for _, ignore := range ignoreSlice { +func IgnoreMatch(url string, ignoreSlice *[]string) bool { + for _, ignore := range *ignoreSlice { if strings.Contains(url, ignore) { return true } From 37c71bbfefe38c793ab3878906252acb6a579315 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sun, 12 Feb 2023 15:56:24 +0100 Subject: [PATCH 05/23] fix false positives --- pkg/scanner/secrets.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/scanner/secrets.go b/pkg/scanner/secrets.go index bad7bf7..bc24304 100644 --- a/pkg/scanner/secrets.go +++ b/pkg/scanner/secrets.go @@ -86,14 +86,14 @@ func GetSecretRegexes() []Secret { "Facebook Secret Key", "Facebook Secret Key", `(?i)(facebook|fb)(.{0,20})?(?-i)['\"][0-9a-f]{32}['\"]`, - []string{"facebook.com/", "facebook.svg"}, + []string{"facebook.com", "facebook.svg"}, "?", }, { "Facebook Client ID", "Facebook Client ID", `(?i)(facebook|fb)(.{0,20})?['\"][0-9]{13,17}['\"]`, - []string{"facebook.com/", "facebook.svg"}, + []string{"facebook.com", "facebook.svg"}, "?", }, { @@ -114,14 +114,14 @@ func GetSecretRegexes() []Secret { "Twitter Secret Key", "Twitter Secret Key", `(?i)twitter(.{0,20})?[0-9a-z]{35,44}`, - []string{}, + []string{"twitter.com"}, "?", }, { "Twitter Client ID", "Twitter Client ID", `(?i)twitter(.{0,20})?[0-9a-z]{18,25}`, - []string{}, + []string{"twitter.com"}, "?", }, { @@ -156,14 +156,14 @@ func GetSecretRegexes() []Secret { "LinkedIn Client ID", "LinkedIn Client ID", `(?i)linkedin(.{0,20})?(?-i)[0-9a-z]{12}`, - []string{"linkedin.com/", "linkedin.svg"}, + []string{"linkedin.com", "linkedin.svg"}, "?", }, { "LinkedIn Secret Key", "LinkedIn Secret Key", `(?i)linkedin(.{0,20})?[0-9a-z]{16}`, - []string{"linkedin.com/", "linkedin.svg"}, + []string{"linkedin.com", "linkedin.svg"}, "?", }, { From b965684213a14e61c2e2c6be2adfda31f488efba Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sun, 12 Feb 2023 15:57:29 +0100 Subject: [PATCH 06/23] v1.3.0 --- pkg/output/beautify.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/output/beautify.go b/pkg/output/beautify.go index 9fae40f..157bac1 100644 --- a/pkg/output/beautify.go +++ b/pkg/output/beautify.go @@ -38,7 +38,7 @@ func Beautify() { banner2 := " ___ __ _ _ __(_) __| | __| (_)\n" banner3 := " / __/ _` | '__| |/ _` |/ _` | |\n" banner4 := " | (_| (_| | | | | (_| | (_| | |\n" - banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.2.1\n" + banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.3.0\n" banner6 := "" banner7 := " > github.com/edoardottt/cariddi\n" banner8 := " > edoardoottavianelli.it\n" From 23a81450322a695c1d22d889091b04d5812dc778 Mon Sep 17 00:00:00 2001 From: vrenzolaverace Date: Mon, 13 Feb 2023 12:30:19 +0100 Subject: [PATCH 07/23] fix twitter link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 87dd13a..cd28028 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@
- Share on Twitter! + Share on Twitter!

From e540f65d1fd256ee1bc074c1d3055e45b96bcbfb Mon Sep 17 00:00:00 2001 From: vrenzolaverace Date: Mon, 13 Feb 2023 17:43:56 +0100 Subject: [PATCH 08/23] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cd28028..56cba0a 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,11 @@ Preview :bar_chart: ---------- -[![asciicast](https://asciinema.org/a/415989.svg)](https://asciinema.org/a/415989) + + +

+ +

Installation 📡 ---------- From 02c4110c05de0cbb59aad5143b65b48597d76fdc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Feb 2023 03:55:16 +0000 Subject: [PATCH 09/23] chore(deps): bump golang.org/x/net from 0.4.0 to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.4.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.4.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 54986e8..6f7a1f8 100644 --- a/go.mod +++ b/go.mod @@ -21,9 +21,9 @@ require ( github.com/mattn/go-isatty v0.0.17 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/temoto/robotstxt v1.1.2 // indirect - golang.org/x/net v0.4.0 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/text v0.5.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index 6fb100d..0c8cb4a 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -55,16 +55,16 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 846d942fa168aa396d23534e76d3c2757468d598 Mon Sep 17 00:00:00 2001 From: vrenzolaverace Date: Sun, 19 Feb 2023 12:20:04 +0100 Subject: [PATCH 10/23] Create snapcraft.yaml --- snapcraft.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 snapcraft.yaml diff --git a/snapcraft.yaml b/snapcraft.yaml new file mode 100644 index 0000000..1a236b3 --- /dev/null +++ b/snapcraft.yaml @@ -0,0 +1,20 @@ +name: cariddi +summary: Fast web crawler and scanner +description: | + Take a list of domains, crawl urls and scan for endpoints, secrets, api keys, file extensions, tokens and more +version: git +grade: stable +base: core20 + +confinement: devmode + +apps: + cariddi: + command: bin/cariddi + plugs: + - home +parts: + cariddi: + plugin: go + source-type: git + source: https://github.com/edoardottt/cariddi From 171dc997ebc3b35fa8bbd76aa5f6778e0f03a314 Mon Sep 17 00:00:00 2001 From: vrenzolaverace Date: Sun, 19 Feb 2023 12:36:32 +0100 Subject: [PATCH 11/23] Delete snapcraft.yaml --- snapcraft.yaml | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 snapcraft.yaml diff --git a/snapcraft.yaml b/snapcraft.yaml deleted file mode 100644 index 1a236b3..0000000 --- a/snapcraft.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: cariddi -summary: Fast web crawler and scanner -description: | - Take a list of domains, crawl urls and scan for endpoints, secrets, api keys, file extensions, tokens and more -version: git -grade: stable -base: core20 - -confinement: devmode - -apps: - cariddi: - command: bin/cariddi - plugs: - - home -parts: - cariddi: - plugin: go - source-type: git - source: https://github.com/edoardottt/cariddi From 9c2076016eaa44588a070b320f7ac53d0fb72f06 Mon Sep 17 00:00:00 2001 From: vrenzolaverace Date: Sun, 19 Feb 2023 12:53:34 +0100 Subject: [PATCH 12/23] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 56cba0a..36dfef2 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,11 @@ Preview :bar_chart: Installation 📡 ---------- +### Using Snap +```bash +sudo snap install cariddi +``` + ### Using Go ```bash go install -v github.com/edoardottt/cariddi/cmd/cariddi@latest From 8a8c8f7cf84230d3a6928041c74954975e76d210 Mon Sep 17 00:00:00 2001 From: vrenzolaverace Date: Sun, 19 Feb 2023 12:54:28 +0100 Subject: [PATCH 13/23] Create snapcraft.yaml --- snapcraft.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 snapcraft.yaml diff --git a/snapcraft.yaml b/snapcraft.yaml new file mode 100644 index 0000000..59f0e91 --- /dev/null +++ b/snapcraft.yaml @@ -0,0 +1,20 @@ +name: cariddi +summary: Fast web crawler and scanner +description: | + Take a list of domains, crawl urls and scan for endpoints, secrets, api keys, file extensions, tokens and more +version: 1.3.0 +grade: stable +base: core20 + +confinement: strict + +apps: + cariddi: + command: bin/cariddi + plugs: + - home +parts: + cariddi: + plugin: go + source-type: git + source: https://github.com/edoardottt/cariddi From 4b34049c8b9e35677229fd716eebbafebcfd4e9e Mon Sep 17 00:00:00 2001 From: edoardottt Date: Tue, 21 Feb 2023 14:34:15 +0100 Subject: [PATCH 14/23] fix snap --- snapcraft.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/snapcraft.yaml b/snapcraft.yaml index 59f0e91..45006dd 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -13,6 +13,8 @@ apps: command: bin/cariddi plugs: - home + - network + parts: cariddi: plugin: go From c3c0d4dc7514a0a19e8101d164facdafb463cb9d Mon Sep 17 00:00:00 2001 From: vrenzolaverace Date: Wed, 22 Feb 2023 07:39:07 +0100 Subject: [PATCH 15/23] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a4156f5..94300f1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ cariddi output-cariddi/ .cariddi_cache/ +*.snap # Binaries for programs and plugins *.exe From 5e47143f0be0fee16dbe0f8d687b46fbe07332aa Mon Sep 17 00:00:00 2001 From: vrenzolaverace Date: Wed, 22 Feb 2023 08:12:50 +0100 Subject: [PATCH 16/23] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 36dfef2..b6ec701 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ go-report-card - + workflows From 59fc232d6511050ff987894bf8df662daaa1b652 Mon Sep 17 00:00:00 2001 From: vrenzolaverace Date: Mon, 27 Feb 2023 08:25:24 +0100 Subject: [PATCH 17/23] Create auto_assign.yml --- .github/auto_assign.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/auto_assign.yml diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml new file mode 100644 index 0000000..fe4b20a --- /dev/null +++ b/.github/auto_assign.yml @@ -0,0 +1,14 @@ +# Set to true to add reviewers to pull requests +addReviewers: true + +# A list of reviewers to be added to pull requests (GitHub user name) +reviewers: + - edoardottt + +# A list of keywords to be skipped the process that add reviewers if pull requests include it +skipKeywords: + - wip + +# A number of reviewers added to the pull request +# Set 0 to add all the reviewers (default: 0) +numberOfReviewers: 0 From fc1455efa7d500ec8709e7d4ca3f7583e36c6ea8 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Mon, 27 Feb 2023 19:10:35 +0100 Subject: [PATCH 18/23] Add 5 new secrets --- pkg/scanner/secrets.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pkg/scanner/secrets.go b/pkg/scanner/secrets.go index bc24304..db2b000 100644 --- a/pkg/scanner/secrets.go +++ b/pkg/scanner/secrets.go @@ -337,6 +337,41 @@ func GetSecretRegexes() []Secret { []string{}, "?", }, + { + "Discord Webhook", + "Discord Webhook", + `https\:\/\/discordapp\.com\/api\/webhooks\/[0-9]+/[A-Za-z0-9\-]+`, + []string{}, + "?", + }, + { + "Google Calendar URI", + "Google Calendar URI", + `https\:\/\/(.*)calendar\.google\.com\/calendar\/[0-9a-z\/]+\/embed\?src=[A-Za-z0-9%@&;=\-_\.\/]+`, + []string{}, + "?", + }, + { + "Google OAuth Access Key", + "Google OAuth Access Key", + `ya29\.[0-9A-Za-z\-_]+`, + []string{}, + "?", + }, + { + "Mapbox Token Disclosure", + "Mapbox Token Disclosure", + `(pk|sk)\.eyJ1Ijoi\w+\.[\w-]*`, + []string{}, + "?", + }, + { + "Microsoft Teams Webhook", + "Microsoft Teams Webhook", + `https\:\/\/outlook\.office\.com\/webhook\/[A-Za-z0-9\-@]+\/IncomingWebhook\/[A-Za-z0-9\-]+\/[A-Za-z0-9\-]+`, + []string{}, + "?", + }, } return regexes From 855253c240457933b2b905a0a99b4fe5ed475e5b Mon Sep 17 00:00:00 2001 From: edoardottt Date: Mon, 27 Feb 2023 19:18:04 +0100 Subject: [PATCH 19/23] Fix data image protocol link --- pkg/crawler/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/crawler/utils.go b/pkg/crawler/utils.go index ac20afa..fab0f1e 100644 --- a/pkg/crawler/utils.go +++ b/pkg/crawler/utils.go @@ -25,7 +25,7 @@ type Event struct { // visitHTMLLink checks if the collector should visit a link or not. func visitHTMLLink(link string, event *Event, e *colly.HTMLElement, c *colly.Collector) { - if len(link) != 0 { + if len(link) != 0 && !strings.HasPrefix(link, "data:image") { absoluteURL := urlUtils.AbsoluteURL(event.ProtocolTemp, event.TargetTemp, e.Request.AbsoluteURL(link)) // Visit link found on page // Only those links are visited which are in AllowedDomains @@ -35,7 +35,7 @@ func visitHTMLLink(link string, event *Event, e *colly.HTMLElement, c *colly.Col // visitXMLLink checks if the collector should visit a link or not. func visitXMLLink(link string, event *Event, e *colly.XMLElement, c *colly.Collector) { - if len(link) != 0 { + if len(link) != 0 && !strings.HasPrefix(link, "data:image") { absoluteURL := urlUtils.AbsoluteURL(event.ProtocolTemp, event.TargetTemp, e.Request.AbsoluteURL(link)) // Visit link found on page // Only those links are visited which are in AllowedDomains From 88d2e9333a762a5c58e3c7c7a22395cecd831b3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Mar 2023 15:05:32 +0000 Subject: [PATCH 20/23] chore(deps): bump github.com/fatih/color from 1.14.1 to 1.15.0 Bumps [github.com/fatih/color](https://github.com/fatih/color) from 1.14.1 to 1.15.0. - [Release notes](https://github.com/fatih/color/releases) - [Commits](https://github.com/fatih/color/compare/v1.14.1...v1.15.0) --- updated-dependencies: - dependency-name: github.com/fatih/color dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 6f7a1f8..1216819 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/edoardottt/cariddi go 1.18 require ( - github.com/fatih/color v1.14.1 + github.com/fatih/color v1.15.0 github.com/gocolly/colly v1.2.0 ) @@ -22,7 +22,7 @@ require ( github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/temoto/robotstxt v1.1.2 // indirect golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect + golang.org/x/sys v0.6.0 // indirect golang.org/x/text v0.7.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index 0c8cb4a..3aeffa3 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/antchfx/xpath v1.2.1 h1:qhp4EW6aCOVr5XIkT+l6LJ9ck/JsUH/yyauNgTQkBF8= github.com/antchfx/xpath v1.2.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= @@ -55,8 +55,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 4f40416f8cca96a8d52a03133df18f3a13f7afc7 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 21 Mar 2023 10:41:41 +0000 Subject: [PATCH 21/23] update event structure to add JSON bool flag --- go.mod | 18 +++++++++--------- go.sum | 39 +++++++++++++++++++++++++++++++++++++++ pkg/crawler/colly.go | 5 ++++- pkg/crawler/utils.go | 1 + 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 1216819..373ee40 100644 --- a/go.mod +++ b/go.mod @@ -8,22 +8,22 @@ require ( ) require ( - github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect - github.com/antchfx/htmlquery v1.2.5 // indirect - github.com/antchfx/xmlquery v1.3.13 // indirect - github.com/antchfx/xpath v1.2.1 // indirect + github.com/antchfx/htmlquery v1.3.0 // indirect + github.com/antchfx/xmlquery v1.3.15 // indirect + github.com/antchfx/xpath v1.2.4 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect - github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/temoto/robotstxt v1.1.2 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/text v0.8.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.30.0 // indirect ) diff --git a/go.sum b/go.sum index 3aeffa3..3a6b9a3 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,22 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/antchfx/htmlquery v1.2.5 h1:1lXnx46/1wtv1E/kzmH8vrfMuUKYgkdDBA9pIdMJnk4= github.com/antchfx/htmlquery v1.2.5/go.mod h1:2MCVBzYVafPBmKbrmwB9F5xdd+IEgRY61ci2oOsOQVw= +github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= +github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= github.com/antchfx/xmlquery v1.3.13 h1:wqhTv2BN5MzYg9rnPVtZb3IWP8kW6WV/ebAY0FCTI7Y= github.com/antchfx/xmlquery v1.3.13/go.mod h1:3w2RvQvTz+DaT5fSgsELkSJcdNgkmg6vuXDEuhdwsPQ= +github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8Tw= +github.com/antchfx/xmlquery v1.3.15/go.mod h1:zMDv5tIGjOxY/JCNNinnle7V/EwthZ5IT8eeCGJKRWA= github.com/antchfx/xpath v1.2.1 h1:qhp4EW6aCOVr5XIkT+l6LJ9ck/JsUH/yyauNgTQkBF8= github.com/antchfx/xpath v1.2.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= +github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= @@ -23,6 +32,8 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= @@ -36,36 +47,62 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= @@ -74,3 +111,5 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index b5356e0..0e9b4ac 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -151,6 +151,7 @@ func New(scan *Scan) *Results { Intensive: scan.Intensive, Ignore: ignoreBool, Debug: scan.Debug, + JSON: scan.JSON, IgnoreSlice: ignoreSlice, URLs: &results.URLs, } @@ -362,7 +363,9 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int, func registerHTMLEvents(c *colly.Collector, event *Event) { // On every request that Colly is making, print the URL it's currently visiting c.OnRequest(func(e *colly.Request) { - fmt.Println(e.URL.String()) + if (!event.JSON){ + fmt.Println(e.URL.String()) + } }) // On every a element which has href attribute call callback diff --git a/pkg/crawler/utils.go b/pkg/crawler/utils.go index 484f25b..7854162 100644 --- a/pkg/crawler/utils.go +++ b/pkg/crawler/utils.go @@ -19,6 +19,7 @@ type Event struct { Intensive bool Ignore bool Debug bool + JSON bool IgnoreSlice []string URLs *[]string } From 03d9ce059bfeb1622b5780faaf886340fff75b6e Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 21 Mar 2023 14:10:16 +0000 Subject: [PATCH 22/23] fix linting --- pkg/crawler/colly.go | 2 +- pkg/output/beautify.go | 4 ++-- pkg/output/json.go | 24 ++++++++++++++---------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index 0e9b4ac..986316e 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -219,7 +219,7 @@ func New(scan *Scan) *Results { } } if scan.JSON { - jsonOutput, err := output.GetJsonString( + jsonOutput, err := output.GetJSONString( r, secrets, parameters, filetype, errors, infos, ) if err == nil { diff --git a/pkg/output/beautify.go b/pkg/output/beautify.go index bb59bd3..29ed3f7 100644 --- a/pkg/output/beautify.go +++ b/pkg/output/beautify.go @@ -49,7 +49,7 @@ func Beautify() { bannerPart2 := banner6 + banner7 + banner8 + banner9 color.Set(color.FgCyan) - fmt.Fprintf(os.Stderr, bannerPart1) + fmt.Fprint(os.Stderr, bannerPart1) color.Unset() - fmt.Fprintf(os.Stderr, bannerPart2) + fmt.Fprint(os.Stderr, bannerPart2) } diff --git a/pkg/output/json.go b/pkg/output/json.go index 673d8ff..5b3e269 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -35,7 +35,7 @@ import ( "github.com/gocolly/colly" ) -type JsonData struct { +type JSONData struct { URL string `json:"url"` Method string `json:"method"` StatusCode int `json:"status_code"` @@ -60,7 +60,7 @@ type MatcherResult struct { Match string `json:"match"` } -func GetJsonString( +func GetJSONString( r *colly.Response, secrets []scanner.SecretMatched, parameters []scanner.Parameter, @@ -68,22 +68,28 @@ func GetJsonString( errors []scanner.ErrorMatched, infos []scanner.InfoMatched, ) ([]byte, error) { - // Parse response headers headers := r.Headers contentTypes := (*headers)["Content-Type"] + contentLengths := (*headers)["Content-Length"] contentType := "" + contentLength := 0 + errorList := []MatcherResult{} + infoList := []MatcherResult{} + secretList := []MatcherResult{} + + // Set content type if len(contentTypes) > 0 { contentType = contentTypes[0] } - contentLength := 0 - contentLengths := (*headers)["Content-Length"] + // Set content length if len(contentLengths) > 0 { ret, err := strconv.Atoi(contentLengths[0]) if err != nil { return nil, err } + contentLength = ret } @@ -94,21 +100,19 @@ func GetJsonString( lines := len(strings.Split(string(r.Body), "\n")) // Process secrets - secretList := []MatcherResult{} for _, secret := range secrets { secretMatch := MatcherResult{secret.Secret.Name, secret.Match} secretList = append(secretList, secretMatch) } // Process infos - infoList := []MatcherResult{} + for _, info := range infos { secretMatch := MatcherResult{info.Info.Name, info.Match} infoList = append(infoList, secretMatch) } - // Process - errorList := []MatcherResult{} + // Process errors for _, error := range errors { errorMatch := MatcherResult{error.Error.ErrorName, error.Match} errorList = append(errorList, errorMatch) @@ -122,7 +126,7 @@ func GetJsonString( Infos: infoList, Secrets: secretList, } - resp := &JsonData{ + resp := &JSONData{ URL: r.Request.URL.String(), Method: r.Request.Method, StatusCode: r.StatusCode, From 1d28243dab462f122a6811e34823aba5f1191c70 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 21 Mar 2023 14:20:44 +0000 Subject: [PATCH 23/23] split content-type to remove charsets --- pkg/output/json.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/output/json.go b/pkg/output/json.go index 5b3e269..7122d07 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -78,9 +78,9 @@ func GetJSONString( infoList := []MatcherResult{} secretList := []MatcherResult{} - // Set content type + // Set content type if len(contentTypes) > 0 { - contentType = contentTypes[0] + contentType = strings.Split(contentTypes[0], "; ")[0] } // Set content length @@ -106,7 +106,6 @@ func GetJSONString( } // Process infos - for _, info := range infos { secretMatch := MatcherResult{info.Info.Name, info.Match} infoList = append(infoList, secretMatch)