From a7e56fd1b1a82501c3ced90ef73e5f092ee21834 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Thu, 22 Jun 2023 10:02:02 +0200 Subject: [PATCH 1/8] JSON lines aggregate results #115 --- pkg/output/jsonl.go | 95 +++++++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/pkg/output/jsonl.go b/pkg/output/jsonl.go index 7acdcc3..6a87a72 100644 --- a/pkg/output/jsonl.go +++ b/pkg/output/jsonl.go @@ -50,16 +50,12 @@ type JSONData struct { type MatcherResults struct { FileType *scanner.FileType `json:"filetype,omitempty"` Parameters []scanner.Parameter `json:"parameters,omitempty"` - Errors []MatcherResult `json:"errors,omitempty"` - Infos []MatcherResult `json:"infos,omitempty"` - Secrets []MatcherResult `json:"secrets,omitempty"` -} - -type MatcherResult struct { - Name string `json:"name"` - Match string `json:"match"` + Errors map[string][]string `json:"errors,omitempty"` + Infos map[string][]string `json:"infos,omitempty"` + Secrets map[string][]string `json:"secrets,omitempty"` } +// GetJSONString returns the JSON byte object. func GetJSONString( r *colly.Response, secrets []scanner.SecretMatched, @@ -74,9 +70,6 @@ func GetJSONString( contentLengths := (*headers)["Content-Length"] contentType := "" contentLength := 0 - errorList := []MatcherResult{} - infoList := []MatcherResult{} - secretList := []MatcherResult{} // Set content type if len(contentTypes) > 0 { @@ -99,31 +92,17 @@ func GetJSONString( // Parse lines from body lines := len(strings.Split(string(r.Body), "\n")) - // Process secrets - for _, secret := range secrets { - secretMatch := MatcherResult{secret.Secret.Name, secret.Match} - secretList = append(secretList, secretMatch) - } - - // Process infos - for _, info := range infos { - infoMatch := MatcherResult{info.Info.Name, info.Match} - infoList = append(infoList, infoMatch) - } - - // Process error list - for _, error := range errors { - errorMatch := MatcherResult{error.Error.ErrorName, error.Match} - errorList = append(errorList, errorMatch) - } + secretM := processSecrets(secrets) + infoM := processInfos(infos) + errorM := processErrors(errors) // Construct matcher results matcherResults := &MatcherResults{ FileType: filetype, Parameters: parameters, - Errors: errorList, - Infos: infoList, - Secrets: secretList, + Errors: errorM, + Infos: infoM, + Secrets: secretM, } // Construct JSON response @@ -143,9 +122,9 @@ func GetJSONString( var ( isFileTypeNill = false isParametersEmpty = len(parameters) == 0 - isErrorsEmpty = len(errorList) == 0 - isInfoEmpty = len(infoList) == 0 - isSecretsEmpty = len(secretList) == 0 + isErrorsEmpty = len(errorM) == 0 + isInfoEmpty = len(infoM) == 0 + isSecretsEmpty = len(secretM) == 0 ) if (*filetype == scanner.FileType{}) { @@ -165,3 +144,51 @@ func GetJSONString( return jsonOutput, nil } + +func processSecrets(secrets []scanner.SecretMatched) map[string][]string { + secretM := map[string][]string{} + + for _, secret := range secrets { + if _, ok := secretM[secret.Secret.Name]; ok { + tempV := secretM[secret.Secret.Name] + tempV = append(tempV, secret.Match) + secretM[secret.Secret.Name] = tempV + } else { + secretM[secret.Secret.Name] = []string{secret.Match} + } + } + + return secretM +} + +func processInfos(infos []scanner.InfoMatched) map[string][]string { + infoM := map[string][]string{} + + for _, info := range infos { + if _, ok := infoM[info.Info.Name]; ok { + tempV := infoM[info.Info.Name] + tempV = append(tempV, info.Match) + infoM[info.Info.Name] = tempV + } else { + infoM[info.Info.Name] = []string{info.Match} + } + } + + return infoM +} + +func processErrors(errors []scanner.ErrorMatched) map[string][]string { + errorM := map[string][]string{} + + for _, er := range errors { + if _, ok := errorM[er.Error.ErrorName]; ok { + tempV := errorM[er.Error.ErrorName] + tempV = append(tempV, er.Match) + errorM[er.Error.ErrorName] = tempV + } else { + errorM[er.Error.ErrorName] = []string{er.Match} + } + } + + return errorM +} From 098a9c1598c0fe69c8c6a78c60d18c9516f04937 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Thu, 22 Jun 2023 10:02:07 +0200 Subject: [PATCH 2/8] JSON lines aggregate results #115 --- pkg/output/jsonl_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/output/jsonl_test.go b/pkg/output/jsonl_test.go index 83a5683..0d127ef 100644 --- a/pkg/output/jsonl_test.go +++ b/pkg/output/jsonl_test.go @@ -122,7 +122,7 @@ func TestJSONOutput(t *testing.T) { filetype: filetype, errors: errors, infos: infos, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":{"MySQL error":["it is a MySQL error happening"]},"infos":{"info1":["its my pleasure to inform you on this great day"]},"secrets":{"mysecret":["it's a random day for my secret regex to be found"]}}}`, //nolint:lll }, { name: "test_all_findings_nocontent", @@ -132,7 +132,7 @@ func TestJSONOutput(t *testing.T) { filetype: filetype, errors: errors, infos: infos, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":{"MySQL error":["it is a MySQL error happening"]},"infos":{"info1":["its my pleasure to inform you on this great day"]},"secrets":{"mysecret":["it's a random day for my secret regex to be found"]}}}`, //nolint:lll }, { name: "test_no_findings", @@ -152,7 +152,7 @@ func TestJSONOutput(t *testing.T) { filetype: &scanner.FileType{}, errors: []scanner.ErrorMatched{}, infos: []scanner.InfoMatched{}, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"secrets":{"mysecret":["it's a random day for my secret regex to be found"]}}}`, //nolint:lll }, { name: "test_only_params", @@ -172,7 +172,7 @@ func TestJSONOutput(t *testing.T) { filetype: &scanner.FileType{}, errors: errors, infos: []scanner.InfoMatched{}, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}]}}`, //nolint:lll + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"errors":{"MySQL error":["it is a MySQL error happening"]}}}`, //nolint:lll }, { name: "test_only_infos", @@ -182,7 +182,7 @@ func TestJSONOutput(t *testing.T) { filetype: &scanner.FileType{}, errors: []scanner.ErrorMatched{}, infos: infos, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}]}}`, //nolint:lll + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"infos":{"info1":["its my pleasure to inform you on this great day"]}}}`, //nolint:lll }, } From e79ecdc72a21f03e4a614f53468fbbd426f95e35 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Thu, 24 Aug 2023 12:08:40 +0200 Subject: [PATCH 3/8] mod update --- go.mod | 14 +++++++------- go.sum | 33 +++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index e90408c..1d48167 100644 --- a/go.mod +++ b/go.mod @@ -9,21 +9,21 @@ require ( require ( github.com/PuerkitoBio/goquery v1.8.1 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect github.com/antchfx/htmlquery v1.3.0 // indirect - github.com/antchfx/xmlquery v1.3.15 // indirect + github.com/antchfx/xmlquery v1.3.17 // indirect github.com/antchfx/xpath v1.2.4 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.18 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/temoto/robotstxt v1.1.2 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/go.sum b/go.sum index ca017b9..852901a 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,12 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= -github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8Tw= -github.com/antchfx/xmlquery v1.3.15/go.mod h1:zMDv5tIGjOxY/JCNNinnle7V/EwthZ5IT8eeCGJKRWA= +github.com/antchfx/xmlquery v1.3.17 h1:d0qWjPp/D+vtRw7ivCwT5ApH/3CkQU8JOeo3245PpTk= +github.com/antchfx/xmlquery v1.3.17/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA= github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= @@ -30,8 +31,8 @@ github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= -github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= @@ -45,17 +46,21 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -66,12 +71,14 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -79,11 +86,13 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -91,5 +100,5 @@ google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= From b2f2033bc96e26a289c13f5ae3a138ad7fb3a8bd Mon Sep 17 00:00:00 2001 From: edoardottt Date: Sat, 23 Sep 2023 10:30:07 +0200 Subject: [PATCH 4/8] update deps --- go.mod | 8 ++++---- go.sum | 21 ++++++++++----------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 1d48167..4055235 100644 --- a/go.mod +++ b/go.mod @@ -21,9 +21,9 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/temoto/robotstxt v1.1.2 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect - google.golang.org/appengine v1.6.7 // indirect + golang.org/x/net v0.15.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/go.sum b/go.sum index 852901a..9be65d9 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,8 @@ github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= @@ -47,7 +47,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -56,8 +55,8 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -72,23 +71,23 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -96,8 +95,8 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= From a43092ca6a51a41cf8b0b4bba8abe805a554e8b0 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Mon, 16 Oct 2023 11:25:06 +0200 Subject: [PATCH 5/8] revert changes on jsonl (don't aggregate results) --- pkg/output/jsonl.go | 110 ++++++++++++--------------------------- pkg/output/jsonl_test.go | 18 ++----- 2 files changed, 39 insertions(+), 89 deletions(-) diff --git a/pkg/output/jsonl.go b/pkg/output/jsonl.go index 6a87a72..4e5c982 100644 --- a/pkg/output/jsonl.go +++ b/pkg/output/jsonl.go @@ -2,28 +2,21 @@ ========== Cariddi ========== - This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License along with this program. If not, see http://www.gnu.org/licenses/. @Repository: https://github.com/edoardottt/cariddi - @Author: edoardottt, https://www.edoardoottavianelli.it - @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE - */ - package output import ( @@ -46,16 +39,19 @@ type JSONData struct { Matches *MatcherResults `json:"matches,omitempty"` // Host string `json:"host"` # TODO: Available in Colly 2.x } - type MatcherResults struct { FileType *scanner.FileType `json:"filetype,omitempty"` Parameters []scanner.Parameter `json:"parameters,omitempty"` - Errors map[string][]string `json:"errors,omitempty"` - Infos map[string][]string `json:"infos,omitempty"` - Secrets map[string][]string `json:"secrets,omitempty"` + Errors []MatcherResult `json:"errors,omitempty"` + Infos []MatcherResult `json:"infos,omitempty"` + Secrets []MatcherResult `json:"secrets,omitempty"` +} + +type MatcherResult struct { + Name string `json:"name"` + Match string `json:"match"` } -// GetJSONString returns the JSON byte object. func GetJSONString( r *colly.Response, secrets []scanner.SecretMatched, @@ -70,39 +66,52 @@ func GetJSONString( contentLengths := (*headers)["Content-Length"] contentType := "" contentLength := 0 + errorList := []MatcherResult{} + infoList := []MatcherResult{} + secretList := []MatcherResult{} // Set content type if len(contentTypes) > 0 { contentType = strings.Split(contentTypes[0], "; ")[0] } - // Set content length if len(contentLengths) > 0 { ret, err := strconv.Atoi(contentLengths[0]) if err != nil { return nil, err } - contentLength = ret } - // Parse words from body words := len(strings.Fields(string(r.Body))) - // Parse lines from body lines := len(strings.Split(string(r.Body), "\n")) - secretM := processSecrets(secrets) - infoM := processInfos(infos) - errorM := processErrors(errors) + // Process secrets + for _, secret := range secrets { + secretMatch := MatcherResult{secret.Secret.Name, secret.Match} + secretList = append(secretList, secretMatch) + } + + // Process infos + for _, info := range infos { + infoMatch := MatcherResult{info.Info.Name, info.Match} + infoList = append(infoList, infoMatch) + } + + // Process error list + for _, error := range errors { + errorMatch := MatcherResult{error.Error.ErrorName, error.Match} + errorList = append(errorList, errorMatch) + } // Construct matcher results matcherResults := &MatcherResults{ FileType: filetype, Parameters: parameters, - Errors: errorM, - Infos: infoM, - Secrets: secretM, + Errors: errorList, + Infos: infoList, + Secrets: secretList, } // Construct JSON response @@ -117,25 +126,22 @@ func GetJSONString( Matches: matcherResults, // Host: "", // TODO: this is available in Colly 2.x } - // Set empty data if no matches to bridge the omitempty gap for empty structs var ( isFileTypeNill = false isParametersEmpty = len(parameters) == 0 - isErrorsEmpty = len(errorM) == 0 - isInfoEmpty = len(infoM) == 0 - isSecretsEmpty = len(secretM) == 0 + isErrorsEmpty = len(errorList) == 0 + isInfoEmpty = len(infoList) == 0 + isSecretsEmpty = len(secretList) == 0 ) if (*filetype == scanner.FileType{}) { matcherResults.FileType = nil isFileTypeNill = true } - if isFileTypeNill && isParametersEmpty && isErrorsEmpty && isInfoEmpty && isSecretsEmpty { resp.Matches = nil } - // Convert struct to JSON string jsonOutput, err := json.Marshal(resp) if err != nil { @@ -144,51 +150,3 @@ func GetJSONString( return jsonOutput, nil } - -func processSecrets(secrets []scanner.SecretMatched) map[string][]string { - secretM := map[string][]string{} - - for _, secret := range secrets { - if _, ok := secretM[secret.Secret.Name]; ok { - tempV := secretM[secret.Secret.Name] - tempV = append(tempV, secret.Match) - secretM[secret.Secret.Name] = tempV - } else { - secretM[secret.Secret.Name] = []string{secret.Match} - } - } - - return secretM -} - -func processInfos(infos []scanner.InfoMatched) map[string][]string { - infoM := map[string][]string{} - - for _, info := range infos { - if _, ok := infoM[info.Info.Name]; ok { - tempV := infoM[info.Info.Name] - tempV = append(tempV, info.Match) - infoM[info.Info.Name] = tempV - } else { - infoM[info.Info.Name] = []string{info.Match} - } - } - - return infoM -} - -func processErrors(errors []scanner.ErrorMatched) map[string][]string { - errorM := map[string][]string{} - - for _, er := range errors { - if _, ok := errorM[er.Error.ErrorName]; ok { - tempV := errorM[er.Error.ErrorName] - tempV = append(tempV, er.Match) - errorM[er.Error.ErrorName] = tempV - } else { - errorM[er.Error.ErrorName] = []string{er.Match} - } - } - - return errorM -} diff --git a/pkg/output/jsonl_test.go b/pkg/output/jsonl_test.go index 0d127ef..6d0c794 100644 --- a/pkg/output/jsonl_test.go +++ b/pkg/output/jsonl_test.go @@ -2,28 +2,21 @@ ========== Cariddi ========== - This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License along with this program. If not, see http://www.gnu.org/licenses/. @Repository: https://github.com/edoardottt/cariddi - @Author: edoardottt, https://www.edoardoottavianelli.it - @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE - */ - package output_test import ( @@ -41,7 +34,6 @@ func TestJSONOutput(t *testing.T) { headers := http.Header{} headers.Set("Content-Type", "application/pdf") headers.Set("Content-Length", "128") - secrets := []scanner.SecretMatched{ { Secret: scanner.Secret{ @@ -122,7 +114,7 @@ func TestJSONOutput(t *testing.T) { filetype: filetype, errors: errors, infos: infos, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":{"MySQL error":["it is a MySQL error happening"]},"infos":{"info1":["its my pleasure to inform you on this great day"]},"secrets":{"mysecret":["it's a random day for my secret regex to be found"]}}}`, //nolint:lll + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll }, { name: "test_all_findings_nocontent", @@ -132,7 +124,7 @@ func TestJSONOutput(t *testing.T) { filetype: filetype, errors: errors, infos: infos, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":{"MySQL error":["it is a MySQL error happening"]},"infos":{"info1":["its my pleasure to inform you on this great day"]},"secrets":{"mysecret":["it's a random day for my secret regex to be found"]}}}`, //nolint:lll + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll }, { name: "test_no_findings", @@ -152,7 +144,7 @@ func TestJSONOutput(t *testing.T) { filetype: &scanner.FileType{}, errors: []scanner.ErrorMatched{}, infos: []scanner.InfoMatched{}, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"secrets":{"mysecret":["it's a random day for my secret regex to be found"]}}}`, //nolint:lll + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll }, { name: "test_only_params", @@ -172,7 +164,7 @@ func TestJSONOutput(t *testing.T) { filetype: &scanner.FileType{}, errors: errors, infos: []scanner.InfoMatched{}, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"errors":{"MySQL error":["it is a MySQL error happening"]}}}`, //nolint:lll + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}]}}`, //nolint:lll }, { name: "test_only_infos", @@ -182,7 +174,7 @@ func TestJSONOutput(t *testing.T) { filetype: &scanner.FileType{}, errors: []scanner.ErrorMatched{}, infos: infos, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"infos":{"info1":["its my pleasure to inform you on this great day"]}}}`, //nolint:lll + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}]}}`, //nolint:lll }, } From 3bb36e25776f5df0b4e542341b4c248b38bbd954 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Mon, 16 Oct 2023 11:28:54 +0200 Subject: [PATCH 6/8] revert changes on jsonl (don't aggregate results) --- pkg/output/jsonl.go | 16 +++++++++++++++- pkg/output/jsonl_test.go | 8 ++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pkg/output/jsonl.go b/pkg/output/jsonl.go index 4e5c982..791a434 100644 --- a/pkg/output/jsonl.go +++ b/pkg/output/jsonl.go @@ -2,21 +2,28 @@ ========== Cariddi ========== + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + You should have received a copy of the GNU General Public License along with this program. If not, see http://www.gnu.org/licenses/. @Repository: https://github.com/edoardottt/cariddi + @Author: edoardottt, https://www.edoardoottavianelli.it + @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE + */ + package output import ( @@ -39,6 +46,7 @@ type JSONData struct { Matches *MatcherResults `json:"matches,omitempty"` // Host string `json:"host"` # TODO: Available in Colly 2.x } + type MatcherResults struct { FileType *scanner.FileType `json:"filetype,omitempty"` Parameters []scanner.Parameter `json:"parameters,omitempty"` @@ -74,14 +82,17 @@ func GetJSONString( if len(contentTypes) > 0 { contentType = strings.Split(contentTypes[0], "; ")[0] } + // Set content length if len(contentLengths) > 0 { ret, err := strconv.Atoi(contentLengths[0]) if err != nil { return nil, err } + contentLength = ret } + // Parse words from body words := len(strings.Fields(string(r.Body))) // Parse lines from body @@ -124,8 +135,9 @@ func GetJSONString( ContentType: contentType, ContentLength: contentLength, Matches: matcherResults, - // Host: "", // TODO: this is available in Colly 2.x + // Host: "", // TODO } + // Set empty data if no matches to bridge the omitempty gap for empty structs var ( isFileTypeNill = false @@ -139,9 +151,11 @@ func GetJSONString( matcherResults.FileType = nil isFileTypeNill = true } + if isFileTypeNill && isParametersEmpty && isErrorsEmpty && isInfoEmpty && isSecretsEmpty { resp.Matches = nil } + // Convert struct to JSON string jsonOutput, err := json.Marshal(resp) if err != nil { diff --git a/pkg/output/jsonl_test.go b/pkg/output/jsonl_test.go index 6d0c794..83a5683 100644 --- a/pkg/output/jsonl_test.go +++ b/pkg/output/jsonl_test.go @@ -2,21 +2,28 @@ ========== Cariddi ========== + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + You should have received a copy of the GNU General Public License along with this program. If not, see http://www.gnu.org/licenses/. @Repository: https://github.com/edoardottt/cariddi + @Author: edoardottt, https://www.edoardoottavianelli.it + @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE + */ + package output_test import ( @@ -34,6 +41,7 @@ func TestJSONOutput(t *testing.T) { headers := http.Header{} headers.Set("Content-Type", "application/pdf") headers.Set("Content-Length", "128") + secrets := []scanner.SecretMatched{ { Secret: scanner.Secret{ From 9829a17750b84efeea91c4c777691653b3eef814 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Mon, 16 Oct 2023 11:29:21 +0200 Subject: [PATCH 7/8] revert changes on jsonl (don't aggregate results) --- pkg/output/jsonl.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/output/jsonl.go b/pkg/output/jsonl.go index 791a434..e54d7d4 100644 --- a/pkg/output/jsonl.go +++ b/pkg/output/jsonl.go @@ -95,6 +95,7 @@ func GetJSONString( // Parse words from body words := len(strings.Fields(string(r.Body))) + // Parse lines from body lines := len(strings.Split(string(r.Body), "\n")) From ae49ebb4ff252dea2cba683e7fcb342b31b20b12 Mon Sep 17 00:00:00 2001 From: Edoardo Ottavianelli Date: Mon, 16 Oct 2023 11:36:09 +0200 Subject: [PATCH 8/8] Update README.md --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 9f04995..aa9e8e6 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,6 @@

- Preview • Install • Get Started • Examples • @@ -56,9 +55,6 @@ License

-Preview 📊 ----------- -