Refactoring

This commit is contained in:
edoardottt
2022-10-14 14:36:00 +02:00
parent 513b5e4a26
commit c35f284c8a
25 changed files with 84 additions and 82 deletions
+12 -11
View File
@@ -29,11 +29,12 @@ package main
import (
"os"
"github.com/edoardottt/cariddi/crawler"
"github.com/edoardottt/cariddi/input"
"github.com/edoardottt/cariddi/output"
"github.com/edoardottt/cariddi/scanner"
"github.com/edoardottt/cariddi/utils"
fileUtils "github.com/edoardottt/cariddi/internal/file"
sliceUtils "github.com/edoardottt/cariddi/internal/slice"
"github.com/edoardottt/cariddi/pkg/crawler"
"github.com/edoardottt/cariddi/pkg/input"
"github.com/edoardottt/cariddi/pkg/output"
"github.com/edoardottt/cariddi/pkg/scanner"
)
// main function.
@@ -74,14 +75,14 @@ func main() {
// from the specified file.
var endpointsFileSlice []string
if flags.EndpointsFile != "" {
endpointsFileSlice = utils.ReadFile(flags.EndpointsFile)
endpointsFileSlice = fileUtils.ReadFile(flags.EndpointsFile)
}
// If it is needed, read custom secrets definition
// from the specified file.
var secretsFileSlice []string
if flags.SecretsFile != "" {
secretsFileSlice = utils.ReadFile(flags.SecretsFile)
secretsFileSlice = fileUtils.ReadFile(flags.SecretsFile)
}
finalResults := []string{}
@@ -94,12 +95,12 @@ func main() {
// Create output files if needed (txt / html).
var ResultTxt = ""
if flags.TXT != "" {
ResultTxt = utils.CreateOutputFile(flags.TXT, "results", "txt")
ResultTxt = fileUtils.CreateOutputFile(flags.TXT, "results", "txt")
}
var ResultHTML = ""
if flags.HTML != "" {
ResultHTML = utils.CreateOutputFile(flags.HTML, "", "html")
ResultHTML = fileUtils.CreateOutputFile(flags.HTML, "", "html")
output.BannerHTML(ResultHTML)
output.HeaderHTML("Results", ResultHTML)
}
@@ -110,7 +111,7 @@ func main() {
if flags.HeadersFile != "" || flags.Headers != "" {
var headersInput string
if flags.HeadersFile != "" {
headersInput = string(utils.ReadEntireFile(flags.HeadersFile))
headersInput = string(fileUtils.ReadEntireFile(flags.HeadersFile))
} else {
headersInput = flags.Headers
}
@@ -134,7 +135,7 @@ func main() {
}
// Remove duplicates from all the results.
finalResults = utils.RemoveDuplicateValues(finalResults)
finalResults = sliceUtils.RemoveDuplicateValues(finalResults)
finalSecret = scanner.RemoveDuplicateSecrets(finalSecret)
finalEndpoints = scanner.RemovDuplicateEndpoints(finalEndpoints)
finalExtensions = scanner.RemoveDuplicateExtensions(finalExtensions)
+1 -1
View File
@@ -24,7 +24,7 @@ along with this program. If not, see http://www.gnu.org/licenses/.
*/
package utils
package slice
import (
"net/http"
@@ -24,14 +24,14 @@ along with this program. If not, see http://www.gnu.org/licenses/.
*/
package utils_test
package slice_test
import (
"net/http"
"reflect"
"testing"
"github.com/edoardottt/cariddi/utils"
sliceUtils "github.com/edoardottt/cariddi/internal/slice"
)
func TestRemoveDuplicateValues(t *testing.T) {
@@ -63,7 +63,7 @@ func TestRemoveDuplicateValues(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := utils.RemoveDuplicateValues(tt.slice); !reflect.DeepEqual(got, tt.want) {
if got := sliceUtils.RemoveDuplicateValues(tt.slice); !reflect.DeepEqual(got, tt.want) {
t.Errorf("RemoveDuplicateValues() = %v, want %v", got, tt.want)
}
})
@@ -99,7 +99,7 @@ func TestCheckInputArray(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := utils.CheckInputArray(tt.input); !reflect.DeepEqual(got, tt.want) {
if got := sliceUtils.CheckInputArray(tt.input); !reflect.DeepEqual(got, tt.want) {
t.Errorf("CheckInputArray() = %v, want %v", got, tt.want)
}
})
@@ -163,7 +163,7 @@ func TestCheckCookies(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := utils.CheckCookies(tt.input); !reflect.DeepEqual(got, tt.want) {
if got := sliceUtils.CheckCookies(tt.input); !reflect.DeepEqual(got, tt.want) {
t.Errorf("CheckCookies() = %v, want %v", got, tt.want)
}
})
+4 -2
View File
@@ -24,12 +24,14 @@ along with this program. If not, see http://www.gnu.org/licenses/.
*/
package utils
package url
import (
"fmt"
"net/url"
"strings"
errUtils "github.com/edoardottt/cariddi/internal/error"
)
// GetHost takes as input a string and
@@ -83,7 +85,7 @@ func GetRootHost(input string) (string, error) {
return parts[len(parts)-2] + "." + parts[len(parts)-1], nil
}
return "", fmt.Errorf("%w", ErrDomainFormat)
return "", fmt.Errorf("%w", errUtils.ErrDomainFormat)
}
// HasProtocol takes as input a string and
@@ -24,13 +24,13 @@ along with this program. If not, see http://www.gnu.org/licenses/.
*/
package utils_test
package url_test
import (
"reflect"
"testing"
"github.com/edoardottt/cariddi/utils"
urlUtils "github.com/edoardottt/cariddi/internal/url"
)
func TestGetHost(t *testing.T) {
@@ -57,7 +57,7 @@ func TestGetHost(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := utils.GetHost(tt.input); !reflect.DeepEqual(got, tt.want) {
if got := urlUtils.GetHost(tt.input); !reflect.DeepEqual(got, tt.want) {
t.Errorf("GetHost() = %v, want %v", got, tt.want)
}
})
@@ -88,7 +88,7 @@ func TestGetProtocol(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := utils.GetProtocol(tt.input); !reflect.DeepEqual(got, tt.want) {
if got := urlUtils.GetProtocol(tt.input); !reflect.DeepEqual(got, tt.want) {
t.Errorf("GetProtocol() = %v, want %v", got, tt.want)
}
})
@@ -124,7 +124,7 @@ func TestGetRootHost(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got, _ := utils.GetRootHost(tt.input); !reflect.DeepEqual(got, tt.want) {
if got, _ := urlUtils.GetRootHost(tt.input); !reflect.DeepEqual(got, tt.want) {
t.Errorf("GetRootHost() = %v, want %v", got, tt.want)
}
})
@@ -165,7 +165,7 @@ func TestHasProtocol(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := utils.HasProtocol(tt.input); !reflect.DeepEqual(got, tt.want) {
if got := urlUtils.HasProtocol(tt.input); !reflect.DeepEqual(got, tt.want) {
t.Errorf("HasProtocol() = %v, want %v", got, tt.want)
}
})
+31 -32
View File
@@ -37,10 +37,12 @@ import (
"strings"
"time"
"github.com/edoardottt/cariddi/input"
"github.com/edoardottt/cariddi/output"
"github.com/edoardottt/cariddi/scanner"
"github.com/edoardottt/cariddi/utils"
fileUtils "github.com/edoardottt/cariddi/internal/file"
sliceUtils "github.com/edoardottt/cariddi/internal/slice"
urlUtils "github.com/edoardottt/cariddi/internal/url"
"github.com/edoardottt/cariddi/pkg/input"
"github.com/edoardottt/cariddi/pkg/output"
"github.com/edoardottt/cariddi/pkg/scanner"
"github.com/gocolly/colly"
"github.com/gocolly/colly/extensions"
)
@@ -59,17 +61,17 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
var targetTemp, protocolTemp string
// if there isn't a scheme use http.
if !utils.HasProtocol(target) {
if !urlUtils.HasProtocol(target) {
protocolTemp = "http"
targetTemp = utils.GetHost(protocolTemp + "://" + target)
targetTemp = urlUtils.GetHost(protocolTemp + "://" + target)
} else {
protocolTemp = utils.GetProtocol(target)
targetTemp = utils.GetHost(target)
protocolTemp = urlUtils.GetProtocol(target)
targetTemp = urlUtils.GetHost(target)
}
if intensive {
var err error
targetTemp, err = utils.GetRootHost(protocolTemp + "://" + targetTemp)
targetTemp, err = urlUtils.GetRootHost(protocolTemp + "://" + targetTemp)
if err != nil {
fmt.Println(err.Error())
@@ -83,7 +85,7 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
}
// clean target input
target = utils.RemoveProtocol(target)
target = urlUtils.RemoveProtocol(target)
ignoreSlice := []string{}
ignoreBool := false
@@ -91,13 +93,13 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
// if ignore -> produce the slice
if ignore != "" {
ignoreBool = true
ignoreSlice = utils.CheckInputArray(ignore)
ignoreSlice = sliceUtils.CheckInputArray(ignore)
}
// if ignoreTxt -> produce the slice
if ignoreTxt != "" {
ignoreBool = true
ignoreSlice = utils.ReadFile(ignoreTxt)
ignoreSlice = fileUtils.ReadFile(ignoreTxt)
}
FinalResults := []string{}
@@ -114,20 +116,17 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
link := e.Attr("href")
if len(link) != 0 && link[0] != '#' {
absoluteURL := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
absoluteURL := urlUtils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
(intensive && intensiveOk(targetTemp, absoluteURL)) {
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
(intensive && intensiveOk(targetTemp, absoluteURL)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) {
err := c.Visit(absoluteURL)
if !errors.Is(err, colly.ErrAlreadyVisited) {
FinalResults = append(FinalResults, absoluteURL)
if err != nil && debug {
log.Println(err)
}
if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) {
err := c.Visit(absoluteURL)
if !errors.Is(err, colly.ErrAlreadyVisited) {
FinalResults = append(FinalResults, absoluteURL)
if err != nil && debug {
log.Println(err)
}
}
}
@@ -157,10 +156,10 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
c.OnHTML("form[action]", func(e *colly.HTMLElement) {
link := e.Attr("action")
if len(link) != 0 {
absoluteURL := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
absoluteURL := urlUtils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
(intensive && intensiveOk(targetTemp, absoluteURL)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) {
err := c.Visit(absoluteURL)
@@ -179,10 +178,10 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
c.OnXML("//urlset/url/loc", func(e *colly.XMLElement) {
link := e.Text
if len(link) != 0 {
absoluteURL := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
absoluteURL := urlUtils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
(intensive && intensiveOk(targetTemp, absoluteURL)) {
if ignoreBool {
if !IgnoreMatch(link, ignoreSlice) {
@@ -263,7 +262,7 @@ func New(target string, txt string, html string, delayTime int, concurrency int,
})
// Start scraping on target
path, err := utils.GetPath(protocolTemp + "://" + target)
path, err := urlUtils.GetPath(protocolTemp + "://" + target)
if err == nil {
if path == "" {
err = c.Visit(protocolTemp + "://" + target + "/" + "robots.txt")
@@ -367,10 +366,10 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
func visitLink(link, protocolTemp, targetTemp, target string, intensive, ignoreBool, debug bool,
ignoreSlice []string, finalResults *[]string, e *colly.HTMLElement, c *colly.Collector) {
if len(link) != 0 {
absoluteURL := utils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
absoluteURL := urlUtils.AbsoluteURL(protocolTemp, targetTemp, e.Request.AbsoluteURL(link))
// Visit link found on page
// Only those links are visited which are in AllowedDomains
if (!intensive && utils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
if (!intensive && urlUtils.SameDomain(protocolTemp+"://"+target, absoluteURL)) ||
(intensive && intensiveOk(targetTemp, absoluteURL)) {
if !ignoreBool || (ignoreBool && !IgnoreMatch(link, ignoreSlice)) {
err := c.Visit(absoluteURL)
@@ -443,7 +442,7 @@ func huntEndpoints(endpointsFile []string, target string) []scanner.EndpointMatc
func EndpointsMatch(target string, endpointsFile []string) []scanner.EndpointMatched {
endpoints := []scanner.EndpointMatched{}
matched := []scanner.Parameter{}
parameters := utils.RetrieveParameters(target)
parameters := urlUtils.RetrieveParameters(target)
if len(endpointsFile) == 0 {
for _, parameter := range scanner.GetJuicyParameters() {
@@ -562,7 +561,7 @@ func IgnoreMatch(url string, ignoreSlice []string) bool {
// in intensive mode (if the 2nd level domain matches with
// the inputted target).
func intensiveOk(target string, urlInput string) bool {
root, err := utils.GetRootHost(urlInput)
root, err := urlUtils.GetRootHost(urlInput)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
+2 -2
View File
@@ -31,7 +31,7 @@ import (
"os"
"strings"
"github.com/edoardottt/cariddi/utils"
fileUtils "github.com/edoardottt/cariddi/internal/file"
)
// CheckOutputFile checks if the string provided as input
@@ -104,7 +104,7 @@ func CheckFlags(flags Input) {
}
if flags.IgnoreTXT != "" {
_ = utils.ReadFile(flags.IgnoreTXT)
_ = fileUtils.ReadFile(flags.IgnoreTXT)
}
if flags.Timeout < 0 {
+2 -2
View File
@@ -32,7 +32,7 @@ import (
"os"
"strings"
"github.com/edoardottt/cariddi/utils"
sliceUtils "github.com/edoardottt/cariddi/internal/slice"
)
// ScanTargets return the array of elements
@@ -49,7 +49,7 @@ func ScanTargets() []string {
}
}
return utils.RemoveDuplicateValues(result)
return sliceUtils.RemoveDuplicateValues(result)
}
// GetHeaders returns the headers provided as input
+6 -6
View File
@@ -30,12 +30,12 @@ import (
"log"
"os"
"github.com/edoardottt/cariddi/utils"
fileUtils "github.com/edoardottt/cariddi/internal/file"
)
// BannerHTML appends the initial banner to html file.
func BannerHTML(filename string) {
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, utils.Permission0644)
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, fileUtils.Permission0644)
if err != nil {
log.Println(err)
os.Exit(1)
@@ -54,7 +54,7 @@ func BannerHTML(filename string) {
// AppendOutputToHTML appends the output to html file.
func AppendOutputToHTML(output string, status string, filename string, isLink bool) {
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, utils.Permission0644)
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, fileUtils.Permission0644)
if err != nil {
log.Println(err)
os.Exit(1)
@@ -88,7 +88,7 @@ func AppendOutputToHTML(output string, status string, filename string, isLink bo
// HeaderHTML appends the html header.
func HeaderHTML(header string, filename string) {
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, utils.Permission0644)
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, fileUtils.Permission0644)
if err != nil {
log.Println(err)
os.Exit(1)
@@ -103,7 +103,7 @@ func HeaderHTML(header string, filename string) {
// FooterHTML appends the footer.
func FooterHTML(filename string) {
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, utils.Permission0644)
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, fileUtils.Permission0644)
if err != nil {
log.Println(err)
os.Exit(1)
@@ -118,7 +118,7 @@ func FooterHTML(filename string) {
// BannerFooterHTML appends the final footer.
func BannerFooterHTML(filename string) {
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, utils.Permission0644)
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, fileUtils.Permission0644)
if err != nil {
log.Println(err)
}
+13 -13
View File
@@ -31,9 +31,9 @@ import (
"os"
"strings"
"github.com/edoardottt/cariddi/input"
"github.com/edoardottt/cariddi/scanner"
"github.com/edoardottt/cariddi/utils"
fileUtils "github.com/edoardottt/cariddi/internal/file"
"github.com/edoardottt/cariddi/pkg/input"
"github.com/edoardottt/cariddi/pkg/scanner"
)
// PrintSimpleOutput prints line by line.
@@ -48,24 +48,24 @@ func PrintSimpleOutput(out []string) {
func TxtOutput(flags input.Input, finalResults []string, finalSecret []scanner.SecretMatched,
finalEndpoints []scanner.EndpointMatched, finalExtensions []scanner.FileTypeMatched,
finalErrors []scanner.ErrorMatched, finalInfos []scanner.InfoMatched) {
exists, err := utils.ElementExists("output-cariddi")
exists, err := fileUtils.ElementExists("output-cariddi")
if err != nil {
fmt.Println("Error while creating the output directory.")
os.Exit(1)
}
if !exists {
utils.CreateOutputFolder()
fileUtils.CreateOutputFolder()
}
ResultFilename := utils.CreateOutputFile(flags.TXT, "results", "txt")
ResultFilename := fileUtils.CreateOutputFile(flags.TXT, "results", "txt")
for _, elem := range finalResults {
AppendOutputToTxt(elem, ResultFilename)
}
// if secrets flag enabled save also secrets
if flags.Secrets {
SecretFilename := utils.CreateOutputFile(flags.TXT, "secrets", "txt")
SecretFilename := fileUtils.CreateOutputFile(flags.TXT, "secrets", "txt")
for _, elem := range finalSecret {
AppendOutputToTxt(elem.Secret.Name+" - "+elem.Match+" in "+elem.URL, SecretFilename)
}
@@ -73,7 +73,7 @@ func TxtOutput(flags input.Input, finalResults []string, finalSecret []scanner.S
// if endpoints flag enabled save also endpoints
if flags.Endpoints {
EndpointFilename := utils.CreateOutputFile(flags.TXT, "endpoints", "txt")
EndpointFilename := fileUtils.CreateOutputFile(flags.TXT, "endpoints", "txt")
for _, elem := range finalEndpoints {
for _, parameter := range elem.Parameters {
@@ -92,7 +92,7 @@ func TxtOutput(flags input.Input, finalResults []string, finalSecret []scanner.S
// if extensions flag enabled save also secrets
if 1 <= flags.Extensions && flags.Extensions <= 7 {
ExtensionsFilename := utils.CreateOutputFile(flags.TXT, "extensions", "txt")
ExtensionsFilename := fileUtils.CreateOutputFile(flags.TXT, "extensions", "txt")
for _, elem := range finalExtensions {
AppendOutputToTxt(elem.Filetype.Extension+" in "+elem.URL, ExtensionsFilename)
}
@@ -100,7 +100,7 @@ func TxtOutput(flags input.Input, finalResults []string, finalSecret []scanner.S
// if errors flag enabled save also errors
if flags.Errors {
ErrorsFilename := utils.CreateOutputFile(flags.TXT, "errors", "txt")
ErrorsFilename := fileUtils.CreateOutputFile(flags.TXT, "errors", "txt")
for _, elem := range finalErrors {
AppendOutputToTxt(elem.Error.ErrorName+" - "+elem.Match+" in "+elem.URL, ErrorsFilename)
}
@@ -108,7 +108,7 @@ func TxtOutput(flags input.Input, finalResults []string, finalSecret []scanner.S
// if info flag enabled save also infos
if flags.Info {
InfosFilename := utils.CreateOutputFile(flags.TXT, "info", "txt")
InfosFilename := fileUtils.CreateOutputFile(flags.TXT, "info", "txt")
for _, elem := range finalInfos {
AppendOutputToTxt(elem.Info.Name+" - "+elem.Match+" in "+elem.URL, InfosFilename)
}
@@ -120,7 +120,7 @@ func TxtOutput(flags input.Input, finalResults []string, finalSecret []scanner.S
func HTMLOutput(flags input.Input, resultFilename string, finalResults []string, finalSecret []scanner.SecretMatched,
finalEndpoints []scanner.EndpointMatched, finalExtensions []scanner.FileTypeMatched,
finalErrors []scanner.ErrorMatched, finalInfos []scanner.InfoMatched) {
exists, err := utils.ElementExists("output-cariddi")
exists, err := fileUtils.ElementExists("output-cariddi")
if err != nil {
fmt.Println("Error while creating the output directory.")
@@ -128,7 +128,7 @@ func HTMLOutput(flags input.Input, resultFilename string, finalResults []string,
}
if !exists {
utils.CreateOutputFolder()
fileUtils.CreateOutputFolder()
}
HeaderHTML("Results found", resultFilename)
+2 -2
View File
@@ -30,13 +30,13 @@ import (
"log"
"os"
"github.com/edoardottt/cariddi/utils"
fileUtils "github.com/edoardottt/cariddi/internal/file"
)
// AppendOutputToTxt opens the output file and append
// the string taken as input.
func AppendOutputToTxt(output string, filename string) {
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, utils.Permission0644)
file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, fileUtils.Permission0644)
if err != nil {
log.Println(err)
}