diff --git a/crawler/colly.go b/crawler/colly.go index 3041729..0db9421 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -2,18 +2,16 @@ package crawler import ( "fmt" - "regexp" "github.com/gocolly/colly" ) //Crawler -func Crawler(target string) { +func Crawler(target string) []string { + var result []string // Instantiate default collector c := colly.NewCollector( - colly.URLFilters( - regexp.MustCompile(".*" + target + "*"), - ), + colly.AllowedDomains(target), ) // On every a element which has href attribute call callback @@ -24,6 +22,7 @@ func Crawler(target string) { // Visit link found on page // Only those links are visited which are in AllowedDomains c.Visit(e.Request.AbsoluteURL(link)) + result = append(result, e.Request.AbsoluteURL(link)) }) // Before making a request print "Visiting ..." @@ -33,4 +32,6 @@ func Crawler(target string) { // Start scraping on target c.Visit("http://" + target) + c.Visit("https://" + target) + return result } diff --git a/main.go b/main.go index 22f7f13..1326a24 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,6 @@ package main import ( - "fmt" - "github.com/edoardottt/cariddi/crawler" "github.com/edoardottt/cariddi/input" "github.com/edoardottt/cariddi/output" @@ -11,11 +9,14 @@ import ( //main func main() { - fmt.Println("cariddi") input := input.ScanInput() + var finalResult []string for _, inp := range input { - crawler.Crawler(inp) + var result []string + result = crawler.Crawler(inp) + finalResult = append(finalResult, result...) } output.Beautify() + output.PrintOutput(finalResult) scanner.Scan() } diff --git a/output/output.go b/output/output.go index 5c4be8d..63516a2 100644 --- a/output/output.go +++ b/output/output.go @@ -2,7 +2,9 @@ package output import "fmt" -//Output -func Output() { - fmt.Println("output") +//PrintOutput +func PrintOutput(out []string) { + for _, elem := range out { + fmt.Println(elem) + } } diff --git a/utils/slice.go b/utils/slice.go new file mode 100644 index 0000000..0a64230 --- /dev/null +++ b/utils/slice.go @@ -0,0 +1,14 @@ +package utils + +//removeDuplicateValues +func removeDuplicateValues(intSlice []string) []string { + keys := make(map[string]bool) + list := []string{} + for _, entry := range intSlice { + if _, value := keys[entry]; !value { + keys[entry] = true + list = append(list, entry) + } + } + return list +}