diff --git a/crawler/colly.go b/crawler/colly.go index 2dac5ad..d5b418f 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -57,7 +57,7 @@ func Crawler(target string, txt string, html string, delayTime int, concurrency var protocolTemp string // if there isn't a scheme use http. - if !utils.HasScheme(target) { + if !utils.HasProtocol(target) { protocolTemp = "http" targetTemp = utils.GetHost(protocolTemp + "://" + target) } else { diff --git a/utils/urls.go b/utils/urls.go index 72fc207..61a8b6c 100644 --- a/utils/urls.go +++ b/utils/urls.go @@ -69,22 +69,10 @@ func GetRootHost(input string) string { return domainutil.Domain(input) } -//GetScheme > -// duplicate of GetProtocol. -// to be deleted -func GetScheme(input string) string { - u, err := url.Parse(input) - if err != nil { - return "" - } - return u.Scheme -} - -//HasScheme takes as input a string and +//HasProtocol takes as input a string and //checks if it has a protocol ( like in a //URI/URL) -//Should be renamed to HasProtocol? -func HasScheme(input string) bool { +func HasProtocol(input string) bool { res := strings.Index(input, "://") return res >= 0 } @@ -131,7 +119,7 @@ func RetrieveParameters(input string) []string { func AbsoluteURL(protocol string, target string, path string) string { // if the path variable starts with a scheme, it means that the // path is itself an absolute path. - if HasScheme(path) { + if HasProtocol(path) { return path } if len(path) != 0 && path[0] == '/' {