diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index 556eb9a..5df23f6 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -312,10 +312,6 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int, c.IgnoreRobotsTxt = true c.AllowURLRevisit = false - if userAgent != "" { - c.UserAgent = userAgent - } - err := c.Limit( &colly.LimitRule{ Parallelism: concurrency, @@ -345,6 +341,10 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int, c.UserAgent = GenerateRandomUserAgent() } + if userAgent != "" { + c.UserAgent = userAgent + } + // Use a Proxy if needed if proxy != "" { err := c.SetProxy(proxy)