diff --git a/Makefile b/Makefile index a71a49f..3b3673d 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,9 @@ update: @make linux @echo "Done." +lint: + @golangci-lint run + linux: @go build -o ./cariddi @sudo mv ./cariddi /usr/bin/ diff --git a/crawler/colly.go b/crawler/colly.go index 8f0e64e..cd47f41 100644 --- a/crawler/colly.go +++ b/crawler/colly.go @@ -364,7 +364,7 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int, c := colly.NewCollector( colly.Async(true), ) - c.IgnoreRobotsTxt = false + c.IgnoreRobotsTxt = true c.AllowURLRevisit = false err := c.Limit( diff --git a/make.bat b/make.bat index 4caefed..195ead8 100644 --- a/make.bat +++ b/make.bat @@ -22,6 +22,11 @@ IF "%ARG%"=="fmt" ( GOTO Done ) +IF "%ARG%"=="lint" ( + CALL :Lint + GOTO Done +) + IF "%ARG%"=="remod" ( del go.mod del go.sum @@ -52,6 +57,10 @@ go fmt ./... echo Done. EXIT /B 0 +:Lint +golangci-lint run +EXIT /B 0 + :Update set GO111MODULE=on echo Updating ...