From e2f2e07b76f2b6d8da7da030bbd2389fd79cafac Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Sat, 22 Dec 2018 14:06:27 -0500 Subject: [PATCH] Moved search method to constants.py also made it so it notifies user if google is blocking incoming requests, added search method to trello. --- discovery/constants.py | 13 ++++++++++++- discovery/googlesearch.py | 16 ++++------------ discovery/trello.py | 5 ++++- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/discovery/constants.py b/discovery/constants.py index bd038403..432c092e 100644 --- a/discovery/constants.py +++ b/discovery/constants.py @@ -237,12 +237,23 @@ user_agents = [ "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/11.10 Chromium/18.0.1025.142 Chrome/18.0.1025.142 Safari/535.19", "Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00" - ] +] googleUA = "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36" + def getDelay(): return random.randint(1, 3) - .5 + def getUserAgent(): return random.choice(user_agents) + + +def search(text): + # helper function to check if google has blocked traffic + for line in text.strip().splitlines(): + if 'This page appears when Google automatically detects requests coming from your computer network' in line: + print('\tGoogle is blocking your iP due too many automated request, wait or change your IP') + return True + return False diff --git a/discovery/googlesearch.py b/discovery/googlesearch.py index 31bafd90..d06c0bb6 100644 --- a/discovery/googlesearch.py +++ b/discovery/googlesearch.py @@ -3,6 +3,7 @@ import time import requests from discovery.constants import * + class search_google: def __init__(self, word, limit, start): @@ -29,7 +30,7 @@ class search_google: except Exception as e: print(e) self.results = r.text - if (self.search(self.results)): + if (search(self.results)): time.sleep(getDelay() * 5) # sleep for a longer time else: time.sleep(getDelay()) @@ -47,7 +48,7 @@ class search_google: except Exception as e: print(e) self.results = r.text - if (self.search(self.results)): + if (search(self.results)): time.sleep(getDelay() * 5) # sleep for a longer time else: time.sleep(getDelay()) @@ -73,8 +74,6 @@ class search_google: if google_dorking == False: while self.counter <= self.limit and self.counter <= 1000: self.do_search() - #more = self.check_next() - time.sleep(1.5) print("\tSearching " + str(self.counter) + " results...") self.counter += 100 else: # google dorking is true @@ -143,17 +142,10 @@ class search_google: link = self.links[i] # get link from dork list req = requests.get(link, headers=headers) self.results = req.text - if (self.search(self.results)): + if (search(self.results)): time.sleep(getDelay() * 5) # sleep for a longer time else: time.sleep(getDelay()) self.totalresults += self.results except: continue - - def search(self, text): - # helper function to check if google has blocked traffic - for line in text.strip().splitlines(): - if 'This page appears when Google automatically detects requests coming from your computer network' in line: - return True - return False diff --git a/discovery/trello.py b/discovery/trello.py index 14a6bae9..f506c49d 100644 --- a/discovery/trello.py +++ b/discovery/trello.py @@ -53,6 +53,9 @@ class search_trello: def process(self): while (self.counter < self.limit): self.do_search() - time.sleep(getDelay()) + if search(self.results): + time.sleep(getDelay() * 5) + else: + time.sleep(getDelay()) self.counter += 100 print("\tSearching " + str(self.counter) + " results..")