From f702f56d63bb70e531963bdbd3ec0fe5df2a8f56 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Thu, 12 Sep 2019 03:01:09 -0400 Subject: [PATCH] Implemented google work around in LinkedIn and Google searches as well as static typing for function in constants.py --- theHarvester/discovery/constants.py | 53 ++++++++++++++++++------ theHarvester/discovery/googlesearch.py | 38 +++++++++++++---- theHarvester/discovery/linkedinsearch.py | 12 +++++- 3 files changed, 80 insertions(+), 23 deletions(-) diff --git a/theHarvester/discovery/constants.py b/theHarvester/discovery/constants.py index 0fbe8c91..c38d8e90 100644 --- a/theHarvester/discovery/constants.py +++ b/theHarvester/discovery/constants.py @@ -1,6 +1,5 @@ import random - googleUA = 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36' @@ -8,7 +7,7 @@ def splitter(links): """ Method that tries to remove duplicates LinkedinLists pulls a lot of profiles with the same name. - This method triest to remove duplicates from the list. + This method tries to remove duplicates from the list. :param links: list of links to remove duplicates from :return: unique-ish list """ @@ -43,32 +42,62 @@ def filter(lst): for item in lst: item = str(item) if (item[0].isalpha() or item[0].isdigit()) and ('xxx' not in item) and ('..' not in item): - if '252f' in item: - item = item.replace('252f', '') - if '2F' in item: - item = item.replace('2F', '') - if '2f' in item: - item = item.replace('2f', '') + item = item.replace('252f', '').replace('2F', '').replace('2f', '') new_lst.append(item.lower()) return new_lst -def getDelay(): +def getDelay() -> int: return random.randint(1, 3) - .5 -def search(text): +def search(text: str) -> bool: # Helper function to check if Google has blocked traffic. for line in text.strip().splitlines(): - if 'This page appears when Google automatically detects requests coming from your computer network' in line: + if 'This page appears when Google automatically detects requests coming from your computer network' in line \ + or 'http://www.google.com/sorry/index' in line or 'https://www.google.com/sorry/index' in line: print('\tGoogle is blocking your IP due to too many automated requests, wait or change your IP') return True return False +def google_workaround(visit_url: str) -> str or bool: + """ + Function that makes a request on our behalf, if Google starts to block us + :param visit_url: Url to scrape + :return: Correct html that can be parsed by BS4 + """ + import requests + url = 'https://websniffer.cc/' + data = { + 'Cookie': '', + 'url': visit_url, + 'submit': 'Submit', + 'type': 'GET&http=1.1', + 'uak': str(random.randint(4, 8)) # select random UA to send to Google + } + resp = requests.post(url,headers={'User-Agent': googleUA}, data=data) + returned_html = resp.text + if search(returned_html): + # indicates that google is serving workaround a captcha + # TODO rework workaround with more websites to send requets on our behalf or utilize proxies option in request + return True + # the html we get is malformed for BS4 as there are no greater than or less than signs + if '<html>' in returned_html: + start_index = returned_html.index('<html>') + else: + start_index = returned_html.index('<html') + + end_index = returned_html.index('</html>') + 1 + correct_html = returned_html[start_index:end_index] + # Slice list to get the response's html + correct_html = ''.join([ch.strip().replace('<', '<').replace('>', '>') for ch in correct_html]) + return correct_html + + class MissingKey(Exception): - def __init__(self, identity_flag): + def __init__(self, identity_flag: bool): if identity_flag: self.message = '\n\033[93m[!] Missing API key. \033[0m' else: diff --git a/theHarvester/discovery/googlesearch.py b/theHarvester/discovery/googlesearch.py index 09fba64a..22c4b41b 100644 --- a/theHarvester/discovery/googlesearch.py +++ b/theHarvester/discovery/googlesearch.py @@ -31,9 +31,15 @@ class SearchGoogle: print(e) self.results = r.text if search(self.results): - time.sleep(getDelay() * 5) # Sleep for a longer time. - else: - time.sleep(getDelay()) + try: + if isinstance(search(self.results), bool): + print('Google is blocking your ip and the workaround, returning') + return + else: + self.results = google_workaround(urly) + except BaseException: + pass + time.sleep(getDelay()) self.totalresults += self.results def do_search_profiles(self): @@ -49,9 +55,14 @@ class SearchGoogle: print(e) self.results = r.text if search(self.results): - time.sleep(getDelay() * 5) # Sleep for a longer time. - else: - time.sleep(getDelay()) + try: + if isinstance(search(self.results), bool): + print('Google is blocking your ip and the workaround, returning') + return + else: + self.results = google_workaround(urly) + except BaseException: + pass self.totalresults += self.results def get_emails(self): @@ -137,9 +148,18 @@ class SearchGoogle: req = requests.get(link, headers=headers) self.results = req.text if search(self.results): - time.sleep(getDelay() * 5) # Sleep for a longer time. - else: - time.sleep(getDelay()) + try: + if isinstance(search(self.results), bool): + print('Google is blocking your ip and the workaround, returning') + return + else: + self.results = google_workaround(link) + except BaseException: + pass self.totalresults += self.results except Exception as e: print(f'\tException Occurred {e}') + +if __name__ == '__main__': + x = SearchGoogle('yale.edu', 100, 0) + x.process(False) \ No newline at end of file diff --git a/theHarvester/discovery/linkedinsearch.py b/theHarvester/discovery/linkedinsearch.py index ef774df3..9aa9d31d 100644 --- a/theHarvester/discovery/linkedinsearch.py +++ b/theHarvester/discovery/linkedinsearch.py @@ -12,7 +12,6 @@ class SearchLinkedin: self.results = "" self.totalresults = "" self.server = 'www.google.com' - self.userAgent = '(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6' self.quantity = '100' self.limit = int(limit) self.counter = 0 @@ -25,9 +24,18 @@ class SearchLinkedin: try: headers = {'User-Agent': Core.get_user_agent()} r = requests.get(urly, headers=headers) + self.results = r.text + if search(self.results): + try: + if isinstance(search(self.results), bool): + print('Google is blocking your ip and the workaround, returning') + return + else: + self.results = google_workaround(urly) + except BaseException: + pass except Exception as e: print(e) - self.results = r.text self.totalresults += self.results def get_people(self):