diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index ff0d131b..7c644ab0 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -29,78 +29,78 @@ jobs: - name: Run theHarvester module baidu run: | - python theHarvester.py -d apple.com -b baidu + python theHarvester.py -d yale.edu -b baidu - name: Run theHarvester module bing run: | - python theHarvester.py -d apple.com -b bing + python theHarvester.py -d yale.edu -b bing - name: Run theHarvester module certspotter run: | - python theHarvester.py -d apple.com -b certspotter + python theHarvester.py -d yale.edu -b certspotter - name: Run theHarvester module crtsh run: | - python theHarvester.py -d apple.com -b crtsh + python theHarvester.py -d yale.edu -b crtsh - name: Run theHarvester module dnsdumpster run: | - python theHarvester.py -d apple.com -b dnsdumpster + python theHarvester.py -d yale.edu -b dnsdumpster - name: Run theHarvester module dogplie run: | - python theHarvester.py -d apple.com -b dogpile + python theHarvester.py -d yale.edu -b dogpile - name: Run theHarvester module duckduckgo run: | - python theHarvester.py -d apple.com -b duckduckgo + python theHarvester.py -d yale.edu -b duckduckgo - name: Run theHarvester module exalead run: | - python theHarvester.py -d apple.com -b exalead + python theHarvester.py -d yale.edu -b exalead - name: Run theHarvester module google run: | - python theHarvester.py -d apple.com -b google + python theHarvester.py -d yale.edu -b google - name: Run theHarvester module Intelx run: | - python theHarvester.py -d apple.com -b intelx + python theHarvester.py -d yale.edu -b intelx - name: Run theHarvester module linkedin run: | - python theHarvester.py -d apple.com -b linkedin + python theHarvester.py -d yale.edu -b linkedin - name: Run theHarvester module linkedin_links run: | - python theHarvester.py -d apple.com -b linkedin_links + python theHarvester.py -d yale.edu -b linkedin_links - name: Run theHarvester module netcraft run: | - python theHarvester.py -d apple.com -b netcraft + python theHarvester.py -d yale.edu -b netcraft - name: Run theHarvester module Otx run: | - python theHarvester.py -d apple.com -b otx + python theHarvester.py -d yale.edu -b otx - name: Run theHarvester module threatcrowd run: | - python theHarvester.py -d apple.com -b threatcrowd + python theHarvester.py -d yale.edu -b threatcrowd - name: Run theHarvester module trello run: | - python theHarvester.py -d apple.com -b trello + python theHarvester.py -d yale.edu -b trello - name: Run theHarvester module twitter run: | - python theHarvester.py -d apple.com -b twitter + python theHarvester.py -d yale.edu -b twitter - name: Run theHarvester module virustotal run: | - python theHarvester.py -d apple.com -b virustotal + python theHarvester.py -d yale.edu -b virustotal - name: Run theHarvester module yahoo run: | - python theHarvester.py -d apple.com -b yahoo + python theHarvester.py -d yale.edu -b yahoo - name: Lint with flake8 run: | diff --git a/.gitignore b/.gitignore index 3f36fa1e..6b185864 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ venv .pytest_cache build/ dist/ -theHarvester.egg-info/ \ No newline at end of file +theHarvester.egg-info/ +api-keys.yaml diff --git a/theHarvester/discovery/baidusearch.py b/theHarvester/discovery/baidusearch.py index c2a7fb11..bb8a63bd 100644 --- a/theHarvester/discovery/baidusearch.py +++ b/theHarvester/discovery/baidusearch.py @@ -27,8 +27,8 @@ class SearchBaidu: async def get_emails(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.emails() + return await rawres.emails() async def get_hostnames(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.hostnames() + return await rawres.hostnames() diff --git a/theHarvester/discovery/bingsearch.py b/theHarvester/discovery/bingsearch.py index 73c1a7bb..666dddd5 100644 --- a/theHarvester/discovery/bingsearch.py +++ b/theHarvester/discovery/bingsearch.py @@ -1,7 +1,6 @@ from theHarvester.discovery.constants import * from theHarvester.lib.core import * from theHarvester.parsers import myparser -from theHarvester.lib.core import AsyncFetcher class SearchBing: @@ -58,15 +57,15 @@ class SearchBing: async def get_emails(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.emails() + return await rawres.emails() async def get_hostnames(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.hostnames() + return await rawres.hostnames() async def get_allhostnames(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.hostnames_all() + return await rawres.hostnames_all() async def process(self, api): if api == 'yes': diff --git a/theHarvester/discovery/dnsdumpster.py b/theHarvester/discovery/dnsdumpster.py index c4b12764..021b75c7 100644 --- a/theHarvester/discovery/dnsdumpster.py +++ b/theHarvester/discovery/dnsdumpster.py @@ -38,7 +38,7 @@ class SearchDnsDumpster: async def get_hostnames(self): rawres = myparser.Parser(self.totalresults, self.word) - return rawres.hostnames() + return await rawres.hostnames() async def process(self): await self.do_search() # Only need to do it once. diff --git a/theHarvester/discovery/dogpilesearch.py b/theHarvester/discovery/dogpilesearch.py index e15d503d..7cc30221 100644 --- a/theHarvester/discovery/dogpilesearch.py +++ b/theHarvester/discovery/dogpilesearch.py @@ -28,8 +28,8 @@ class SearchDogpile: async def get_emails(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.emails() + return await rawres.emails() async def get_hostnames(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.hostnames() + return await rawres.hostnames() diff --git a/theHarvester/discovery/duckduckgosearch.py b/theHarvester/discovery/duckduckgosearch.py index 7900102c..a8f80532 100644 --- a/theHarvester/discovery/duckduckgosearch.py +++ b/theHarvester/discovery/duckduckgosearch.py @@ -73,11 +73,11 @@ class SearchDuckDuckGo: async def get_emails(self): rawres = myparser.Parser(self.totalresults, self.word) - return rawres.emails() + return await rawres.emails() async def get_hostnames(self): rawres = myparser.Parser(self.totalresults, self.word) - return rawres.hostnames() + return await rawres.hostnames() async def process(self): await self.do_search() # Only need to search once since using API. diff --git a/theHarvester/discovery/exaleadsearch.py b/theHarvester/discovery/exaleadsearch.py index 0ca0c205..19f9256b 100644 --- a/theHarvester/discovery/exaleadsearch.py +++ b/theHarvester/discovery/exaleadsearch.py @@ -52,15 +52,15 @@ class SearchExalead: async def get_emails(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.emails() + return await rawres.emails() async def get_hostnames(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.hostnames() + return await rawres.hostnames() async def get_files(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.fileurls(self.files) + return await rawres.fileurls(self.files) async def process(self): print('Searching results') diff --git a/theHarvester/discovery/githubcode.py b/theHarvester/discovery/githubcode.py index 3c79379e..aa9caa0c 100644 --- a/theHarvester/discovery/githubcode.py +++ b/theHarvester/discovery/githubcode.py @@ -1,10 +1,9 @@ from theHarvester.discovery.constants import * from theHarvester.lib.core import * from theHarvester.parsers import myparser -import requests -from requests import Response -import time -from typing import List, Dict, Any, Optional, NamedTuple +from typing import List, Dict, Any, Optional, NamedTuple, Tuple +import asyncio +import aiohttp import urllib.parse as urlparse @@ -40,20 +39,21 @@ class SearchGithubCode: raise MissingKey(True) @staticmethod - def fragments_from_response(response: Response) -> List[str]: - items: List[Dict[str, Any]] = response.json().get('items') or list() + async def fragments_from_response(json_data: dict) -> List[str]: + items: List[Dict[str, Any]] = json_data.get('items') or list() fragments: List[str] = list() for item in items: matches = item.get("text_matches") or list() for match in matches: fragments.append(match.get("fragment")) + return [fragment for fragment in fragments if fragment is not None] @staticmethod - def page_from_response(page: str, response: Response) -> Optional[Any]: - page_link = response.links.get(page) + async def page_from_response(page: str, links) -> Optional[Any]: + page_link = links.get(page) if page_link: - parsed = urlparse.urlparse(page_link.get("url")) + parsed = urlparse.urlparse(str(page_link.get("url"))) params = urlparse.parse_qs(parsed.query) pages: List[Any] = params.get('page', [None]) page_number = pages[0] and int(pages[0]) @@ -61,21 +61,22 @@ class SearchGithubCode: else: return None - def handle_response(self, response: Response) -> Optional[Any]: - if response.ok: - results = self.fragments_from_response(response) - next_page = self.page_from_response("next", response) - last_page = self.page_from_response("last", response) + async def handle_response(self, response: Tuple[str, dict, int, Any]): + text, json_data, status, links = response + if status == 200: + results = await self.fragments_from_response(json_data) + next_page = await self.page_from_response("next", links) + last_page = await self.page_from_response("last", links) return SuccessResult(results, next_page, last_page) - elif response.status_code == 429 or response.status_code == 403: + elif status == 429 or status == 403: return RetryResult(60) else: try: - return ErrorResult(response.status_code, response.json()) + return ErrorResult(status, json_data) except ValueError: - return ErrorResult(response.status_code, response.text) + return ErrorResult(status, text) - def do_search(self, page: Optional[int]) -> Response: + async def do_search(self, page: Optional[int]) -> Tuple[str, dict, int, Any]: if page is None: url = f'https://{self.server}/search/code?q="{self.word}"' else: @@ -84,42 +85,46 @@ class SearchGithubCode: 'Host': self.server, 'User-agent': Core.get_user_agent(), 'Accept': "application/vnd.github.v3.text-match+json", - 'Authorization': 'token {}'.format(self.key) + 'Authorization': f'token {self.key}' } - return requests.get(url=url, headers=headers, verify=True) + async with aiohttp.ClientSession(headers=headers) as sess: + async with sess.get(url) as resp: + return await resp.text(), await resp.json(), resp.status, resp.links @staticmethod - def next_page_or_end(result: SuccessResult) -> Optional[int]: + async def next_page_or_end(result: SuccessResult) -> Optional[int]: if result.next_page is not None: return result.next_page else: return result.last_page - def process(self): - while self.counter <= self.limit and self.page is not None: - api_response = self.do_search(self.page) - result = self.handle_response(api_response) - if type(result) == SuccessResult: - print(f'\tSearching {self.counter} results.') - for fragment in result.fragments: - self.total_results += fragment - self.counter = self.counter + 1 + async def process(self): + try: + while self.counter <= self.limit and self.page is not None: + api_response = await self.do_search(self.page) + result = await self.handle_response(api_response) + if type(result) == SuccessResult: + print(f'\tSearching {self.counter} results.') + for fragment in result.fragments: + self.total_results += fragment + self.counter = self.counter + 1 + self.page = await self.next_page_or_end(result) + await asyncio.sleep(getDelay()) + elif type(result) == RetryResult: + sleepy_time = getDelay() + result.time + print(f'\tRetrying page in {sleepy_time} seconds...') + await asyncio.sleep(sleepy_time) + elif type(result) == ErrorResult: + raise Exception(f"\tException occurred: status_code: {result.status_code} reason: {result.body}") + else: + raise Exception("\tUnknown exception occurred") + except Exception as e: + print(f'An exception has occurred: {e}') - self.page = self.next_page_or_end(result) - time.sleep(getDelay()) - elif type(result) == RetryResult: - sleepy_time = getDelay() + result.time - print(f'\tRetrying page in {sleepy_time} seconds...') - time.sleep(sleepy_time) - elif type(result) == ErrorResult: - raise Exception(f"\tException occurred: status_code: {result.status_code} reason: {result.body}") - else: - raise Exception("\tUnknown exception occurred") - - def get_emails(self): + async def get_emails(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.emails() + return await rawres.emails() - def get_hostnames(self): + async def get_hostnames(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.hostnames() + return await rawres.hostnames() diff --git a/theHarvester/discovery/huntersearch.py b/theHarvester/discovery/huntersearch.py index e9ca90cf..b1a5760e 100644 --- a/theHarvester/discovery/huntersearch.py +++ b/theHarvester/discovery/huntersearch.py @@ -25,12 +25,12 @@ class SearchHunter: async def get_emails(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.emails() + return await rawres.emails() async def get_hostnames(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.hostnames() + return await rawres.hostnames() async def get_profiles(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.profiles() + return await rawres.profiles() diff --git a/theHarvester/discovery/netcraft.py b/theHarvester/discovery/netcraft.py index 498cafd4..33a2fb48 100644 --- a/theHarvester/discovery/netcraft.py +++ b/theHarvester/discovery/netcraft.py @@ -91,7 +91,7 @@ class SearchNetcraft: async def get_hostnames(self): rawres = myparser.Parser(self.totalresults, self.word) - return rawres.hostnames() + return await rawres.hostnames() async def process(self): await self.do_search() diff --git a/theHarvester/discovery/securitytrailssearch.py b/theHarvester/discovery/securitytrailssearch.py index 9b30b9be..b83f883d 100644 --- a/theHarvester/discovery/securitytrailssearch.py +++ b/theHarvester/discovery/securitytrailssearch.py @@ -53,4 +53,3 @@ class SearchSecuritytrail: async def get_hostnames(self) -> set: return self.info[1] - \ No newline at end of file diff --git a/theHarvester/discovery/threatcrowd.py b/theHarvester/discovery/threatcrowd.py index 2b396c4e..cbdbcda3 100644 --- a/theHarvester/discovery/threatcrowd.py +++ b/theHarvester/discovery/threatcrowd.py @@ -1,3 +1,4 @@ +from typing import Coroutine from theHarvester.lib.core import * from theHarvester.parsers import myparser @@ -19,7 +20,7 @@ class SearchThreatcrowd: print(e) self.totalresults += self.results - async def get_hostnames(self) -> set: + async def get_hostnames(self) -> Coroutine: return myparser.Parser(self.results, self.word).hostnames() async def process(self): diff --git a/theHarvester/discovery/twittersearch.py b/theHarvester/discovery/twittersearch.py index e4a04e7b..b5f9f345 100644 --- a/theHarvester/discovery/twittersearch.py +++ b/theHarvester/discovery/twittersearch.py @@ -14,21 +14,20 @@ class SearchTwitter: self.limit = int(limit) self.counter = 0 - def do_search(self): + async def do_search(self): base_url = f'https://{self.server}/search?num=100&start=xx&hl=en&meta=&q=site%3Atwitter.com%20intitle%3A%22on+Twitter%22%20{self.word}' headers = {'User-Agent': Core.get_user_agent()} try: urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 10) if num <= self.limit] - request = (grequests.get(url, headers=headers) for url in urls) - response = grequests.imap(request, size=5) - for entry in response: - self.totalresults += entry.content.decode('UTF-8') + responses = await AsyncFetcher.fetch_all(urls, headers=headers) + for response in responses: + self.totalresults += response except Exception as error: print(error) - def get_people(self): + async def get_people(self): rawres = myparser.Parser(self.totalresults, self.word) - to_parse = rawres.people_twitter() + to_parse = await rawres.people_twitter() # fix invalid handles that look like @user other_output handles = set() for handle in to_parse: @@ -37,5 +36,5 @@ class SearchTwitter: handles.add(result.group(0)) return handles - def process(self): - self.do_search() + async def process(self): + await self.do_search() diff --git a/theHarvester/discovery/virustotal.py b/theHarvester/discovery/virustotal.py index 852a6947..ef16c018 100644 --- a/theHarvester/discovery/virustotal.py +++ b/theHarvester/discovery/virustotal.py @@ -20,7 +20,7 @@ class SearchVirustotal: async def get_hostnames(self): rawres = myparser.Parser(self.results, self.word) - return rawres.hostnames() + return await rawres.hostnames() async def process(self): print('\tSearching results.') diff --git a/theHarvester/discovery/yahoosearch.py b/theHarvester/discovery/yahoosearch.py index 562def0f..f02f480f 100644 --- a/theHarvester/discovery/yahoosearch.py +++ b/theHarvester/discovery/yahoosearch.py @@ -26,7 +26,7 @@ class SearchYahoo: async def get_emails(self): rawres = myparser.Parser(self.total_results, self.word) - toparse_emails = rawres.emails() + toparse_emails = await rawres.emails() emails = set() # strip out numbers and dashes for emails that look like xxx-xxx-xxxemail@host.tld for email in toparse_emails: @@ -39,4 +39,4 @@ class SearchYahoo: async def get_hostnames(self): rawres = myparser.Parser(self.total_results, self.word) - return rawres.hostnames() + return await rawres.hostnames() diff --git a/theHarvester/parsers/myparser.py b/theHarvester/parsers/myparser.py index 16a74865..e2cfb0ac 100644 --- a/theHarvester/parsers/myparser.py +++ b/theHarvester/parsers/myparser.py @@ -8,7 +8,7 @@ class Parser: self.word = word self.temp = [] - def genericClean(self): + async def genericClean(self): self.results = self.results.replace('', '').replace('', '').replace('', '').replace('', '')\ .replace('%2f', '').replace('%3a', '').replace('', '').replace('', '')\ .replace('', '').replace('', '') @@ -16,28 +16,28 @@ class Parser: for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '/', '\\'): self.results = self.results.replace(search, ' ') - def urlClean(self): + async def urlClean(self): self.results = self.results.replace('', '').replace('', '').replace('%2f', '').replace('%3a', '') for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C'): self.results = self.results.replace(search, ' ') - def emails(self): - self.genericClean() + async def emails(self): + await self.genericClean() # Local part is required, charset is flexible. # https://tools.ietf.org/html/rfc6531 (removed * and () as they provide FP mostly) reg_emails = re.compile(r'[a-zA-Z0-9.\-_+#~!$&\',;=:]+' + '@' + '[a-zA-Z0-9.-]*' + self.word.replace('www.', '')) self.temp = reg_emails.findall(self.results) - emails = self.unique() + emails = await self.unique() true_emails = {str(email)[1:].lower().strip() if len(str(email)) > 1 and str(email)[0] == '.' else len(str(email)) > 1 and str(email).lower().strip() for email in emails} # if email starts with dot shift email string and make sure all emails are lowercase return true_emails - def fileurls(self, file): + async def fileurls(self, file): urls = [] reg_urls = re.compile('', '', self.results) self.results = re.sub('', '', self.results) reg_people = re.compile(r'>[a-zA-Z0-9._ ]* - Google\+') @@ -71,7 +71,7 @@ class Parser: resul.append(delete) return resul - def hostnames_all(self): + async def hostnames_all(self): reg_hosts = re.compile('(.*?)') temp = reg_hosts.findall(self.results) for iteration in temp: @@ -80,10 +80,10 @@ class Parser: else: res = iteration.split('/')[0] self.temp.append(res) - hostnames = self.unique() + hostnames = await self.unique() return hostnames - def links_linkedin(self): + async def links_linkedin(self): reg_links = re.compile(r"url=https:\/\/www\.linkedin.com(.*?)&") self.temp = reg_links.findall(self.results) resul = [] @@ -92,7 +92,7 @@ class Parser: resul.append("https://www.linkedin.com" + final_url) return resul - def people_linkedin(self): + async def people_linkedin(self): reg_people = re.compile(r'">[a-zA-Z0-9._ -]* \| LinkedIn') self.temp = reg_people.findall(self.results) resul = [] @@ -106,10 +106,10 @@ class Parser: resul.append(delete) return resul - def people_twitter(self): + async def people_twitter(self): reg_people = re.compile(r'(@[a-zA-Z0-9._ -]*)') self.temp = reg_people.findall(self.results) - users = self.unique() + users = await self.unique() resul = [] for iteration in users: delete = iteration.replace(' | LinkedIn', '') @@ -121,7 +121,7 @@ class Parser: resul.append(delete) return resul - def profiles(self): + async def profiles(self): reg_people = re.compile(r'">[a-zA-Z0-9._ -]* - Google Profile') self.temp = reg_people.findall(self.results) resul = [] @@ -133,7 +133,7 @@ class Parser: resul.append(delete) return resul - def set(self): + async def set(self): reg_sets = re.compile(r'>[a-zA-Z0-9]*') self.temp = reg_sets.findall(self.results) sets = [] @@ -143,10 +143,10 @@ class Parser: sets.append(delete) return sets - def urls(self): + async def urls(self): found = re.finditer(r'(http|https)://(www\.)?trello.com/([a-zA-Z0-9\-_\.]+/?)*', self.results) urls = {match.group().strip() for match in found} return urls - def unique(self) -> list: + async def unique(self) -> list: return list(set(self.temp))