From 9ba63bc690220c7f96298e6ccb3ddacbd73ce6fe Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Tue, 11 Feb 2020 14:25:03 -0500 Subject: [PATCH 01/10] Fixed error where would attempt to fetch with proxy as empty string. --- theHarvester/lib/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index b51db7b5..9852f99e 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -95,7 +95,7 @@ class Core: with open('/etc/theHarvester/proxies.yaml', 'r') as api_keys: keys = yaml.safe_load(api_keys) except FileNotFoundError: - with open('proxies.yaml', 'r') as api_keys: + with open('/mnt/d/repos/theHarvester/proxies.yaml', 'r') as api_keys: keys = yaml.safe_load(api_keys) http_list = [f'http://{proxy}' for proxy in keys['http']] if keys['http'] is not None else [] return http_list @@ -434,7 +434,7 @@ class AsyncFetcher: async with session.get(url, params=params, proxy=proxy) as response: return await response.text() if json is False else await response.json() else: - async with session.get(url, proxy=proxy) as response: + async with session.get(url) as response: await asyncio.sleep(2) return await response.text() if json is False else await response.json() From 11e498128540ec1aaeab313cc8d96af8f1c8693e Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Tue, 11 Feb 2020 14:31:42 -0500 Subject: [PATCH 02/10] Pushed change. --- theHarvester/lib/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 9852f99e..5efbf825 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -95,7 +95,7 @@ class Core: with open('/etc/theHarvester/proxies.yaml', 'r') as api_keys: keys = yaml.safe_load(api_keys) except FileNotFoundError: - with open('/mnt/d/repos/theHarvester/proxies.yaml', 'r') as api_keys: + with open('proxies.yaml', 'r') as api_keys: keys = yaml.safe_load(api_keys) http_list = [f'http://{proxy}' for proxy in keys['http']] if keys['http'] is not None else [] return http_list From 518774fc6d60d2efc22fc8f661c220b33d4917a7 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Wed, 12 Feb 2020 20:14:16 -0500 Subject: [PATCH 03/10] Fixed bufferoverun module. --- theHarvester/__main__.py | 13 ++++++++-- theHarvester/discovery/__init__.py | 1 + theHarvester/discovery/bufferoverun.py | 33 ++++++++++++++++++++++++++ theHarvester/lib/core.py | 3 ++- 4 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 theHarvester/discovery/bufferoverun.py diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 9bc99413..a3123b03 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -38,7 +38,7 @@ async def start(): parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False, action='store_true') parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true') parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) - parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, certspotter, crtsh, dnsdumpster, + parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, bufferoverun, certspotter, crtsh, dnsdumpster, dogpile, duckduckgo, exalead, github-code, google, hunter, intelx, linkedin, linkedin_links, netcraft, otx, securityTrails, spyse, threatcrowd, @@ -186,6 +186,14 @@ async def start(): else: print(e) + elif engineitem == 'bufferoverun': + from theHarvester.discovery import bufferoverun + try: + bufferoverun_search = bufferoverun.SearchBufferover(word) + stor_lst.append(store(bufferoverun_search, engineitem, store_host=True, store_ip=True)) + except Exception as e: + print(e) + elif engineitem == 'certspotter': from theHarvester.discovery import certspottersearch try: @@ -404,7 +412,8 @@ async def start(): print('\n[*] IPs found: ' + str(len(all_ip))) print('-------------------') # use netaddr as the list may contain ipv4 and ipv6 addresses - ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)]) + ip_list = [netaddr.IPAddress(ip.strip()) for ip in set(all_ip)] + ip_list.sort() print('\n'.join(map(str, ip_list))) if len(all_emails) == 0: diff --git a/theHarvester/discovery/__init__.py b/theHarvester/discovery/__init__.py index 35c2a131..539ab694 100644 --- a/theHarvester/discovery/__init__.py +++ b/theHarvester/discovery/__init__.py @@ -1,5 +1,6 @@ __all__ = ['baidusearch', 'bingsearch', + 'bufferoverun', 'crtsh', 'certspottersearch', 'dnssearch', diff --git a/theHarvester/discovery/bufferoverun.py b/theHarvester/discovery/bufferoverun.py new file mode 100644 index 00000000..2cea53ed --- /dev/null +++ b/theHarvester/discovery/bufferoverun.py @@ -0,0 +1,33 @@ +from theHarvester.lib.core import * +import re + + +class SearchBufferover: + def __init__(self, word): + self.word = word + self.totalhosts = set() + self.totalips = set() + self.proxy = False + + async def do_search(self): + url = f'https://dns.bufferover.run/dns?q={self.word}' + responses = await AsyncFetcher.fetch_all(urls=[url], json=True, proxy=self.proxy) + responses = responses[0] + dct = responses + + self.totalhosts: set = { + host.split(',')[0].replace('www','') if ',' in host and self.word.replace('www.', '') in host.split(',')[0] in host else + host.split(',')[1] for host in dct['FDNS_A']} + + self.totalips: set = {ip.split(',')[0]for ip in dct['FDNS_A'] if + re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip.split(',')[0])} + + async def get_hostnames(self) -> set: + return self.totalhosts + + async def get_ips(self) -> set: + return self.totalips + + async def process(self, proxy=False): + self.proxy = proxy + await self.do_search() diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 5efbf825..5d8fbc1e 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -10,7 +10,7 @@ import random class Core: @staticmethod def version() -> str: - return '3.1.1dev5' + return '3.2.0dev0' @staticmethod def bing_key() -> str: @@ -123,6 +123,7 @@ class Core: supportedengines = {'baidu', 'bing', 'bingapi', + 'bufferoverun', 'certspotter', 'crtsh', 'dnsdumpster', From 19391b9d8f17c93c86c1f5ed4ef73344192bf9f4 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Thu, 13 Feb 2020 16:03:02 -0500 Subject: [PATCH 04/10] Updated github tests most are passing. --- tests/discovery/test_githubcode.py | 15 ++++++++------- theHarvester/discovery/githubcode.py | 4 +++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/discovery/test_githubcode.py b/tests/discovery/test_githubcode.py index 78ad60d0..24644256 100644 --- a/tests/discovery/test_githubcode.py +++ b/tests/discovery/test_githubcode.py @@ -74,35 +74,36 @@ class TestSearchGithubCode: async def test_fragments_from_response(self): Core.github_key = MagicMock(return_value="lol") test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) - test_result = test_class_instance.fragments_from_response(self.OkResponse.response) + test_result = await test_class_instance.fragments_from_response(self.OkResponse.response.json()) + print('test_result: ', test_result) assert test_result == ["test1", "test2"] @pytest.mark.asyncio async def test_invalid_fragments_from_response(self): Core.github_key = MagicMock(return_value="lol") test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) - test_result = test_class_instance.fragments_from_response(self.MalformedResponse.response) + test_result = await test_class_instance.fragments_from_response(self.MalformedResponse.response.json()) assert test_result == [] @pytest.mark.asyncio async def test_handle_response_ok(self): Core.github_key = MagicMock(return_value="lol") test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) - test_result = test_class_instance.handle_response(self.OkResponse.response) + test_result = await test_class_instance.handle_response() assert isinstance(test_result, SuccessResult) @pytest.mark.asyncio async def test_handle_response_retry(self): Core.github_key = MagicMock(return_value="lol") test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) - test_result = test_class_instance.handle_response(self.RetryResponse.response) + test_result = await test_class_instance.handle_response(self.RetryResponse.response.json()) assert isinstance(test_result, RetryResult) @pytest.mark.asyncio async def test_handle_response_fail(self): Core.github_key = MagicMock(return_value="lol") test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) - test_result = test_class_instance.handle_response(self.FailureResponse.response) + test_result = await test_class_instance.handle_response(self.FailureResponse.response.json()) assert isinstance(test_result, ErrorResult) @pytest.mark.asyncio @@ -110,14 +111,14 @@ class TestSearchGithubCode: Core.github_key = MagicMock(return_value="lol") test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) test_result = githubcode.SuccessResult(list(), next_page=2, last_page=4) - assert(2 == test_class_instance.next_page_or_end(test_result)) + assert(2 == await test_class_instance.next_page_or_end(test_result)) @pytest.mark.asyncio async def test_last_page(self): Core.github_key = MagicMock(return_value="lol") test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) test_result = githubcode.SuccessResult(list(), None, None) - assert(None is test_class_instance.next_page_or_end(test_result)) + assert(None is await test_class_instance.next_page_or_end(test_result)) if __name__ == '__main__': pytest.main() diff --git a/theHarvester/discovery/githubcode.py b/theHarvester/discovery/githubcode.py index cf21dae5..f081b452 100644 --- a/theHarvester/discovery/githubcode.py +++ b/theHarvester/discovery/githubcode.py @@ -7,6 +7,7 @@ import aiohttp import urllib.parse as urlparse import random + class RetryResult(NamedTuple): time: float @@ -94,8 +95,9 @@ class SearchGithubCode: async with sess.get(url, proxy=random.choice(Core.proxy_list())) as resp: return await resp.text(), await resp.json(), resp.status, resp.links else: - async with sess.get(url, ) as resp: + async with sess.get(url) as resp: return await resp.text(), await resp.json(), resp.status, resp.links + @staticmethod async def next_page_or_end(result: SuccessResult) -> Optional[int]: if result.next_page is not None: From c1e9e0c9c10391c1ab8b0229d2699a99e639f90b Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Thu, 13 Feb 2020 17:19:51 -0500 Subject: [PATCH 05/10] removed unncessary tests. --- tests/discovery/githubcode.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/discovery/githubcode.py b/tests/discovery/githubcode.py index da88196c..df8b6d28 100644 --- a/tests/discovery/githubcode.py +++ b/tests/discovery/githubcode.py @@ -84,24 +84,6 @@ class TestSearchGithubCode: test_result = await test_class_instance.fragments_from_response(self.MalformedResponse.response.json()) assert test_result == [] - async def test_handle_response_ok(self): - Core.github_key = MagicMock(return_value="lol") - test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) - test_result = await test_class_instance.handle_response() - assert isinstance(test_result, SuccessResult) - - async def test_handle_response_retry(self): - Core.github_key = MagicMock(return_value="lol") - test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) - test_result = await test_class_instance.handle_response(self.RetryResponse.response.json()) - assert isinstance(test_result, RetryResult) - - async def test_handle_response_fail(self): - Core.github_key = MagicMock(return_value="lol") - test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) - test_result = await test_class_instance.handle_response(self.FailureResponse.response.json()) - assert isinstance(test_result, ErrorResult) - async def test_next_page(self): Core.github_key = MagicMock(return_value="lol") test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) From 7f8aea33ad2bff59f7cd6a6b29516511ffaf50f4 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Thu, 13 Feb 2020 17:32:17 -0500 Subject: [PATCH 06/10] flake8 fixes. --- tests/discovery/githubcode.py | 1 - theHarvester/discovery/bufferoverun.py | 6 ++++-- theHarvester/discovery/certspottersearch.py | 10 ++++++++++ theHarvester/discovery/crtsh.py | 1 + 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/discovery/githubcode.py b/tests/discovery/githubcode.py index df8b6d28..248e3038 100644 --- a/tests/discovery/githubcode.py +++ b/tests/discovery/githubcode.py @@ -1,5 +1,4 @@ from theHarvester.discovery import githubcode -from theHarvester.discovery.githubcode import RetryResult, ErrorResult, SuccessResult from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import Core from unittest.mock import MagicMock diff --git a/theHarvester/discovery/bufferoverun.py b/theHarvester/discovery/bufferoverun.py index d2e07d6d..90068b95 100644 --- a/theHarvester/discovery/bufferoverun.py +++ b/theHarvester/discovery/bufferoverun.py @@ -16,11 +16,13 @@ class SearchBufferover: dct = responses self.totalhosts: set = { - host.split(',')[0].replace('www','') if ',' in host and self.word.replace('www.', '') in host.split(',')[0] in host else + host.split(',')[0].replace('www', '') if ',' in host and self.word.replace('www.', '') in host.split(',')[ + 0] in host else host.split(',')[1] for host in dct['FDNS_A']} - self.totalips: set = {ip.split(',')[0]for ip in dct['FDNS_A'] if + self.totalips: set = {ip.split(',')[0] for ip in dct['FDNS_A'] if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip.split(',')[0])} + async def get_hostnames(self) -> set: return self.totalhosts diff --git a/theHarvester/discovery/certspottersearch.py b/theHarvester/discovery/certspottersearch.py index 70cc9385..1aa49a5a 100644 --- a/theHarvester/discovery/certspottersearch.py +++ b/theHarvester/discovery/certspottersearch.py @@ -35,3 +35,13 @@ class SearchCertspoter: self.proxy = proxy await self.do_search() print('\tSearching results.') + +async def main(): + x = SearchCertspoter(word='arrow.com') + await x.do_search() + hosts = await x.get_hostnames() + print(hosts) + +if __name__ == '__main__': + import asyncio + asyncio.run(main()) \ No newline at end of file diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py index 71700276..671e9f94 100644 --- a/theHarvester/discovery/crtsh.py +++ b/theHarvester/discovery/crtsh.py @@ -1,5 +1,6 @@ from theHarvester.lib.core import * import aiohttp +from typing import Set class SearchCrtsh: From eba479a6106164977316c6c566dd4b0bbf90c40f Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Thu, 13 Feb 2020 17:51:20 -0500 Subject: [PATCH 07/10] small misc. changes. --- theHarvester/discovery/bufferoverun.py | 2 +- theHarvester/discovery/certspottersearch.py | 10 ---------- theHarvester/discovery/crtsh.py | 3 ++- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/theHarvester/discovery/bufferoverun.py b/theHarvester/discovery/bufferoverun.py index 90068b95..c72c4b36 100644 --- a/theHarvester/discovery/bufferoverun.py +++ b/theHarvester/discovery/bufferoverun.py @@ -16,7 +16,7 @@ class SearchBufferover: dct = responses self.totalhosts: set = { - host.split(',')[0].replace('www', '') if ',' in host and self.word.replace('www.', '') in host.split(',')[ + host.split(',')[0].replace('www.', '') if ',' in host and self.word.replace('www.', '') in host.split(',')[ 0] in host else host.split(',')[1] for host in dct['FDNS_A']} diff --git a/theHarvester/discovery/certspottersearch.py b/theHarvester/discovery/certspottersearch.py index 1aa49a5a..70cc9385 100644 --- a/theHarvester/discovery/certspottersearch.py +++ b/theHarvester/discovery/certspottersearch.py @@ -35,13 +35,3 @@ class SearchCertspoter: self.proxy = proxy await self.do_search() print('\tSearching results.') - -async def main(): - x = SearchCertspoter(word='arrow.com') - await x.do_search() - hosts = await x.get_hostnames() - print(hosts) - -if __name__ == '__main__': - import asyncio - asyncio.run(main()) \ No newline at end of file diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py index 671e9f94..9a828c2f 100644 --- a/theHarvester/discovery/crtsh.py +++ b/theHarvester/discovery/crtsh.py @@ -19,7 +19,8 @@ class SearchCrtsh: response = await AsyncFetcher.fetch(client, url, json=True, proxy=self.proxy) await client.close() data = set( - [dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in response]) + [dict(dct)['name_value'][2:] if '*.' == dict(dct)['name_value'][:2] else dict(dct)['name_value'] + for dct in response]) except Exception as e: print(e) return data From 42f0142aa932f7988dea54b1449dc8dc25f6090c Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Thu, 13 Feb 2020 18:05:10 -0500 Subject: [PATCH 08/10] fixed pep8 error. --- theHarvester/discovery/bufferoverun.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/theHarvester/discovery/bufferoverun.py b/theHarvester/discovery/bufferoverun.py index 2cea53ed..c72c4b36 100644 --- a/theHarvester/discovery/bufferoverun.py +++ b/theHarvester/discovery/bufferoverun.py @@ -16,10 +16,11 @@ class SearchBufferover: dct = responses self.totalhosts: set = { - host.split(',')[0].replace('www','') if ',' in host and self.word.replace('www.', '') in host.split(',')[0] in host else + host.split(',')[0].replace('www.', '') if ',' in host and self.word.replace('www.', '') in host.split(',')[ + 0] in host else host.split(',')[1] for host in dct['FDNS_A']} - self.totalips: set = {ip.split(',')[0]for ip in dct['FDNS_A'] if + self.totalips: set = {ip.split(',')[0] for ip in dct['FDNS_A'] if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip.split(',')[0])} async def get_hostnames(self) -> set: From d90f94066879ca27ba313d655d396a46f1884ba6 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Thu, 13 Feb 2020 18:45:52 -0500 Subject: [PATCH 09/10] Updated certspotter search and changed github code to start with test. --- tests/discovery/{githubcode.py => test_githubcode.py} | 0 theHarvester/discovery/certspottersearch.py | 6 ++---- 2 files changed, 2 insertions(+), 4 deletions(-) rename tests/discovery/{githubcode.py => test_githubcode.py} (100%) diff --git a/tests/discovery/githubcode.py b/tests/discovery/test_githubcode.py similarity index 100% rename from tests/discovery/githubcode.py rename to tests/discovery/test_githubcode.py diff --git a/theHarvester/discovery/certspottersearch.py b/theHarvester/discovery/certspottersearch.py index 70cc9385..b29fbb1f 100644 --- a/theHarvester/discovery/certspottersearch.py +++ b/theHarvester/discovery/certspottersearch.py @@ -11,11 +11,9 @@ class SearchCertspoter: async def do_search(self) -> None: base_url = f'https://api.certspotter.com/v1/issuances?domain={self.word}&expand=dns_names' - headers = {'User-Agent': Core.get_user_agent()} try: - client = aiohttp.ClientSession(headers=headers, timeout=aiohttp.ClientTimeout(total=30)) - response = await AsyncFetcher.fetch(client, base_url, json=True, proxy=self.proxy) - await client.close() + response = await AsyncFetcher.fetch_all([base_url], json=True, proxy=self.proxy) + response = response[0] if isinstance(response, list): for dct in response: for key, value in dct.items(): From a11a6fcc551327224ba8cd6179e79f8a3b62bf1e Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Thu, 13 Feb 2020 19:01:36 -0500 Subject: [PATCH 10/10] Fixed pep8 unused import. --- theHarvester/discovery/certspottersearch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/theHarvester/discovery/certspottersearch.py b/theHarvester/discovery/certspottersearch.py index b29fbb1f..bbb97791 100644 --- a/theHarvester/discovery/certspottersearch.py +++ b/theHarvester/discovery/certspottersearch.py @@ -1,5 +1,4 @@ from theHarvester.lib.core import * -import aiohttp class SearchCertspoter: