diff --git a/README.md b/README.md index efc60dfb..09d5366d 100644 --- a/README.md +++ b/README.md @@ -85,11 +85,12 @@ Add your keys to api-keys.yaml * shodan * spyse -Dependencies: +Install & Dependencies: ------------- * Python 3.7+ * python3 -m pip install pipenv -* pipenv install +* https://github.com/laramies/theHarvester/wiki/Installation + Comments, bugs, or requests? ---------------------------- diff --git a/tests/test_theHarvester.py b/tests/test_theHarvester.py deleted file mode 100644 index aca3b5e3..00000000 --- a/tests/test_theHarvester.py +++ /dev/null @@ -1,218 +0,0 @@ -import os -import sys -from unittest.mock import patch, MagicMock -import theHarvester.__main__ as harvester -import pytest - -domain = 'metasploit.com' -sys.argv = args = [os.path.curdir + 'theHarvester.py', '-d', domain, '-b', 'domain'] - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.baidusearch.SearchBaidu') -@patch('theHarvester.lib.stash.StashManager') -async def test_baidu(stash, search_engine): - args[-1] = 'baidu' - await harvester.start() - assert stash().store_all.call_count == 2 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.bingsearch.SearchBing') -@patch('theHarvester.lib.stash.StashManager') -async def test_bing(stash, search_engine): - args[-1] = 'bing' - await harvester.start() - args[-1] = 'bingapi' - await harvester.start() - assert stash().store_all.call_count == 4 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.certspottersearch.SearchCertspoter') -@patch('theHarvester.lib.stash.StashManager') -async def test_certspotter(stash, search_engine): - args[-1] = 'certspotter' - await harvester.start() - assert stash().store_all.call_count == 1 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.crtsh.SearchCrtsh') -@patch('theHarvester.lib.stash.StashManager') -async def test_crtsh(stash, search_engine): - args[-1] = 'crtsh' - await harvester.start() - assert stash().store_all.call_count == 1 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.dnsdumpster.SearchDnsDumpster') -@patch('theHarvester.lib.stash.StashManager') -async def test_dnsdumpster(stash, search_engine): - args[-1] = 'dnsdumpster' - await harvester.start() - assert stash().store_all.call_count == 1 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.dogpilesearch.SearchDogpile') -@patch('theHarvester.lib.stash.StashManager') -async def test_dogpile(stash, search_engine): - args[-1] = 'dogpile' - await harvester.start() - assert stash().store_all.call_count == 2 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.duckduckgosearch.SearchDuckDuckGo') -@patch('theHarvester.lib.stash.StashManager') -async def test_duckduckgo(stash, search_engine): - args[-1] = 'duckduckgo' - await harvester.start() - assert stash().store_all.call_count == 2 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.githubcode.SearchGithubCode') -@patch('theHarvester.lib.stash.StashManager') -async def test_github(stash, search_engine): - args[-1] = 'github-code' - await harvester.start() - assert stash().store_all.call_count == 2 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.exaleadsearch.SearchExalead') -@patch('theHarvester.lib.stash.StashManager') -async def test_exalead(stash, search_engine): - args[-1] = 'exalead' - await harvester.start() - assert stash().store_all.call_count == 2 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.googlesearch.SearchGoogle') -@patch('theHarvester.lib.stash.StashManager') -async def test_google(stash, search_engine): - args[-1] = 'google' - await harvester.start() - assert stash().store_all.call_count == 2 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.huntersearch.SearchHunter') -@patch('theHarvester.lib.stash.StashManager') -async def test_hunter(stash, search_engine): - args[-1] = 'hunter' - await harvester.start() - assert stash().store_all.call_count == 2 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.intelxsearch.SearchIntelx') -@patch('theHarvester.lib.stash.StashManager') -async def test_intelx(stash, search_engine): - args[-1] = 'intelx' - await harvester.start() - assert stash().store_all.call_count == 2 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.linkedinsearch.SearchLinkedin') -@patch('theHarvester.lib.stash.StashManager') -async def test_linkedin(stash, search_engine): - args[-1] = 'linkedin' - await harvester.start() - assert stash().store_all.call_count == 1 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.linkedinsearch.SearchLinkedin') -@patch('theHarvester.lib.stash.StashManager') -async def test_linkedin_links(stash, search_engine): - args[-1] = 'linkedin_links' - await harvester.start() - assert stash().store_all.call_count == 1 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.netcraft.SearchNetcraft') -@patch('theHarvester.lib.stash.StashManager') -async def test_netcraft(stash, search_engine): - args[-1] = 'netcraft' - await harvester.start() - assert stash().store_all.call_count == 1 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.otxsearch.SearchOtx') -@patch('theHarvester.lib.stash.StashManager') -async def test_otx(stash, search_engine): - args[-1] = 'otx' - await harvester.start() - assert stash().store_all.call_count == 2 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.securitytrailssearch.SearchSecuritytrail') -@patch('theHarvester.lib.stash.StashManager') -async def test_security_trails(stash, search_engine): - args[-1] = 'securityTrails' - await harvester.start() - assert stash().store_all.call_count == 2 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.suip.SearchSuip') -@patch('theHarvester.lib.stash.StashManager') -async def test_suip(stash, search_engine): - args[-1] = 'suip' - await harvester.start() - assert stash().store_all.call_count == 1 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.threatcrowd.SearchThreatcrowd') -@patch('theHarvester.lib.stash.StashManager') -async def test_threatcrowd(stash, search_engine): - args[-1] = 'threatcrowd' - await harvester.start() - assert stash().store_all.call_count == 1 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.trello.SearchTrello') -@patch('theHarvester.lib.stash.StashManager') -async def test_trello(stash, search_engine): - search_engine().get_results = MagicMock(return_value=('user@trello.com', 'trello', 'trello.com')) - args[-1] = 'trello' - await harvester.start() - assert stash().store_all.call_count == 3 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.twittersearch.SearchTwitter') -@patch('theHarvester.lib.stash.StashManager') -async def test_twitter(stash, search_engine): - args[-1] = 'twitter' - await harvester.start() - assert stash().store_all.call_count == 1 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.virustotal.SearchVirustotal') -@patch('theHarvester.lib.stash.StashManager') -async def test_virustotal(stash, search_engine): - args[-1] = 'virustotal' - await harvester.start() - assert stash().store_all.call_count == 1 - - -@pytest.mark.asyncio -@patch('theHarvester.discovery.yahoosearch.SearchYahoo') -@patch('theHarvester.lib.stash.StashManager') -async def test_yahoo(stash, search_engine): - args[-1] = 'yahoo' - await harvester.start() - assert stash().store_all.call_count == 2 diff --git a/theHarvester.py b/theHarvester.py index 4be68223..707860ff 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -13,4 +13,4 @@ if python_version()[0:3] < '3.7': from theHarvester import __main__ uvloop.install() -asyncio.run(__main__.entry_point(), debug=True) +asyncio.run(__main__.entry_point()) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 3021fc5a..09c9a1ff 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -35,9 +35,9 @@ async def start(): parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true') parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, certspotter, crtsh, dnsdumpster, - dogpile, duckduckgo, github-code, google, + dogpile, duckduckgo, exalead, github-code, google, hunter, intelx, - linkedin, linkedin_links, netcraft, otx, securityTrails, spyse(disabled for now), threatcrowd, + linkedin, linkedin_links, netcraft, otx, securityTrails, spyse, threatcrowd, trello, twitter, vhost, virustotal, yahoo, all''') args = parser.parse_args() @@ -306,14 +306,15 @@ async def start(): print(e) elif engineitem == 'spyse': - from theHarvester.discovery import spyse - try: - pass - #spysesearch = spyse.SearchSpyse(word) - #spysesearch.process() - - except Exception as e: - print(e) + from theHarvester.discovery import spyse + try: + spysesearch = spyse.SearchSpyse(word) + stor_lst.append(store(spysesearch, engineitem, store_host=True)) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + pass elif engineitem == 'threatcrowd': from theHarvester.discovery import threatcrowd @@ -677,4 +678,4 @@ async def entry_point(): if __name__ == '__main__': - asyncio.run(main=entry_point(), debug=True) + asyncio.run(main=entry_point()) diff --git a/theHarvester/discovery/otxsearch.py b/theHarvester/discovery/otxsearch.py index e9c9df65..94aa153d 100644 --- a/theHarvester/discovery/otxsearch.py +++ b/theHarvester/discovery/otxsearch.py @@ -1,6 +1,6 @@ from theHarvester.lib.core import * import json -import grequests +#import grequests import re diff --git a/theHarvester/discovery/spyse.py b/theHarvester/discovery/spyse.py index 5ce17d25..27936253 100644 --- a/theHarvester/discovery/spyse.py +++ b/theHarvester/discovery/spyse.py @@ -1,6 +1,6 @@ from theHarvester.discovery.constants import * from theHarvester.lib.core import * -import requests +from pprint import pprint class SearchSpyse: @@ -10,23 +10,25 @@ class SearchSpyse: self.key = Core.spyse_key() if self.key is None: raise MissingKey(True) - self.results = '' - self.totalresults = '' + self.totalhosts = set() - def do_search(self): + async def do_search(self): try: - base_url = f'https://api.spyse.com/v1/subdomains?domain={self.word}&api_token={self.key}&page=2' + url = f'https://api.spyse.com/v1/subdomains-aggregate?domain={self.word}&api_token={self.key}' headers = {'User-Agent': Core.get_user_agent()} - request = requests.get(base_url, headers=headers) - self.results = request.json() - # self.totalresults += self.results + client = aiohttp.ClientSession(headers=headers, timeout=aiohttp.ClientTimeout(total=30)) + responses = await AsyncFetcher.fetch(client, url, json=True) + await client.close() + + dct = responses['cidr']['cidr16']['results']['data'] + pprint(dct, indent=4) + except Exception as e: print(f'An exception has occurred: {e}') - def get_hostnames(self): - return self.totalresults + # async def get_hostnames(self): + # return self.totalhosts - def process(self): - self.do_search() - print('\tSearching results.') + async def process(self): + await self.do_search()