Remove 1 test and tidy up a few things and work on sypse module also add exalead to sources and update README.md about how to install theHarvester

README.md

@@ -85,11 +85,12 @@ Add your keys to api-keys.yaml
 * shodan
 * spyse
 
-Dependencies:
+Install & Dependencies:
 -------------
 * Python 3.7+
 * python3 -m pip install pipenv
-* pipenv install
+* https://github.com/laramies/theHarvester/wiki/Installation
+
 
 Comments, bugs, or requests?
 ----------------------------

tests/test_theHarvester.py

@@ -1,218 +0,0 @@
-import os
-import sys
-from unittest.mock import patch, MagicMock
-import theHarvester.__main__ as harvester
-import pytest
-
-domain = 'metasploit.com'
-sys.argv = args = [os.path.curdir + 'theHarvester.py', '-d', domain, '-b', 'domain']
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.baidusearch.SearchBaidu')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_baidu(stash, search_engine):
-    args[-1] = 'baidu'
-    await harvester.start()
-    assert stash().store_all.call_count == 2
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.bingsearch.SearchBing')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_bing(stash, search_engine):
-    args[-1] = 'bing'
-    await harvester.start()
-    args[-1] = 'bingapi'
-    await harvester.start()
-    assert stash().store_all.call_count == 4
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.certspottersearch.SearchCertspoter')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_certspotter(stash, search_engine):
-    args[-1] = 'certspotter'
-    await harvester.start()
-    assert stash().store_all.call_count == 1
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.crtsh.SearchCrtsh')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_crtsh(stash, search_engine):
-    args[-1] = 'crtsh'
-    await harvester.start()
-    assert stash().store_all.call_count == 1
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.dnsdumpster.SearchDnsDumpster')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_dnsdumpster(stash, search_engine):
-    args[-1] = 'dnsdumpster'
-    await harvester.start()
-    assert stash().store_all.call_count == 1
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.dogpilesearch.SearchDogpile')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_dogpile(stash, search_engine):
-    args[-1] = 'dogpile'
-    await harvester.start()
-    assert stash().store_all.call_count == 2
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.duckduckgosearch.SearchDuckDuckGo')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_duckduckgo(stash, search_engine):
-    args[-1] = 'duckduckgo'
-    await harvester.start()
-    assert stash().store_all.call_count == 2
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.githubcode.SearchGithubCode')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_github(stash, search_engine):
-    args[-1] = 'github-code'
-    await harvester.start()
-    assert stash().store_all.call_count == 2
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.exaleadsearch.SearchExalead')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_exalead(stash, search_engine):
-    args[-1] = 'exalead'
-    await harvester.start()
-    assert stash().store_all.call_count == 2
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.googlesearch.SearchGoogle')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_google(stash, search_engine):
-    args[-1] = 'google'
-    await harvester.start()
-    assert stash().store_all.call_count == 2
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.huntersearch.SearchHunter')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_hunter(stash, search_engine):
-    args[-1] = 'hunter'
-    await harvester.start()
-    assert stash().store_all.call_count == 2
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.intelxsearch.SearchIntelx')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_intelx(stash, search_engine):
-    args[-1] = 'intelx'
-    await harvester.start()
-    assert stash().store_all.call_count == 2
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.linkedinsearch.SearchLinkedin')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_linkedin(stash, search_engine):
-    args[-1] = 'linkedin'
-    await harvester.start()
-    assert stash().store_all.call_count == 1
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.linkedinsearch.SearchLinkedin')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_linkedin_links(stash, search_engine):
-    args[-1] = 'linkedin_links'
-    await harvester.start()
-    assert stash().store_all.call_count == 1
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.netcraft.SearchNetcraft')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_netcraft(stash, search_engine):
-    args[-1] = 'netcraft'
-    await harvester.start()
-    assert stash().store_all.call_count == 1
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.otxsearch.SearchOtx')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_otx(stash, search_engine):
-    args[-1] = 'otx'
-    await harvester.start()
-    assert stash().store_all.call_count == 2
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.securitytrailssearch.SearchSecuritytrail')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_security_trails(stash, search_engine):
-    args[-1] = 'securityTrails'
-    await harvester.start()
-    assert stash().store_all.call_count == 2
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.suip.SearchSuip')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_suip(stash, search_engine):
-    args[-1] = 'suip'
-    await harvester.start()
-    assert stash().store_all.call_count == 1
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.threatcrowd.SearchThreatcrowd')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_threatcrowd(stash, search_engine):
-    args[-1] = 'threatcrowd'
-    await harvester.start()
-    assert stash().store_all.call_count == 1
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.trello.SearchTrello')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_trello(stash, search_engine):
-    search_engine().get_results = MagicMock(return_value=('user@trello.com', 'trello', 'trello.com'))
-    args[-1] = 'trello'
-    await harvester.start()
-    assert stash().store_all.call_count == 3
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.twittersearch.SearchTwitter')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_twitter(stash, search_engine):
-    args[-1] = 'twitter'
-    await harvester.start()
-    assert stash().store_all.call_count == 1
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.virustotal.SearchVirustotal')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_virustotal(stash, search_engine):
-    args[-1] = 'virustotal'
-    await harvester.start()
-    assert stash().store_all.call_count == 1
-
-
-@pytest.mark.asyncio
-@patch('theHarvester.discovery.yahoosearch.SearchYahoo')
-@patch('theHarvester.lib.stash.StashManager')
-async def test_yahoo(stash, search_engine):
-    args[-1] = 'yahoo'
-    await harvester.start()
-    assert stash().store_all.call_count == 2

theHarvester.py

@@ -13,4 +13,4 @@ if python_version()[0:3] < '3.7':
 from theHarvester import __main__
 
 uvloop.install()
-asyncio.run(__main__.entry_point(), debug=True)
+asyncio.run(__main__.entry_point())

theHarvester/__main__.py

@@ -35,9 +35,9 @@ async def start():
     parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true')
     parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str)
     parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, certspotter, crtsh, dnsdumpster,
-                        dogpile, duckduckgo, github-code, google,
+                        dogpile, duckduckgo, exalead, github-code, google,
                         hunter, intelx,
-                        linkedin, linkedin_links, netcraft, otx, securityTrails, spyse(disabled for now), threatcrowd,
+                        linkedin, linkedin_links, netcraft, otx, securityTrails, spyse, threatcrowd,
                         trello, twitter, vhost, virustotal, yahoo, all''')
 
     args = parser.parse_args()
@@ -306,14 +306,15 @@ async def start():
                         print(e)
 
                 elif engineitem == 'spyse':
-                 from theHarvester.discovery import spyse
-                 try:
-                     pass
-                     #spysesearch = spyse.SearchSpyse(word)
-                     #spysesearch.process()
-
-                 except Exception as e:
-                     print(e)
+                    from theHarvester.discovery import spyse
+                    try:
+                        spysesearch = spyse.SearchSpyse(word)
+                        stor_lst.append(store(spysesearch, engineitem, store_host=True))
+                    except Exception as e:
+                        if isinstance(e, MissingKey):
+                            print(e)
+                        else:
+                            pass
 
                 elif engineitem == 'threatcrowd':
                     from theHarvester.discovery import threatcrowd
@@ -677,4 +678,4 @@ async def entry_point():
 
 
 if __name__ == '__main__':
-    asyncio.run(main=entry_point(), debug=True)
+    asyncio.run(main=entry_point())

theHarvester/discovery/otxsearch.py

@@ -1,6 +1,6 @@
 from theHarvester.lib.core import *
 import json
-import grequests
+#import grequests
 import re
 
 

theHarvester/discovery/spyse.py

@@ -1,6 +1,6 @@
 from theHarvester.discovery.constants import *
 from theHarvester.lib.core import *
-import requests
+from pprint import pprint
 
 
 class SearchSpyse:
@@ -10,23 +10,25 @@ class SearchSpyse:
         self.key = Core.spyse_key()
         if self.key is None:
             raise MissingKey(True)
-        self.results = ''
-        self.totalresults = ''
+        self.totalhosts = set()
 
-    def do_search(self):
+    async def do_search(self):
         try:
-            base_url = f'https://api.spyse.com/v1/subdomains?domain={self.word}&api_token={self.key}&page=2'
+            url = f'https://api.spyse.com/v1/subdomains-aggregate?domain={self.word}&api_token={self.key}'
             headers = {'User-Agent': Core.get_user_agent()}
-            request = requests.get(base_url, headers=headers)
-            self.results = request.json()
-            # self.totalresults += self.results
+            client = aiohttp.ClientSession(headers=headers, timeout=aiohttp.ClientTimeout(total=30))
+            responses = await AsyncFetcher.fetch(client, url, json=True)
+            await client.close()
+
+            dct = responses['cidr']['cidr16']['results']['data']
+            pprint(dct, indent=4)
+
 
         except Exception as e:
             print(f'An exception has occurred: {e}')
 
-    def get_hostnames(self):
-        return self.totalresults
+    # async def get_hostnames(self):
+    #     return self.totalhosts
 
-    def process(self):
-        self.do_search()
-        print('\tSearching results.')
+    async def process(self):
+        await self.do_search()