Add new sublist3r module and remove vhost from sources

README.md

@@ -61,6 +61,8 @@ Passive:
 
 * spyse: Web research tools for professionals (Requires an API key.) - https://spyse.com
 
+* sublist3r:  Fast subdomains enumeration tool for penetration testers - https://api.sublist3r.com/search.php?domain=example.com
+
 * Suip: Web research tools that can take over 10 minutes to run, but worth the wait - https://suip.biz
 
 * threatcrowd: Open source threat intelligence - www.threatcrowd.org

tests/discovery/test_sublist3r.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# coding=utf-8
+import requests
+from theHarvester.lib.core import *
+from theHarvester.discovery import sublist3r
+import pytest
+
+pytestmark = pytest.mark.asyncio
+
+
+class TestSublist3r(object):
+    @staticmethod
+    def domain() -> str:
+        return 'target.com'
+
+    async def test_api(self):
+        base_url = f'https://api.sublist3r.com/search.php?domain={TestSublist3r.domain()}'
+        headers = {'User-Agent': Core.get_user_agent()}
+        request = requests.get(base_url, headers=headers)
+        assert request.status_code == 200
+
+    async def test_search(self):
+        search = sublist3r.SearchSublist3r(TestSublist3r.domain())
+        await search.process()
+        assert isinstance(await search.get_hostnames(), list)
+
+
+if __name__ == '__main__':
+    pytest.main()

theHarvester/__main__.py

@@ -37,8 +37,8 @@ async def start():
     parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, bufferoverun, certspotter, crtsh, dnsdumpster,
                         dogpile, duckduckgo, exalead, github-code, google,
                         hackertarget, hunter, intelx, linkedin, linkedin_links, netcraft, otx, pentesttools,
-                        rapiddns, securityTrails, spyse, suip, threatcrowd,
-                        trello, twitter, vhost, virustotal, yahoo, all''')
+                        rapiddns, securityTrails, spyse, sublist3r, suip, threatcrowd,
+                        trello, twitter, virustotal, yahoo, all''')
 
     args = parser.parse_args()
     try:
@@ -338,6 +338,14 @@ async def start():
                     except Exception as e:
                         print(e)
 
+                elif engineitem == 'sublist3r':
+                    from theHarvester.discovery import sublist3r
+                    try:
+                        sublist3r_search = sublist3r.SearchSublist3r(word)
+                        stor_lst.append(store(sublist3r_search, engineitem, store_host=True))
+                    except Exception as e:
+                        print(e)
+
                 elif engineitem == 'spyse':
                     from theHarvester.discovery import spyse
                     try:

theHarvester/discovery/__init__.py

@@ -20,6 +20,7 @@ __all__ = ['baidusearch',
            'securitytrailssearch',
            'shodansearch',
            'spyse',
+           'sublist3r',
            'takeover',
            'threatcrowd',
            'trello',

theHarvester/discovery/sublist3r.py

@@ -0,0 +1,22 @@
+from typing import Type
+from theHarvester.lib.core import *
+
+
+class SearchSublist3r:
+
+    def __init__(self, word):
+        self.word = word
+        self.totalhosts = list
+        self.proxy = False
+
+    async def do_search(self):
+        url = f'https://api.sublist3r.com/search.php?domain={self.word}'
+        response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy)
+        self.totalhosts: list = response[0]
+
+    async def get_hostnames(self) -> Type[list]:
+        return self.totalhosts
+
+    async def process(self, proxy=False):
+        self.proxy = proxy
+        await self.do_search()

theHarvester/lib/core.py

@@ -156,6 +156,7 @@ class Core:
                             'rapiddns',
                             'securityTrails',
                             'suip',
+                            'sublist3r',
                             'spyse',
                             'threatcrowd',
                             'trello',