From d9b4ed3b2893fb4c4ecc025fbf38d7fcfbce18ce Mon Sep 17 00:00:00 2001
From: L1ghtn1ng <townsend891@hotmail.com>
Date: Sun, 17 May 2020 23:29:22 +0100
Subject: [PATCH] Add new sublist3r module and remove vhost from sources

---
 README.md                           |  2 ++
 tests/discovery/test_sublist3r.py   | 29 +++++++++++++++++++++++++++++
 theHarvester/__main__.py            | 12 ++++++++++--
 theHarvester/discovery/__init__.py  |  1 +
 theHarvester/discovery/sublist3r.py | 22 ++++++++++++++++++++++
 theHarvester/lib/core.py            |  1 +
 6 files changed, 65 insertions(+), 2 deletions(-)
 create mode 100644 tests/discovery/test_sublist3r.py
 create mode 100644 theHarvester/discovery/sublist3r.py

diff --git a/README.md b/README.md
index fe206869..a59760eb 100644
--- a/README.md
+++ b/README.md
@@ -61,6 +61,8 @@ Passive:
 
 * spyse: Web research tools for professionals (Requires an API key.) - https://spyse.com
 
+* sublist3r:  Fast subdomains enumeration tool for penetration testers - https://api.sublist3r.com/search.php?domain=example.com
+
 * Suip: Web research tools that can take over 10 minutes to run, but worth the wait - https://suip.biz
 
 * threatcrowd: Open source threat intelligence - www.threatcrowd.org
diff --git a/tests/discovery/test_sublist3r.py b/tests/discovery/test_sublist3r.py
new file mode 100644
index 00000000..afd6b234
--- /dev/null
+++ b/tests/discovery/test_sublist3r.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# coding=utf-8
+import requests
+from theHarvester.lib.core import *
+from theHarvester.discovery import sublist3r
+import pytest
+
+pytestmark = pytest.mark.asyncio
+
+
+class TestSublist3r(object):
+    @staticmethod
+    def domain() -> str:
+        return 'target.com'
+
+    async def test_api(self):
+        base_url = f'https://api.sublist3r.com/search.php?domain={TestSublist3r.domain()}'
+        headers = {'User-Agent': Core.get_user_agent()}
+        request = requests.get(base_url, headers=headers)
+        assert request.status_code == 200
+
+    async def test_search(self):
+        search = sublist3r.SearchSublist3r(TestSublist3r.domain())
+        await search.process()
+        assert isinstance(await search.get_hostnames(), list)
+
+
+if __name__ == '__main__':
+    pytest.main()
diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py
index 594666db..d862546b 100644
--- a/theHarvester/__main__.py
+++ b/theHarvester/__main__.py
@@ -37,8 +37,8 @@ async def start():
     parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, bufferoverun, certspotter, crtsh, dnsdumpster,
                         dogpile, duckduckgo, exalead, github-code, google,
                         hackertarget, hunter, intelx, linkedin, linkedin_links, netcraft, otx, pentesttools,
-                        rapiddns, securityTrails, spyse, suip, threatcrowd,
-                        trello, twitter, vhost, virustotal, yahoo, all''')
+                        rapiddns, securityTrails, spyse, sublist3r, suip, threatcrowd,
+                        trello, twitter, virustotal, yahoo, all''')
 
     args = parser.parse_args()
     try:
@@ -338,6 +338,14 @@ async def start():
                     except Exception as e:
                         print(e)
 
+                elif engineitem == 'sublist3r':
+                    from theHarvester.discovery import sublist3r
+                    try:
+                        sublist3r_search = sublist3r.SearchSublist3r(word)
+                        stor_lst.append(store(sublist3r_search, engineitem, store_host=True))
+                    except Exception as e:
+                        print(e)
+
                 elif engineitem == 'spyse':
                     from theHarvester.discovery import spyse
                     try:
diff --git a/theHarvester/discovery/__init__.py b/theHarvester/discovery/__init__.py
index ab75d7b7..6d8d7e3d 100644
--- a/theHarvester/discovery/__init__.py
+++ b/theHarvester/discovery/__init__.py
@@ -20,6 +20,7 @@ __all__ = ['baidusearch',
            'securitytrailssearch',
            'shodansearch',
            'spyse',
+           'sublist3r',
            'takeover',
            'threatcrowd',
            'trello',
diff --git a/theHarvester/discovery/sublist3r.py b/theHarvester/discovery/sublist3r.py
new file mode 100644
index 00000000..ee0611ae
--- /dev/null
+++ b/theHarvester/discovery/sublist3r.py
@@ -0,0 +1,22 @@
+from typing import Type
+from theHarvester.lib.core import *
+
+
+class SearchSublist3r:
+
+    def __init__(self, word):
+        self.word = word
+        self.totalhosts = list
+        self.proxy = False
+
+    async def do_search(self):
+        url = f'https://api.sublist3r.com/search.php?domain={self.word}'
+        response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy)
+        self.totalhosts: list = response[0]
+
+    async def get_hostnames(self) -> Type[list]:
+        return self.totalhosts
+
+    async def process(self, proxy=False):
+        self.proxy = proxy
+        await self.do_search()
diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py
index f00b0fcf..7718dc58 100644
--- a/theHarvester/lib/core.py
+++ b/theHarvester/lib/core.py
@@ -156,6 +156,7 @@ class Core:
                             'rapiddns',
                             'securityTrails',
                             'suip',
+                            'sublist3r',
                             'spyse',
                             'threatcrowd',
                             'trello',