Merge pull request #438 from L1ghtn1ng/dev

New Sublist3r module
2026-09-14 05:37:41 +02:00 · 2020-05-18 00:32:46 +01:00
parent 5505d4b0bd 3a44dca9d1
commit 503ff092ab
10 changed files with 100 additions and 34 deletions
@@ -27,47 +27,47 @@ jobs:
        pip install --upgrade pip
        pip install -r requirements/dev.txt

-    - name: Run theHarvester module baidu
+    - name: Run theHarvester module Baidu
      run: |
        python theHarvester.py -d yale.edu -b baidu

-    - name: Run theHarvester module bufferoverun
+    - name: Run theHarvester module Bufferoverun
      run: |
        python theHarvester.py -d yale.edu -b bufferoverun

-    - name: Run theHarvester module bing
+    - name: Run theHarvester module Bing
      run: |
        python theHarvester.py -d yale.edu -b bing

-    - name: Run theHarvester module certspotter
+    - name: Run theHarvester module CertSpotter
      run: |
        python theHarvester.py -d yale.edu -b certspotter

-    - name: Run theHarvester module crtsh
+    - name: Run theHarvester module Crtsh
      run: |
        python theHarvester.py -d hcl.com -b crtsh

-    - name: Run theHarvester module dnsdumpster
+    - name: Run theHarvester module DnsDumpster
      run: |
        python theHarvester.py -d yale.edu -b dnsdumpster

-    - name: Run theHarvester module dogplie
+    - name: Run theHarvester module Dogplie
      run: |
        python theHarvester.py -d yale.edu -b dogpile

-    - name: Run theHarvester module duckduckgo
+    - name: Run theHarvester module DuckDuckGo
      run: |
        python theHarvester.py -d yale.edu -b duckduckgo

-    - name: Run theHarvester module exalead
+    - name: Run theHarvester module Exalead
      run: |
        python theHarvester.py -d yale.edu -b exalead

-    - name: Run theHarvester module google
+    - name: Run theHarvester module Google
      run: |
        python theHarvester.py -d yale.edu -b google

-    - name: Run theHarvester module hackertarget
+    - name: Run theHarvester module HackerTarget
      run: |
        python theHarvester.py -d yale.edu -b hackertarget

@@ -75,7 +75,7 @@ jobs:
      run: |
        python theHarvester.py -d yale.edu -b intelx

-    - name: Run theHarvester module linkedin
+    - name: Run theHarvester module LinkedIn
      run: |
        python theHarvester.py -d yale.edu -b linkedin

@@ -83,7 +83,7 @@ jobs:
      run: |
        python theHarvester.py -d yale.edu -b linkedin_links

-    - name: Run theHarvester module netcraft
+    - name: Run theHarvester module Netcraft
      run: |
        python theHarvester.py -d yale.edu -b netcraft

@@ -95,23 +95,27 @@ jobs:
      run: |
        python theHarvester.py -d yale.edu -b rapiddns

-    - name: Run theHarvester module threatcrowd
+    - name: Run theHarvester module Sublist3r
+      run: |
+        python theHarvester.py -d yale.edu -b sublist3r
+
+    - name: Run theHarvester module Threatcrowd
      run: |
        python theHarvester.py -d yale.edu -b threatcrowd

-    - name: Run theHarvester module trello
+    - name: Run theHarvester module Trello
      run: |
        python theHarvester.py -d yale.edu -b trello

-    - name: Run theHarvester module twitter
+    - name: Run theHarvester module Twitter
      run: |
        python theHarvester.py -d yale.edu -b twitter

-    - name: Run theHarvester module virustotal
+    - name: Run theHarvester module Virustotal
      run: |
        python theHarvester.py -d yale.edu -b virustotal

-    - name: Run theHarvester module yahoo
+    - name: Run theHarvester module Yahoo
      run: |
        python theHarvester.py -d yale.edu -b yahoo

@@ -7,7 +7,7 @@ name = "pypi"
 aiodns = "==2.0.0"
 aiohttp = "==3.6.2"
 aiosqlite = "==0.13.0"
-beautifulsoup4 = "==4.9.0"
+beautifulsoup4 = "==4.9.1"
 dnspython = "==1.16.0"
 netaddr = "==0.7.19"
 plotly = "==4.7.1"
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "f4097511706eba2c516cf941dfcae2a08d10ea30d227b96bb359b3a57fe28373"
+            "sha256": "4e3e0a2cf9165de2411f72187b936c85e11bf2e730636665fa6486571c528ab5"
        },
        "pipfile-spec": 6,
        "requires": {},
@@ -64,12 +64,12 @@
        },
        "beautifulsoup4": {
            "hashes": [
-                "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8",
-                "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368",
-                "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0"
+                "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7",
+                "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8",
+                "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c"
            ],
            "index": "pypi",
-            "version": "==4.9.0"
+            "version": "==4.9.1"
        },
        "certifi": {
            "hashes": [
@@ -208,7 +208,6 @@
                "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255",
                "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d"
            ],
-            "index": "pypi",
            "version": "==4.7.6"
        },
        "netaddr": {
@@ -316,10 +315,10 @@
        },
        "soupsieve": {
            "hashes": [
-                "sha256:e914534802d7ffd233242b785229d5ba0766a7f487385e3f714446a07bf540ae",
-                "sha256:fcd71e08c0aee99aca1b73f45478549ee7e7fc006d51b37bec9e9def7dc22b69"
+                "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
+                "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
            ],
-            "version": "==2.0"
+            "version": "==2.0.1"
        },
        "texttable": {
            "hashes": [
@@ -406,10 +405,10 @@
        },
        "more-itertools": {
            "hashes": [
-                "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c",
-                "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507"
+                "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be",
+                "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982"
            ],
-            "version": "==8.2.0"
+            "version": "==8.3.0"
        },
        "mypy": {
            "hashes": [
@@ -61,6 +61,8 @@ Passive:

 * spyse: Web research tools for professionals (Requires an API key.) - https://spyse.com

+* sublist3r:  Fast subdomains enumeration tool for penetration testers - https://api.sublist3r.com/search.php?domain=example.com
+
 * Suip: Web research tools that can take over 10 minutes to run, but worth the wait - https://suip.biz

 * threatcrowd: Open source threat intelligence - www.threatcrowd.org
@@ -1,7 +1,7 @@
 aiodns==2.0.0
 aiohttp==3.6.2
 aiosqlite==0.13.0
-beautifulsoup4==4.9.0
+beautifulsoup4==4.9.1
 dnspython==1.16.0
 netaddr==0.7.19
 plotly==4.7.1
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# coding=utf-8
+import requests
+from theHarvester.lib.core import *
+from theHarvester.discovery import sublist3r
+import pytest
+
+pytestmark = pytest.mark.asyncio
+
+
+class TestSublist3r(object):
+    @staticmethod
+    def domain() -> str:
+        return 'target.com'
+
+    async def test_api(self):
+        base_url = f'https://api.sublist3r.com/search.php?domain={TestSublist3r.domain()}'
+        headers = {'User-Agent': Core.get_user_agent()}
+        request = requests.get(base_url, headers=headers)
+        assert request.status_code == 200
+
+    async def test_search(self):
+        search = sublist3r.SearchSublist3r(TestSublist3r.domain())
+        await search.process()
+        assert isinstance(await search.get_hostnames(), list)
+
+
+if __name__ == '__main__':
+    pytest.main()
@@ -37,8 +37,8 @@ async def start():
    parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, bufferoverun, certspotter, crtsh, dnsdumpster,
                        dogpile, duckduckgo, exalead, github-code, google,
                        hackertarget, hunter, intelx, linkedin, linkedin_links, netcraft, otx, pentesttools,
-                        rapiddns, securityTrails, spyse, suip, threatcrowd,
-                        trello, twitter, vhost, virustotal, yahoo, all''')
+                        rapiddns, securityTrails, spyse, sublist3r, suip, threatcrowd,
+                        trello, twitter, virustotal, yahoo, all''')

    args = parser.parse_args()
    try:
@@ -338,6 +338,14 @@ async def start():
                    except Exception as e:
                        print(e)

+                elif engineitem == 'sublist3r':
+                    from theHarvester.discovery import sublist3r
+                    try:
+                        sublist3r_search = sublist3r.SearchSublist3r(word)
+                        stor_lst.append(store(sublist3r_search, engineitem, store_host=True))
+                    except Exception as e:
+                        print(e)
+
                elif engineitem == 'spyse':
                    from theHarvester.discovery import spyse
                    try:
@@ -20,6 +20,7 @@ __all__ = ['baidusearch',
           'securitytrailssearch',
           'shodansearch',
           'spyse',
+           'sublist3r',
           'takeover',
           'threatcrowd',
           'trello',
@@ -0,0 +1,22 @@
+from typing import Type
+from theHarvester.lib.core import *
+
+
+class SearchSublist3r:
+
+    def __init__(self, word):
+        self.word = word
+        self.totalhosts = list
+        self.proxy = False
+
+    async def do_search(self):
+        url = f'https://api.sublist3r.com/search.php?domain={self.word}'
+        response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy)
+        self.totalhosts: list = response[0]
+
+    async def get_hostnames(self) -> Type[list]:
+        return self.totalhosts
+
+    async def process(self, proxy=False):
+        self.proxy = proxy
+        await self.do_search()
@@ -156,6 +156,7 @@ class Core:
                            'rapiddns',
                            'securityTrails',
                            'suip',
+                            'sublist3r',
                            'spyse',
                            'threatcrowd',
                            'trello',