From d9b4ed3b2893fb4c4ecc025fbf38d7fcfbce18ce Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sun, 17 May 2020 23:29:22 +0100 Subject: [PATCH 1/5] Add new sublist3r module and remove vhost from sources --- README.md | 2 ++ tests/discovery/test_sublist3r.py | 29 +++++++++++++++++++++++++++++ theHarvester/__main__.py | 12 ++++++++++-- theHarvester/discovery/__init__.py | 1 + theHarvester/discovery/sublist3r.py | 22 ++++++++++++++++++++++ theHarvester/lib/core.py | 1 + 6 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tests/discovery/test_sublist3r.py create mode 100644 theHarvester/discovery/sublist3r.py diff --git a/README.md b/README.md index fe206869..a59760eb 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ Passive: * spyse: Web research tools for professionals (Requires an API key.) - https://spyse.com +* sublist3r: Fast subdomains enumeration tool for penetration testers - https://api.sublist3r.com/search.php?domain=example.com + * Suip: Web research tools that can take over 10 minutes to run, but worth the wait - https://suip.biz * threatcrowd: Open source threat intelligence - www.threatcrowd.org diff --git a/tests/discovery/test_sublist3r.py b/tests/discovery/test_sublist3r.py new file mode 100644 index 00000000..afd6b234 --- /dev/null +++ b/tests/discovery/test_sublist3r.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# coding=utf-8 +import requests +from theHarvester.lib.core import * +from theHarvester.discovery import sublist3r +import pytest + +pytestmark = pytest.mark.asyncio + + +class TestSublist3r(object): + @staticmethod + def domain() -> str: + return 'target.com' + + async def test_api(self): + base_url = f'https://api.sublist3r.com/search.php?domain={TestSublist3r.domain()}' + headers = {'User-Agent': Core.get_user_agent()} + request = requests.get(base_url, headers=headers) + assert request.status_code == 200 + + async def test_search(self): + search = sublist3r.SearchSublist3r(TestSublist3r.domain()) + await search.process() + assert isinstance(await search.get_hostnames(), list) + + +if __name__ == '__main__': + pytest.main() diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 594666db..d862546b 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -37,8 +37,8 @@ async def start(): parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, bufferoverun, certspotter, crtsh, dnsdumpster, dogpile, duckduckgo, exalead, github-code, google, hackertarget, hunter, intelx, linkedin, linkedin_links, netcraft, otx, pentesttools, - rapiddns, securityTrails, spyse, suip, threatcrowd, - trello, twitter, vhost, virustotal, yahoo, all''') + rapiddns, securityTrails, spyse, sublist3r, suip, threatcrowd, + trello, twitter, virustotal, yahoo, all''') args = parser.parse_args() try: @@ -338,6 +338,14 @@ async def start(): except Exception as e: print(e) + elif engineitem == 'sublist3r': + from theHarvester.discovery import sublist3r + try: + sublist3r_search = sublist3r.SearchSublist3r(word) + stor_lst.append(store(sublist3r_search, engineitem, store_host=True)) + except Exception as e: + print(e) + elif engineitem == 'spyse': from theHarvester.discovery import spyse try: diff --git a/theHarvester/discovery/__init__.py b/theHarvester/discovery/__init__.py index ab75d7b7..6d8d7e3d 100644 --- a/theHarvester/discovery/__init__.py +++ b/theHarvester/discovery/__init__.py @@ -20,6 +20,7 @@ __all__ = ['baidusearch', 'securitytrailssearch', 'shodansearch', 'spyse', + 'sublist3r', 'takeover', 'threatcrowd', 'trello', diff --git a/theHarvester/discovery/sublist3r.py b/theHarvester/discovery/sublist3r.py new file mode 100644 index 00000000..ee0611ae --- /dev/null +++ b/theHarvester/discovery/sublist3r.py @@ -0,0 +1,22 @@ +from typing import Type +from theHarvester.lib.core import * + + +class SearchSublist3r: + + def __init__(self, word): + self.word = word + self.totalhosts = list + self.proxy = False + + async def do_search(self): + url = f'https://api.sublist3r.com/search.php?domain={self.word}' + response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy) + self.totalhosts: list = response[0] + + async def get_hostnames(self) -> Type[list]: + return self.totalhosts + + async def process(self, proxy=False): + self.proxy = proxy + await self.do_search() diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index f00b0fcf..7718dc58 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -156,6 +156,7 @@ class Core: 'rapiddns', 'securityTrails', 'suip', + 'sublist3r', 'spyse', 'threatcrowd', 'trello', From 99e0c31f0be720d378d8e91089b879b834b440cf Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sun, 17 May 2020 23:33:50 +0100 Subject: [PATCH 2/5] Add sublist3r to ci --- .github/workflows/theHarvester.yml | 40 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index d972d334..e921b28e 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -27,47 +27,47 @@ jobs: pip install --upgrade pip pip install -r requirements/dev.txt - - name: Run theHarvester module baidu + - name: Run theHarvester module Baidu run: | python theHarvester.py -d yale.edu -b baidu - - name: Run theHarvester module bufferoverun + - name: Run theHarvester module Bufferoverun run: | python theHarvester.py -d yale.edu -b bufferoverun - - name: Run theHarvester module bing + - name: Run theHarvester module Bing run: | python theHarvester.py -d yale.edu -b bing - - name: Run theHarvester module certspotter + - name: Run theHarvester module CertSpotter run: | python theHarvester.py -d yale.edu -b certspotter - - name: Run theHarvester module crtsh + - name: Run theHarvester module Crtsh run: | python theHarvester.py -d hcl.com -b crtsh - - name: Run theHarvester module dnsdumpster + - name: Run theHarvester module DnsDumpster run: | python theHarvester.py -d yale.edu -b dnsdumpster - - name: Run theHarvester module dogplie + - name: Run theHarvester module Dogplie run: | python theHarvester.py -d yale.edu -b dogpile - - name: Run theHarvester module duckduckgo + - name: Run theHarvester module DuckDuckGo run: | python theHarvester.py -d yale.edu -b duckduckgo - - name: Run theHarvester module exalead + - name: Run theHarvester module Exalead run: | python theHarvester.py -d yale.edu -b exalead - - name: Run theHarvester module google + - name: Run theHarvester module Google run: | python theHarvester.py -d yale.edu -b google - - name: Run theHarvester module hackertarget + - name: Run theHarvester module HackerTarget run: | python theHarvester.py -d yale.edu -b hackertarget @@ -75,7 +75,7 @@ jobs: run: | python theHarvester.py -d yale.edu -b intelx - - name: Run theHarvester module linkedin + - name: Run theHarvester module LinkedIn run: | python theHarvester.py -d yale.edu -b linkedin @@ -83,7 +83,7 @@ jobs: run: | python theHarvester.py -d yale.edu -b linkedin_links - - name: Run theHarvester module netcraft + - name: Run theHarvester module Netcraft run: | python theHarvester.py -d yale.edu -b netcraft @@ -95,23 +95,27 @@ jobs: run: | python theHarvester.py -d yale.edu -b rapiddns - - name: Run theHarvester module threatcrowd + - name: Run theHarvester module Sublist3r + run: | + python theHarvester.py -d yale.edu -b sublist3r + + - name: Run theHarvester module Threatcrowd run: | python theHarvester.py -d yale.edu -b threatcrowd - - name: Run theHarvester module trello + - name: Run theHarvester module Trello run: | python theHarvester.py -d yale.edu -b trello - - name: Run theHarvester module twitter + - name: Run theHarvester module Twitter run: | python theHarvester.py -d yale.edu -b twitter - - name: Run theHarvester module virustotal + - name: Run theHarvester module Virustotal run: | python theHarvester.py -d yale.edu -b virustotal - - name: Run theHarvester module yahoo + - name: Run theHarvester module Yahoo run: | python theHarvester.py -d yale.edu -b yahoo From 4011783838a2da87af6a93eb0a7933480d98a715 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sun, 17 May 2020 22:36:51 +0000 Subject: [PATCH 3/5] Bump beautifulsoup4 from 4.9.0 to 4.9.1 in /requirements Bumps [beautifulsoup4](http://www.crummy.com/software/BeautifulSoup/bs4/) from 4.9.0 to 4.9.1. Signed-off-by: dependabot-preview[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index d45227fb..3c28a3ca 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,7 +1,7 @@ aiodns==2.0.0 aiohttp==3.6.2 aiosqlite==0.13.0 -beautifulsoup4==4.9.0 +beautifulsoup4==4.9.1 dnspython==1.16.0 netaddr==0.7.19 plotly==4.7.1 From 2763cc43c47fa580c924de398ef2c6b41bbaf788 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sun, 17 May 2020 22:38:18 +0000 Subject: [PATCH 4/5] Bump soupsieve from 2.0 to 2.0.1 Bumps [soupsieve](https://github.com/facelessuser/soupsieve) from 2.0 to 2.0.1. - [Release notes](https://github.com/facelessuser/soupsieve/releases) - [Commits](https://github.com/facelessuser/soupsieve/compare/2.0.0...2.0.1) Signed-off-by: dependabot-preview[bot] --- Pipfile.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index ada19df0..abd63ee0 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -208,7 +208,6 @@ "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255", "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d" ], - "index": "pypi", "version": "==4.7.6" }, "netaddr": { @@ -316,10 +315,11 @@ }, "soupsieve": { "hashes": [ - "sha256:e914534802d7ffd233242b785229d5ba0766a7f487385e3f714446a07bf540ae", - "sha256:fcd71e08c0aee99aca1b73f45478549ee7e7fc006d51b37bec9e9def7dc22b69" + "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", + "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" ], - "version": "==2.0" + "index": "pypi", + "version": "==2.0.1" }, "texttable": { "hashes": [ @@ -406,10 +406,10 @@ }, "more-itertools": { "hashes": [ - "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", - "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" + "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be", + "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982" ], - "version": "==8.2.0" + "version": "==8.3.0" }, "mypy": { "hashes": [ From 43acac8708b567906e6351636301486536064895 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sun, 17 May 2020 22:55:15 +0000 Subject: [PATCH 5/5] Bump beautifulsoup4 from 4.9.0 to 4.9.1 Bumps [beautifulsoup4](http://www.crummy.com/software/BeautifulSoup/bs4/) from 4.9.0 to 4.9.1. Signed-off-by: dependabot-preview[bot] --- Pipfile | 2 +- Pipfile.lock | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Pipfile b/Pipfile index 39e73edb..de08141a 100644 --- a/Pipfile +++ b/Pipfile @@ -7,7 +7,7 @@ name = "pypi" aiodns = "==2.0.0" aiohttp = "==3.6.2" aiosqlite = "==0.13.0" -beautifulsoup4 = "==4.9.0" +beautifulsoup4 = "==4.9.1" dnspython = "==1.16.0" netaddr = "==0.7.19" plotly = "==4.7.1" diff --git a/Pipfile.lock b/Pipfile.lock index abd63ee0..c6be56c5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f4097511706eba2c516cf941dfcae2a08d10ea30d227b96bb359b3a57fe28373" + "sha256": "4e3e0a2cf9165de2411f72187b936c85e11bf2e730636665fa6486571c528ab5" }, "pipfile-spec": 6, "requires": {}, @@ -64,12 +64,12 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8", - "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368", - "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0" + "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", + "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8", + "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c" ], "index": "pypi", - "version": "==4.9.0" + "version": "==4.9.1" }, "certifi": { "hashes": [ @@ -318,7 +318,6 @@ "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" ], - "index": "pypi", "version": "==2.0.1" }, "texttable": {