diff --git a/bin/restfulHarvest b/bin/restfulHarvest
index 7cc31812..ae19e97f 100755
--- a/bin/restfulHarvest
+++ b/bin/restfulHarvest
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import argparse
+
import uvicorn
parser = argparse.ArgumentParser()
diff --git a/bin/theHarvester b/bin/theHarvester
index 2e22c5c0..7004504e 100755
--- a/bin/theHarvester
+++ b/bin/theHarvester
@@ -1,7 +1,8 @@
#!/usr/bin/env python3
# Note: This script runs theHarvester
-import sys
import asyncio
+import sys
+
from theHarvester import __main__
if sys.version_info.major < 3 or sys.version_info.minor < 9:
diff --git a/restfulHarvest.py b/restfulHarvest.py
index d7b02cf7..bde18442 100755
--- a/restfulHarvest.py
+++ b/restfulHarvest.py
@@ -1,14 +1,43 @@
#!/usr/bin/env python3
import argparse
+
import uvicorn
parser = argparse.ArgumentParser()
-parser.add_argument('-H', '--host', default='127.0.0.1', help='IP address to listen on default is 127.0.0.1')
-parser.add_argument('-p', '--port', default=5000, help='Port to bind the web server to, default is 5000', type=int)
-parser.add_argument('-l', '--log-level', default='info', help='Set logging level, default is info but [critical|error|warning|info|debug|trace] can be set')
-parser.add_argument('-r', '--reload', default=False, help='Enable automatic reload used during development of the api', action='store_true')
+parser.add_argument(
+ "-H",
+ "--host",
+ default="127.0.0.1",
+ help="IP address to listen on default is 127.0.0.1",
+)
+parser.add_argument(
+ "-p",
+ "--port",
+ default=5000,
+ help="Port to bind the web server to, default is 5000",
+ type=int,
+)
+parser.add_argument(
+ "-l",
+ "--log-level",
+ default="info",
+ help="Set logging level, default is info but [critical|error|warning|info|debug|trace] can be set",
+)
+parser.add_argument(
+ "-r",
+ "--reload",
+ default=False,
+ help="Enable automatic reload used during development of the api",
+ action="store_true",
+)
args: argparse.Namespace = parser.parse_args()
-if __name__ == '__main__':
- uvicorn.run('theHarvester.lib.api.api:app', host=args.host, port=args.port, log_level=args.log_level, reload=args.reload)
+if __name__ == "__main__":
+ uvicorn.run(
+ "theHarvester.lib.api.api:app",
+ host=args.host,
+ port=args.port,
+ log_level=args.log_level,
+ reload=args.reload,
+ )
diff --git a/setup.py b/setup.py
index 393092e7..aa8904a8 100755
--- a/setup.py
+++ b/setup.py
@@ -1,11 +1,12 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
+
from theHarvester.lib.version import version
-with open('README.md', 'r') as fh:
+with open("README.md", "r") as fh:
long_description: str = fh.read()
setup(
- name='theHarvester',
+ name="theHarvester",
version=version(),
author="Christian Martorella",
author_email="cmartorella@edge-security.com",
@@ -13,11 +14,9 @@ setup(
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/laramies/theHarvester",
- packages=find_packages(exclude=['tests']),
- python_requires='>=3.9',
- scripts=['bin/theHarvester',
- 'bin/restfulHarvest'],
-
+ packages=find_packages(exclude=["tests"]),
+ python_requires=">=3.9",
+ scripts=["bin/theHarvester", "bin/restfulHarvest"],
classifiers=[
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.9",
@@ -27,15 +26,17 @@ setup(
"Operating System :: OS Independent",
],
data_files=[
- ('/etc/theHarvester', [
- 'wordlists/general/common.txt',
- 'wordlists/dns-big.txt',
- 'wordlists/dns-names.txt',
- 'wordlists/dorks.txt',
- 'wordlists/names_small.txt',
- 'api-keys.yaml',
- 'proxies.yaml'
- ]
+ (
+ "/etc/theHarvester",
+ [
+ "wordlists/general/common.txt",
+ "wordlists/dns-big.txt",
+ "wordlists/dns-names.txt",
+ "wordlists/dorks.txt",
+ "wordlists/names_small.txt",
+ "api-keys.yaml",
+ "proxies.yaml",
+ ],
)
],
)
diff --git a/tests/discovery/test_anubis.py b/tests/discovery/test_anubis.py
index 1b1d492b..9786320d 100644
--- a/tests/discovery/test_anubis.py
+++ b/tests/discovery/test_anubis.py
@@ -1,25 +1,29 @@
#!/usr/bin/env python3
# coding=utf-8
-import requests
-from theHarvester.lib.core import *
-from theHarvester.discovery import anubis
import os
-import pytest
-from _pytest.mark.structures import MarkDecorator
from typing import Optional
+import pytest
+import requests
+from _pytest.mark.structures import MarkDecorator
+
+from theHarvester.discovery import anubis
+from theHarvester.lib.core import *
+
pytestmark: MarkDecorator = pytest.mark.asyncio
-github_ci: Optional[str] = os.getenv('GITHUB_ACTIONS') # Github set this to be the following: true instead of True
+github_ci: Optional[str] = os.getenv(
+ "GITHUB_ACTIONS"
+) # Github set this to be the following: true instead of True
class TestAnubis:
@staticmethod
def domain() -> str:
- return 'apple.com'
+ return "apple.com"
async def test_api(self) -> None:
- base_url = f'https://jldc.me/anubis/subdomains/{TestAnubis.domain()}'
- headers = {'User-Agent': Core.get_user_agent()}
+ base_url = f"https://jldc.me/anubis/subdomains/{TestAnubis.domain()}"
+ headers = {"User-Agent": Core.get_user_agent()}
request = requests.get(base_url, headers=headers)
assert request.status_code == 200
diff --git a/tests/discovery/test_certspotter.py b/tests/discovery/test_certspotter.py
index 19152693..b8f98c1f 100644
--- a/tests/discovery/test_certspotter.py
+++ b/tests/discovery/test_certspotter.py
@@ -1,25 +1,29 @@
#!/usr/bin/env python3
# coding=utf-8
-from theHarvester.lib.core import *
-from theHarvester.discovery import certspottersearch
import os
-import requests
-import pytest
-from _pytest.mark.structures import MarkDecorator
from typing import Optional
+import pytest
+import requests
+from _pytest.mark.structures import MarkDecorator
+
+from theHarvester.discovery import certspottersearch
+from theHarvester.lib.core import *
+
pytestmark: MarkDecorator = pytest.mark.asyncio
-github_ci: Optional[str] = os.getenv('GITHUB_ACTIONS') # Github set this to be the following: true instead of True
+github_ci: Optional[str] = os.getenv(
+ "GITHUB_ACTIONS"
+) # Github set this to be the following: true instead of True
class TestCertspotter(object):
@staticmethod
def domain() -> str:
- return 'metasploit.com'
+ return "metasploit.com"
async def test_api(self) -> None:
- base_url = f'https://api.certspotter.com/v1/issuances?domain={TestCertspotter.domain()}&expand=dns_names'
- headers = {'User-Agent': Core.get_user_agent()}
+ base_url = f"https://api.certspotter.com/v1/issuances?domain={TestCertspotter.domain()}&expand=dns_names"
+ headers = {"User-Agent": Core.get_user_agent()}
request = requests.get(base_url, headers=headers)
assert request.status_code == 200
@@ -29,10 +33,10 @@ class TestCertspotter(object):
assert isinstance(await search.get_hostnames(), set)
async def test_search_no_results(self) -> None:
- search = certspottersearch.SearchCertspoter('radiant.eu')
+ search = certspottersearch.SearchCertspoter("radiant.eu")
await search.process()
assert len(await search.get_hostnames()) == 0
-if __name__ == '__main__':
+if __name__ == "__main__":
pytest.main()
diff --git a/tests/discovery/test_githubcode.py b/tests/discovery/test_githubcode.py
index 0eb9154d..ff6ac5f5 100644
--- a/tests/discovery/test_githubcode.py
+++ b/tests/discovery/test_githubcode.py
@@ -1,34 +1,23 @@
+from unittest.mock import MagicMock
+
+import pytest
+from _pytest.mark.structures import MarkDecorator
+from requests import Response
+
from theHarvester.discovery import githubcode
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import Core
-from unittest.mock import MagicMock
-from requests import Response
-import pytest
-from _pytest.mark.structures import MarkDecorator
pytestmark: MarkDecorator = pytest.mark.asyncio
class TestSearchGithubCode:
-
class OkResponse:
response = Response()
json = {
"items": [
- {
- "text_matches": [
- {
- "fragment": "test1"
- }
- ]
- },
- {
- "text_matches": [
- {
- "fragment": "test2"
- }
- ]
- }
+ {"text_matches": [{"fragment": "test1"}]},
+ {"text_matches": [{"fragment": "test2"}]},
]
}
response.status_code = 200
@@ -48,19 +37,9 @@ class TestSearchGithubCode:
response = Response()
json = {
"items": [
- {
- "fail": True
- },
- {
- "text_matches": []
- },
- {
- "text_matches": [
- {
- "weird": "result"
- }
- ]
- }
+ {"fail": True},
+ {"text_matches": []},
+ {"text_matches": [{"weird": "result"}]},
]
}
response.json = MagicMock(return_value=json)
@@ -74,27 +53,31 @@ class TestSearchGithubCode:
async def test_fragments_from_response(self) -> None:
Core.github_key = MagicMock(return_value="lol")
test_class_instance = githubcode.SearchGithubCode(word="test", limit=500)
- test_result = await test_class_instance.fragments_from_response(self.OkResponse.response.json())
- print('test_result: ', test_result)
+ test_result = await test_class_instance.fragments_from_response(
+ self.OkResponse.response.json()
+ )
+ print("test_result: ", test_result)
assert test_result == ["test1", "test2"]
async def test_invalid_fragments_from_response(self) -> None:
Core.github_key = MagicMock(return_value="lol")
test_class_instance = githubcode.SearchGithubCode(word="test", limit=500)
- test_result = await test_class_instance.fragments_from_response(self.MalformedResponse.response.json())
+ test_result = await test_class_instance.fragments_from_response(
+ self.MalformedResponse.response.json()
+ )
assert test_result == []
async def test_next_page(self) -> None:
Core.github_key = MagicMock(return_value="lol")
test_class_instance = githubcode.SearchGithubCode(word="test", limit=500)
test_result = githubcode.SuccessResult(list(), next_page=2, last_page=4)
- assert (2 == await test_class_instance.next_page_or_end(test_result))
+ assert 2 == await test_class_instance.next_page_or_end(test_result)
async def test_last_page(self) -> None:
Core.github_key = MagicMock(return_value="lol")
test_class_instance = githubcode.SearchGithubCode(word="test", limit=500)
test_result = githubcode.SuccessResult(list(), None, None)
- assert (None is await test_class_instance.next_page_or_end(test_result))
+ assert None is await test_class_instance.next_page_or_end(test_result)
- if __name__ == '__main__':
+ if __name__ == "__main__":
pytest.main()
diff --git a/tests/discovery/test_otx.py b/tests/discovery/test_otx.py
index 851afb9d..bb48111a 100644
--- a/tests/discovery/test_otx.py
+++ b/tests/discovery/test_otx.py
@@ -1,25 +1,29 @@
#!/usr/bin/env python3
# coding=utf-8
-from theHarvester.lib.core import *
-from theHarvester.discovery import otxsearch
import os
-import requests
-import pytest
-from _pytest.mark.structures import MarkDecorator
from typing import Optional
+import pytest
+import requests
+from _pytest.mark.structures import MarkDecorator
+
+from theHarvester.discovery import otxsearch
+from theHarvester.lib.core import *
+
pytestmark: MarkDecorator = pytest.mark.asyncio
-github_ci: Optional[str] = os.getenv('GITHUB_ACTIONS') # Github set this to be the following: true instead of True
+github_ci: Optional[str] = os.getenv(
+ "GITHUB_ACTIONS"
+) # Github set this to be the following: true instead of True
class TestOtx(object):
@staticmethod
def domain() -> str:
- return 'metasploit.com'
+ return "metasploit.com"
async def test_api(self) -> None:
- base_url = f'https://otx.alienvault.com/api/v1/indicators/domain/{TestOtx.domain()}/passive_dns'
- headers = {'User-Agent': Core.get_user_agent()}
+ base_url = f"https://otx.alienvault.com/api/v1/indicators/domain/{TestOtx.domain()}/passive_dns"
+ headers = {"User-Agent": Core.get_user_agent()}
request = requests.get(base_url, headers=headers)
assert request.status_code == 200
@@ -30,5 +34,5 @@ class TestOtx(object):
assert isinstance(await search.get_ips(), set)
-if __name__ == '__main__':
+if __name__ == "__main__":
pytest.main()
diff --git a/tests/discovery/test_threatminer.py b/tests/discovery/test_threatminer.py
index e3e13f61..58ba2d9b 100644
--- a/tests/discovery/test_threatminer.py
+++ b/tests/discovery/test_threatminer.py
@@ -1,23 +1,27 @@
#!/usr/bin/env python3
# coding=utf-8
-import requests
-from theHarvester.lib.core import *
-from theHarvester.discovery import threatminer
import os
+
import pytest
+import requests
+
+from theHarvester.discovery import threatminer
+from theHarvester.lib.core import *
pytestmark = pytest.mark.asyncio
-github_ci = os.getenv('GITHUB_ACTIONS') # Github set this to be the following: true instead of True
+github_ci = os.getenv(
+ "GITHUB_ACTIONS"
+) # Github set this to be the following: true instead of True
class TestThreatminer(object):
@staticmethod
def domain() -> str:
- return 'target.com'
+ return "target.com"
async def test_api(self):
- base_url = f'https://api.threatminer.org/v2/domain.php?q={TestThreatminer.domain()}&rt=5'
- headers = {'User-Agent': Core.get_user_agent()}
+ base_url = f"https://api.threatminer.org/v2/domain.php?q={TestThreatminer.domain()}&rt=5"
+ headers = {"User-Agent": Core.get_user_agent()}
request = requests.get(base_url, headers=headers)
assert request.status_code == 200
@@ -28,5 +32,5 @@ class TestThreatminer(object):
assert isinstance(await search.get_ips(), set)
-if __name__ == '__main__':
+if __name__ == "__main__":
pytest.main()
diff --git a/tests/test_myparser.py b/tests/test_myparser.py
index 6e624941..bcbb5e5d 100755
--- a/tests/test_myparser.py
+++ b/tests/test_myparser.py
@@ -1,20 +1,20 @@
#!/usr/bin/env python3
# coding=utf-8
-from theHarvester.parsers import myparser
import pytest
+from theHarvester.parsers import myparser
+
class TestMyParser(object):
-
@pytest.mark.asyncio
async def test_emails(self) -> None:
- word = 'domain.com'
- results = '@domain.com***a@domain***banotherdomain.com***c@domain.com***d@sub.domain.com***'
+ word = "domain.com"
+ results = "@domain.com***a@domain***banotherdomain.com***c@domain.com***d@sub.domain.com***"
parse = myparser.Parser(results, word)
emails = sorted(await parse.emails())
- assert emails, ['c@domain.com', 'd@sub.domain.com']
+ assert emails, ["c@domain.com", "d@sub.domain.com"]
-if __name__ == '__main__':
+if __name__ == "__main__":
pytest.main()
diff --git a/theHarvester.py b/theHarvester.py
index 2e22c5c0..7f1811e5 100755
--- a/theHarvester.py
+++ b/theHarvester.py
@@ -1,16 +1,17 @@
#!/usr/bin/env python3
# Note: This script runs theHarvester
-import sys
import asyncio
+import sys
+
from theHarvester import __main__
if sys.version_info.major < 3 or sys.version_info.minor < 9:
- print('\033[93m[!] Make sure you have Python 3.9+ installed, quitting.\n\n \033[0m')
+ print("\033[93m[!] Make sure you have Python 3.9+ installed, quitting.\n\n \033[0m")
sys.exit(1)
-if __name__ == '__main__':
+if __name__ == "__main__":
platform = sys.platform
- if platform == 'win32':
+ if platform == "win32":
# Required or things will break if trying to take screenshots
import multiprocessing
@@ -18,6 +19,7 @@ if __name__ == '__main__':
asyncio.DefaultEventLoopPolicy = asyncio.WindowsSelectorEventLoopPolicy
else:
import uvloop
+
uvloop.install()
if "linux" in platform:
diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py
index 8fdf088f..9cf8acaf 100644
--- a/theHarvester/__main__.py
+++ b/theHarvester/__main__.py
@@ -1,56 +1,121 @@
#!/usr/bin/env python3
-from typing import Optional, Dict, List
-from theHarvester.discovery import *
-from theHarvester.discovery import dnssearch, takeover, shodansearch
-from theHarvester.discovery.constants import *
-from theHarvester.lib import hostchecker
-from theHarvester.lib import stash
-from theHarvester.lib.core import *
import argparse
import asyncio
-import ujson
-import netaddr
import re
-import sys
-import string
import secrets
+import string
+import sys
+from typing import Dict, List, Optional
+
+import netaddr
+import ujson
+
+from theHarvester.discovery import *
+from theHarvester.discovery import dnssearch, shodansearch, takeover
+from theHarvester.discovery.constants import *
+from theHarvester.lib import hostchecker, stash
+from theHarvester.lib.core import *
async def start(rest_args: Optional[argparse.Namespace] = None):
"""Main program function"""
parser = argparse.ArgumentParser(
- description='theHarvester is used to gather open source intelligence (OSINT) on a company or domain.')
- parser.add_argument('-d', '--domain', help='Company name or domain to search.', required=True)
- parser.add_argument('-l', '--limit', help='Limit the number of search results, default=500.', default=500, type=int)
- parser.add_argument('-S', '--start', help='Start with result number X, default=0.', default=0, type=int)
- parser.add_argument('-p', '--proxies', help='Use proxies for requests, enter proxies in proxies.yaml.',
- default=False, action='store_true')
- parser.add_argument('-s', '--shodan', help='Use Shodan to query discovered hosts.', default=False,
- action='store_true')
- parser.add_argument('--screenshot',
- help='Take screenshots of resolved domains specify output directory: --screenshot output_directory',
- default="", type=str)
- parser.add_argument('-v', '--virtual-host',
- help='Verify host name via DNS resolution and search for virtual hosts.', action='store_const',
- const='basic', default=False)
- parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup.')
- parser.add_argument('-t', '--take-over', help='Check for takeovers.', default=False, action='store_true')
- parser.add_argument('-r', '--dns-resolve',
- help='Perform DNS resolution on subdomains with a resolver list or passed in resolvers, default False.',
- default="", type=str, nargs='?')
- parser.add_argument('-n', '--dns-lookup', help='Enable DNS server lookup, default False.', default=False,
- action='store_true')
- parser.add_argument('-c', '--dns-brute', help='Perform a DNS brute force on the domain.', default=False,
- action='store_true')
- parser.add_argument('-f', '--filename', help='Save the results to an XML and JSON file.', default='', type=str)
- parser.add_argument('-b', '--source', help='''anubis, baidu, bevigil, binaryedge, bing, bingapi, bufferoverun, brave,
+ description="theHarvester is used to gather open source intelligence (OSINT) on a company or domain."
+ )
+ parser.add_argument(
+ "-d", "--domain", help="Company name or domain to search.", required=True
+ )
+ parser.add_argument(
+ "-l",
+ "--limit",
+ help="Limit the number of search results, default=500.",
+ default=500,
+ type=int,
+ )
+ parser.add_argument(
+ "-S",
+ "--start",
+ help="Start with result number X, default=0.",
+ default=0,
+ type=int,
+ )
+ parser.add_argument(
+ "-p",
+ "--proxies",
+ help="Use proxies for requests, enter proxies in proxies.yaml.",
+ default=False,
+ action="store_true",
+ )
+ parser.add_argument(
+ "-s",
+ "--shodan",
+ help="Use Shodan to query discovered hosts.",
+ default=False,
+ action="store_true",
+ )
+ parser.add_argument(
+ "--screenshot",
+ help="Take screenshots of resolved domains specify output directory: --screenshot output_directory",
+ default="",
+ type=str,
+ )
+ parser.add_argument(
+ "-v",
+ "--virtual-host",
+ help="Verify host name via DNS resolution and search for virtual hosts.",
+ action="store_const",
+ const="basic",
+ default=False,
+ )
+ parser.add_argument("-e", "--dns-server", help="DNS server to use for lookup.")
+ parser.add_argument(
+ "-t",
+ "--take-over",
+ help="Check for takeovers.",
+ default=False,
+ action="store_true",
+ )
+ parser.add_argument(
+ "-r",
+ "--dns-resolve",
+ help="Perform DNS resolution on subdomains with a resolver list or passed in resolvers, default False.",
+ default="",
+ type=str,
+ nargs="?",
+ )
+ parser.add_argument(
+ "-n",
+ "--dns-lookup",
+ help="Enable DNS server lookup, default False.",
+ default=False,
+ action="store_true",
+ )
+ parser.add_argument(
+ "-c",
+ "--dns-brute",
+ help="Perform a DNS brute force on the domain.",
+ default=False,
+ action="store_true",
+ )
+ parser.add_argument(
+ "-f",
+ "--filename",
+ help="Save the results to an XML and JSON file.",
+ default="",
+ type=str,
+ )
+ parser.add_argument(
+ "-b",
+ "--source",
+ help="""anubis, baidu, bevigil, binaryedge, bing, bingapi, bufferoverun, brave,
censys, certspotter, criminalip, crtsh, dnsdumpster, duckduckgo, fullhunt, github-code,
hackertarget, hunter, hunterhow, intelx, netlas, onyphe, otx, pentesttools, projectdiscovery,
rapiddns, rocketreach, securityTrails, sitedossier, subdomaincenter, subdomainfinderc99, threatminer, tomba,
- urlscan, virustotal, yahoo, zoomeye''')
+ urlscan, virustotal, yahoo, zoomeye""",
+ )
# determines if filename is coming from rest api or user
- rest_filename = ''
+ rest_filename = ""
# indicates this from the rest API
if rest_args:
if rest_args.source and rest_args.source == "getsources":
@@ -63,8 +128,11 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
# We need to make sure the filename is random as to not overwrite other files
filename: str = args.filename
alphabet = string.ascii_letters + string.digits
- rest_filename += f"{''.join(secrets.choice(alphabet) for _ in range(32))}_{filename}" \
- if len(filename) != 0 else ""
+ rest_filename += (
+ f"{''.join(secrets.choice(alphabet) for _ in range(32))}_{filename}"
+ if len(filename) != 0
+ else ""
+ )
else:
args = parser.parse_args()
filename = args.filename
@@ -75,6 +143,7 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
except Exception:
pass
import os
+
if len(filename) > 2 and filename[:2] == "~/":
filename = os.path.expanduser(filename)
@@ -82,7 +151,9 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
all_hosts: List = []
all_ip: List = []
dnslookup = args.dns_lookup
- dnsserver = args.dns_server # TODO arg is not used anywhere replace with resolvers wordlist arg dnsresolve
+ dnsserver = (
+ args.dns_server
+ ) # TODO arg is not used anywhere replace with resolvers wordlist arg dnsresolve
dnsresolve = args.dns_resolve
final_dns_resolver_list = []
if dnsresolve is not None and len(dnsresolve) > 0:
@@ -91,7 +162,7 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
# 1.1.1.1,8.8.8.8 or 1.1.1.1, 8.8.8.8
# resolvers.txt
if os.path.exists(dnsresolve):
- with open(dnsresolve, mode='r', encoding='UTF-8') as fp:
+ with open(dnsresolve, mode="r", encoding="UTF-8") as fp:
for line in fp:
line = line.strip()
try:
@@ -99,14 +170,16 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
_ = netaddr.IPAddress(line)
final_dns_resolver_list.append(line)
except Exception as e:
- print(f'An exception has occurred while reading from: {dnsresolve}, {e}')
- print(f'Current line: {line}')
+ print(
+ f"An exception has occurred while reading from: {dnsresolve}, {e}"
+ )
+ print(f"Current line: {line}")
return
else:
try:
- if ',' in dnsresolve:
- cleaned = dnsresolve.replace(' ', '')
- for item in cleaned.split(','):
+ if "," in dnsresolve:
+ cleaned = dnsresolve.replace(" ", "")
+ for item in cleaned.split(","):
_ = netaddr.IPAddress(item)
final_dns_resolver_list.append(item)
else:
@@ -114,8 +187,10 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
_ = netaddr.IPAddress(dnsresolve)
final_dns_resolver_list.append(dnsresolve)
except Exception as e:
- print(f'Passed in DNS resolvers are invalid double check, got error: {e}')
- print(f'Dumping resolvers passed in: {e}')
+ print(
+ f"Passed in DNS resolvers are invalid double check, got error: {e}"
+ )
+ print(f"Dumping resolvers passed in: {e}")
sys.exit(0)
# if for some reason, there are duplicates
@@ -132,7 +207,7 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
all_urls: list = []
vhost: list = []
virtual = args.virtual_host
- word: str = args.domain.rstrip('\n')
+ word: str = args.domain.rstrip("\n")
takeover_status = args.take_over
use_proxy = args.proxies
linkedin_people_list_tracker: List = []
@@ -148,10 +223,19 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
interesting_urls = []
total_asns = []
- async def store(search_engine: Any, source: str, process_param: Any = None, store_host: bool = False,
- store_emails: bool = False, store_ip: bool = False, store_people: bool = False,
- store_links: bool = False, store_results: bool = False,
- store_interestingurls: bool = False, store_asns: bool = False) -> None:
+ async def store(
+ search_engine: Any,
+ source: str,
+ process_param: Any = None,
+ store_host: bool = False,
+ store_emails: bool = False,
+ store_ip: bool = False,
+ store_people: bool = False,
+ store_links: bool = False,
+ store_results: bool = False,
+ store_interestingurls: bool = False,
+ store_asns: bool = False,
+ ) -> None:
"""
Persist details into the database.
The details to be stored are controlled by the parameters passed to the method.
@@ -168,24 +252,43 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
:param store_interestingurls: whether to store interesting urls
:param store_asns: whether to store asns
"""
- await search_engine.process(use_proxy) if process_param is None else await \
- search_engine.process(process_param, use_proxy)
+ await search_engine.process(
+ use_proxy
+ ) if process_param is None else await search_engine.process(
+ process_param, use_proxy
+ )
db_stash = stash.StashManager()
if source:
- print(f'\033[94m[*] Searching {source[0].upper() + source[1:]}. ')
+ print(f"\033[94m[*] Searching {source[0].upper() + source[1:]}. ")
if store_host:
- host_names = list({host for host in filter(await search_engine.get_hostnames()) if f'.{word}' in host})
+ host_names = list(
+ {
+ host
+ for host in filter(await search_engine.get_hostnames())
+ if f".{word}" in host
+ }
+ )
host_names = list(host_names)
- if source != 'hackertarget' and source != 'pentesttools' and source != 'rapiddns':
+ if (
+ source != "hackertarget"
+ and source != "pentesttools"
+ and source != "rapiddns"
+ ):
# If a source is inside this conditional, it means the hosts returned must be resolved to obtain ip
# This should only be checked if --dns-resolve has a wordlist
if dnsresolve is None or len(final_dns_resolver_list) > 0:
# indicates that -r was passed in if dnsresolve is None
- full_hosts_checker = hostchecker.Checker(host_names, final_dns_resolver_list)
+ full_hosts_checker = hostchecker.Checker(
+ host_names, final_dns_resolver_list
+ )
# If full this is only getting resolved hosts
- resolved_pair, temp_hosts, temp_ips = await full_hosts_checker.check()
+ (
+ resolved_pair,
+ temp_hosts,
+ temp_ips,
+ ) = await full_hosts_checker.check()
all_ip.extend(temp_ips)
full.extend(resolved_pair)
# full.extend(temp_hosts)
@@ -194,382 +297,610 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
else:
full.extend(host_names)
all_hosts.extend(host_names)
- await db_stash.store_all(word, all_hosts, 'host', source)
+ await db_stash.store_all(word, all_hosts, "host", source)
if store_emails:
email_list = filter(await search_engine.get_emails())
all_emails.extend(email_list)
- await db_stash.store_all(word, email_list, 'email', source)
+ await db_stash.store_all(word, email_list, "email", source)
if store_ip:
ips_list = await search_engine.get_ips()
all_ip.extend(ips_list)
- await db_stash.store_all(word, all_ip, 'ip', source)
+ await db_stash.store_all(word, all_ip, "ip", source)
if store_results:
email_list, host_names, urls = await search_engine.get_results()
all_emails.extend(email_list)
- host_names = [host for host in filter(host_names) if f'.{word}' in host]
+ host_names = [host for host in filter(host_names) if f".{word}" in host]
all_urls.extend(filter(urls))
all_hosts.extend(host_names)
- await db.store_all(word, all_hosts, 'host', source)
- await db.store_all(word, all_emails, 'email', source)
+ await db.store_all(word, all_hosts, "host", source)
+ await db.store_all(word, all_emails, "email", source)
if store_people:
people_list = await search_engine.get_people()
- await db_stash.store_all(word, people_list, 'people', source)
+ await db_stash.store_all(word, people_list, "people", source)
if store_links:
links = await search_engine.get_links()
linkedin_links_tracker.extend(links)
if len(links) > 0:
- await db.store_all(word, links, 'linkedinlinks', engineitem)
+ await db.store_all(word, links, "linkedinlinks", engineitem)
if store_interestingurls:
iurls = await search_engine.get_interestingurls()
interesting_urls.extend(iurls)
if len(iurls) > 0:
- await db.store_all(word, iurls, 'interestingurls', engineitem)
+ await db.store_all(word, iurls, "interestingurls", engineitem)
if store_asns:
fasns = await search_engine.get_asns()
total_asns.extend(fasns)
if len(fasns) > 0:
- await db.store_all(word, fasns, 'asns', engineitem)
+ await db.store_all(word, fasns, "asns", engineitem)
stor_lst = []
if args.source is not None:
- if args.source.lower() != 'all':
- engines = sorted(set(map(str.strip, args.source.split(','))))
+ if args.source.lower() != "all":
+ engines = sorted(set(map(str.strip, args.source.split(","))))
else:
engines = Core.get_supportedengines()
# Iterate through search engines in order
if set(engines).issubset(Core.get_supportedengines()):
- print(f'\n[*] Target: {word} \n')
+ print(f"\n[*] Target: {word} \n")
for engineitem in engines:
- if engineitem == 'anubis':
+ if engineitem == "anubis":
from theHarvester.discovery import anubis
+
try:
anubis_search = anubis.SearchAnubis(word)
- stor_lst.append(store(anubis_search, engineitem, store_host=True))
+ stor_lst.append(
+ store(anubis_search, engineitem, store_host=True)
+ )
except Exception as e:
print(e)
- elif engineitem == 'baidu':
+ elif engineitem == "baidu":
from theHarvester.discovery import baidusearch
+
try:
baidu_search = baidusearch.SearchBaidu(word, limit)
- stor_lst.append(store(baidu_search, engineitem, store_host=True, store_emails=True))
+ stor_lst.append(
+ store(
+ baidu_search,
+ engineitem,
+ store_host=True,
+ store_emails=True,
+ )
+ )
except Exception as e:
print(e)
- elif engineitem == 'bevigil':
+ elif engineitem == "bevigil":
from theHarvester.discovery import bevigil
+
try:
bevigil_search = bevigil.SearchBeVigil(word)
- stor_lst.append(store(bevigil_search, engineitem, store_host=True, store_interestingurls=True))
+ stor_lst.append(
+ store(
+ bevigil_search,
+ engineitem,
+ store_host=True,
+ store_interestingurls=True,
+ )
+ )
except Exception as e:
print(e)
- elif engineitem == 'binaryedge':
+ elif engineitem == "binaryedge":
from theHarvester.discovery import binaryedgesearch
+
try:
- binaryedge_search = binaryedgesearch.SearchBinaryEdge(word, limit)
- stor_lst.append(store(binaryedge_search, engineitem, store_host=True))
+ binaryedge_search = binaryedgesearch.SearchBinaryEdge(
+ word, limit
+ )
+ stor_lst.append(
+ store(binaryedge_search, engineitem, store_host=True)
+ )
except Exception as e:
print(e)
- elif engineitem == 'bing' or engineitem == 'bingapi':
+ elif engineitem == "bing" or engineitem == "bingapi":
from theHarvester.discovery import bingsearch
+
try:
bing_search = bingsearch.SearchBing(word, limit, start)
- bingapi = ''
- if engineitem == 'bingapi':
- bingapi += 'yes'
+ bingapi = ""
+ if engineitem == "bingapi":
+ bingapi += "yes"
else:
- bingapi += 'no'
+ bingapi += "no"
stor_lst.append(
- store(bing_search, 'bing', process_param=bingapi, store_host=True, store_emails=True))
+ store(
+ bing_search,
+ "bing",
+ process_param=bingapi,
+ store_host=True,
+ store_emails=True,
+ )
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
print(e)
- elif engineitem == 'bufferoverun':
+ elif engineitem == "bufferoverun":
from theHarvester.discovery import bufferoverun
+
try:
bufferoverun_search = bufferoverun.SearchBufferover(word)
- stor_lst.append(store(bufferoverun_search, engineitem, store_host=True, store_ip=True))
+ stor_lst.append(
+ store(
+ bufferoverun_search,
+ engineitem,
+ store_host=True,
+ store_ip=True,
+ )
+ )
except Exception as e:
print(e)
- elif engineitem == 'brave':
+ elif engineitem == "brave":
from theHarvester.discovery import bravesearch
+
try:
brave_search = bravesearch.SearchBrave(word, limit)
- stor_lst.append(store(brave_search, engineitem, store_host=True, store_emails=True))
+ stor_lst.append(
+ store(
+ brave_search,
+ engineitem,
+ store_host=True,
+ store_emails=True,
+ )
+ )
except Exception as e:
print(e)
- elif engineitem == 'censys':
+ elif engineitem == "censys":
from theHarvester.discovery import censysearch
+
try:
censys_search = censysearch.SearchCensys(word, limit)
- stor_lst.append(store(censys_search, engineitem, store_host=True, store_emails=True))
+ stor_lst.append(
+ store(
+ censys_search,
+ engineitem,
+ store_host=True,
+ store_emails=True,
+ )
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
- elif engineitem == 'certspotter':
+ elif engineitem == "certspotter":
from theHarvester.discovery import certspottersearch
+
try:
certspotter_search = certspottersearch.SearchCertspoter(word)
- stor_lst.append(store(certspotter_search, engineitem, None, store_host=True))
+ stor_lst.append(
+ store(certspotter_search, engineitem, None, store_host=True)
+ )
except Exception as e:
print(e)
- elif engineitem == 'criminalip':
+ elif engineitem == "criminalip":
from theHarvester.discovery import criminalip
+
try:
criminalip_search = criminalip.SearchCriminalIP(word)
- stor_lst.append(store(criminalip_search, engineitem, store_host=True, store_ip=True,
- store_asns=True))
+ stor_lst.append(
+ store(
+ criminalip_search,
+ engineitem,
+ store_host=True,
+ store_ip=True,
+ store_asns=True,
+ )
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
- print(f'An excepion has occurred in criminalip: {e}')
+ print(f"An excepion has occurred in criminalip: {e}")
- elif engineitem == 'crtsh':
+ elif engineitem == "crtsh":
try:
from theHarvester.discovery import crtsh
- crtsh_search = crtsh.SearchCrtsh(word)
- stor_lst.append(store(crtsh_search, 'CRTsh', store_host=True))
- except Exception as e:
- print(f'[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}')
- elif engineitem == 'dnsdumpster':
+ crtsh_search = crtsh.SearchCrtsh(word)
+ stor_lst.append(store(crtsh_search, "CRTsh", store_host=True))
+ except Exception as e:
+ print(
+ f"[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}"
+ )
+
+ elif engineitem == "dnsdumpster":
try:
from theHarvester.discovery import dnsdumpster
+
dns_dumpster_search = dnsdumpster.SearchDnsDumpster(word)
- stor_lst.append(store(dns_dumpster_search, engineitem, store_host=True, store_ip=True))
+ stor_lst.append(
+ store(
+ dns_dumpster_search,
+ engineitem,
+ store_host=True,
+ store_ip=True,
+ )
+ )
except Exception as e:
- print(f'[!] An error occurred with dnsdumpster: {e}')
+ print(f"[!] An error occurred with dnsdumpster: {e}")
- elif engineitem == 'duckduckgo':
+ elif engineitem == "duckduckgo":
from theHarvester.discovery import duckduckgosearch
- duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(word, limit)
- stor_lst.append(store(duckduckgo_search, engineitem, store_host=True, store_emails=True))
- elif engineitem == 'fullhunt':
+ duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(word, limit)
+ stor_lst.append(
+ store(
+ duckduckgo_search,
+ engineitem,
+ store_host=True,
+ store_emails=True,
+ )
+ )
+
+ elif engineitem == "fullhunt":
from theHarvester.discovery import fullhuntsearch
+
try:
fullhunt_search = fullhuntsearch.SearchFullHunt(word)
- stor_lst.append(store(fullhunt_search, engineitem, store_host=True))
+ stor_lst.append(
+ store(fullhunt_search, engineitem, store_host=True)
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
- elif engineitem == 'github-code':
+ elif engineitem == "github-code":
try:
from theHarvester.discovery import githubcode
+
github_search = githubcode.SearchGithubCode(word, limit)
- stor_lst.append(store(github_search, engineitem, store_host=True, store_emails=True))
+ stor_lst.append(
+ store(
+ github_search,
+ engineitem,
+ store_host=True,
+ store_emails=True,
+ )
+ )
except MissingKey as ex:
print(ex)
- elif engineitem == 'hackertarget':
+ elif engineitem == "hackertarget":
from theHarvester.discovery import hackertarget
- hackertarget_search = hackertarget.SearchHackerTarget(word)
- stor_lst.append(store(hackertarget_search, engineitem, store_host=True))
- elif engineitem == 'hunter':
+ hackertarget_search = hackertarget.SearchHackerTarget(word)
+ stor_lst.append(
+ store(hackertarget_search, engineitem, store_host=True)
+ )
+
+ elif engineitem == "hunter":
from theHarvester.discovery import huntersearch
+
# Import locally or won't work.
try:
hunter_search = huntersearch.SearchHunter(word, limit, start)
- stor_lst.append(store(hunter_search, engineitem, store_host=True, store_emails=True))
+ stor_lst.append(
+ store(
+ hunter_search,
+ engineitem,
+ store_host=True,
+ store_emails=True,
+ )
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
- elif engineitem == 'hunterhow':
+ elif engineitem == "hunterhow":
from theHarvester.discovery import searchhunterhow
+
try:
hunterhow_search = searchhunterhow.SearchHunterHow(word)
- stor_lst.append(store(hunterhow_search, engineitem, store_host=True))
+ stor_lst.append(
+ store(hunterhow_search, engineitem, store_host=True)
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
- print(f'An exception has occurred in hunterhow search: {e}')
+ print(f"An exception has occurred in hunterhow search: {e}")
- elif engineitem == 'intelx':
+ elif engineitem == "intelx":
from theHarvester.discovery import intelxsearch
+
# Import locally or won't work.
try:
intelx_search = intelxsearch.SearchIntelx(word)
- stor_lst.append(store(intelx_search, engineitem, store_interestingurls=True, store_emails=True))
+ stor_lst.append(
+ store(
+ intelx_search,
+ engineitem,
+ store_interestingurls=True,
+ store_emails=True,
+ )
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
- print(f'An exception has occurred in Intelx search: {e}')
+ print(f"An exception has occurred in Intelx search: {e}")
- elif engineitem == 'netlas':
+ elif engineitem == "netlas":
from theHarvester.discovery import netlas
+
try:
netlas_search = netlas.SearchNetlas(word)
- stor_lst.append(store(netlas_search, engineitem, store_host=True, store_ip=True))
+ stor_lst.append(
+ store(
+ netlas_search,
+ engineitem,
+ store_host=True,
+ store_ip=True,
+ )
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
- elif engineitem == 'onyphe':
+ elif engineitem == "onyphe":
from theHarvester.discovery import onyphe
+
try:
onyphe_search = onyphe.SearchOnyphe(word)
- stor_lst.append(store(onyphe_search, engineitem, store_host=True, store_ip=True,
- store_asns=True))
+ stor_lst.append(
+ store(
+ onyphe_search,
+ engineitem,
+ store_host=True,
+ store_ip=True,
+ store_asns=True,
+ )
+ )
except Exception as e:
print(e)
- elif engineitem == 'otx':
+ elif engineitem == "otx":
from theHarvester.discovery import otxsearch
+
try:
otxsearch_search = otxsearch.SearchOtx(word)
- stor_lst.append(store(otxsearch_search, engineitem, store_host=True, store_ip=True))
+ stor_lst.append(
+ store(
+ otxsearch_search,
+ engineitem,
+ store_host=True,
+ store_ip=True,
+ )
+ )
except Exception as e:
print(e)
- elif engineitem == 'pentesttools':
+ elif engineitem == "pentesttools":
from theHarvester.discovery import pentesttools
+
try:
pentesttools_search = pentesttools.SearchPentestTools(word)
- stor_lst.append(store(pentesttools_search, engineitem, store_host=True))
+ stor_lst.append(
+ store(pentesttools_search, engineitem, store_host=True)
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
- print(f'An exception has occurred in PentestTools search: {e}')
+ print(
+ f"An exception has occurred in PentestTools search: {e}"
+ )
- elif engineitem == 'projectdiscovery':
+ elif engineitem == "projectdiscovery":
from theHarvester.discovery import projectdiscovery
+
try:
projectdiscovery_search = projectdiscovery.SearchDiscovery(word)
- stor_lst.append(store(projectdiscovery_search, engineitem, store_host=True))
+ stor_lst.append(
+ store(projectdiscovery_search, engineitem, store_host=True)
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
- print('An exception has occurred in ProjectDiscovery')
+ print("An exception has occurred in ProjectDiscovery")
- elif engineitem == 'rapiddns':
+ elif engineitem == "rapiddns":
from theHarvester.discovery import rapiddns
+
try:
rapiddns_search = rapiddns.SearchRapidDns(word)
- stor_lst.append(store(rapiddns_search, engineitem, store_host=True))
+ stor_lst.append(
+ store(rapiddns_search, engineitem, store_host=True)
+ )
except Exception as e:
print(e)
- elif engineitem == 'rocketreach':
+ elif engineitem == "rocketreach":
from theHarvester.discovery import rocketreach
+
try:
rocketreach_search = rocketreach.SearchRocketReach(word, limit)
- stor_lst.append(store(rocketreach_search, engineitem, store_links=True))
+ stor_lst.append(
+ store(rocketreach_search, engineitem, store_links=True)
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
- print(f'An exception has occurred in RocketReach: {e}')
+ print(f"An exception has occurred in RocketReach: {e}")
- elif engineitem == 'subdomaincenter':
+ elif engineitem == "subdomaincenter":
from theHarvester.discovery import subdomaincenter
+
try:
subdomaincenter_search = subdomaincenter.SubdomainCenter(word)
- stor_lst.append(store(subdomaincenter_search, engineitem, store_host=True))
+ stor_lst.append(
+ store(subdomaincenter_search, engineitem, store_host=True)
+ )
except Exception as e:
print(e)
- elif engineitem == 'securityTrails':
+ elif engineitem == "securityTrails":
from theHarvester.discovery import securitytrailssearch
+
try:
- securitytrails_search = securitytrailssearch.SearchSecuritytrail(word)
- stor_lst.append(store(securitytrails_search, engineitem, store_host=True, store_ip=True))
+ securitytrails_search = (
+ securitytrailssearch.SearchSecuritytrail(word)
+ )
+ stor_lst.append(
+ store(
+ securitytrails_search,
+ engineitem,
+ store_host=True,
+ store_ip=True,
+ )
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
- elif engineitem == 'sitedossier':
+ elif engineitem == "sitedossier":
from theHarvester.discovery import sitedossier
+
try:
sitedossier_search = sitedossier.SearchSitedossier(word)
- stor_lst.append(store(sitedossier_search, engineitem, store_host=True))
+ stor_lst.append(
+ store(sitedossier_search, engineitem, store_host=True)
+ )
except Exception as e:
print(e)
- elif engineitem == 'subdomainfinderc99':
+ elif engineitem == "subdomainfinderc99":
from theHarvester.discovery import subdomainfinderc99
+
try:
- subdomainfinderc99_search = subdomainfinderc99.SearchSubdomainfinderc99(word)
- stor_lst.append(store(subdomainfinderc99_search, engineitem, store_host=True))
+ subdomainfinderc99_search = (
+ subdomainfinderc99.SearchSubdomainfinderc99(word)
+ )
+ stor_lst.append(
+ store(
+ subdomainfinderc99_search, engineitem, store_host=True
+ )
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
- print(f'An exception has occurred in Subdomainfinderc99 search: {e}')
+ print(
+ f"An exception has occurred in Subdomainfinderc99 search: {e}"
+ )
- elif engineitem == 'threatminer':
+ elif engineitem == "threatminer":
from theHarvester.discovery import threatminer
+
try:
threatminer_search = threatminer.SearchThreatminer(word)
- stor_lst.append(store(threatminer_search, engineitem, store_host=True, store_ip=True))
+ stor_lst.append(
+ store(
+ threatminer_search,
+ engineitem,
+ store_host=True,
+ store_ip=True,
+ )
+ )
except Exception as e:
print(e)
- elif engineitem == 'tomba':
+ elif engineitem == "tomba":
try:
from theHarvester.discovery import tombasearch
+
tomba_search = tombasearch.SearchTomba(word, limit, start)
- stor_lst.append(store(tomba_search, engineitem, store_host=True, store_emails=True))
+ stor_lst.append(
+ store(
+ tomba_search,
+ engineitem,
+ store_host=True,
+ store_emails=True,
+ )
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
- elif engineitem == 'urlscan':
+ elif engineitem == "urlscan":
from theHarvester.discovery import urlscan
+
try:
urlscan_search = urlscan.SearchUrlscan(word)
- stor_lst.append(store(urlscan_search, engineitem, store_host=True, store_ip=True,
- store_interestingurls=True, store_asns=True))
+ stor_lst.append(
+ store(
+ urlscan_search,
+ engineitem,
+ store_host=True,
+ store_ip=True,
+ store_interestingurls=True,
+ store_asns=True,
+ )
+ )
except Exception as e:
print(e)
- elif engineitem == 'virustotal':
+ elif engineitem == "virustotal":
try:
from theHarvester.discovery import virustotal
+
virustotal_search = virustotal.SearchVirustotal(word)
- stor_lst.append(store(virustotal_search, engineitem, store_host=True))
+ stor_lst.append(
+ store(virustotal_search, engineitem, store_host=True)
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
- elif engineitem == 'yahoo':
+ elif engineitem == "yahoo":
from theHarvester.discovery import yahoosearch
+
try:
yahoo_search = yahoosearch.SearchYahoo(word, limit)
- stor_lst.append(store(yahoo_search, engineitem, store_host=True, store_emails=True))
+ stor_lst.append(
+ store(
+ yahoo_search,
+ engineitem,
+ store_host=True,
+ store_emails=True,
+ )
+ )
except Exception as e:
print(e)
- elif engineitem == 'zoomeye':
+ elif engineitem == "zoomeye":
try:
from theHarvester.discovery import zoomeyesearch
+
zoomeye_search = zoomeyesearch.SearchZoomEye(word, limit)
- stor_lst.append(store(zoomeye_search, engineitem, store_host=True, store_emails=True,
- store_ip=True, store_interestingurls=True, store_asns=True))
+ stor_lst.append(
+ store(
+ zoomeye_search,
+ engineitem,
+ store_host=True,
+ store_emails=True,
+ store_ip=True,
+ store_interestingurls=True,
+ store_asns=True,
+ )
+ )
except Exception as e:
if isinstance(e, MissingKey):
print(e)
@@ -578,10 +909,10 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
try:
rest_args.dns_brute
except Exception:
- print('\n[!] Invalid source.\n')
+ print("\n[!] Invalid source.\n")
sys.exit(1)
else:
- print('\n[!] Invalid source.\n')
+ print("\n[!] Invalid source.\n")
sys.exit(1)
async def worker(queue):
@@ -617,157 +948,191 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
await handler(lst=stor_lst)
return_ips: List = []
- if rest_args is not None and len(rest_filename) == 0 and rest_args.dns_brute is False:
+ if (
+ rest_args is not None
+ and len(rest_filename) == 0
+ and rest_args.dns_brute is False
+ ):
# Indicates user is using REST api but not wanting output to be saved to a file
# for host in full:
# full = [host if ':' in host and word in host else word in host.split(':')[0] and host for host in full]
# full = list({host for host in full if host})
# full.sort()
# cast to string so Rest API can understand type
- return_ips.extend([str(ip) for ip in sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)])])
+ return_ips.extend(
+ [
+ str(ip)
+ for ip in sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)])
+ ]
+ )
# return list(set(all_emails)), return_ips, full, '', ''
- all_hosts = [host.replace('www.', '') for host in all_hosts if host.replace('www.', '') in all_hosts]
+ all_hosts = [
+ host.replace("www.", "")
+ for host in all_hosts
+ if host.replace("www.", "") in all_hosts
+ ]
all_hosts = list(sorted(set(all_hosts)))
- return total_asns, interesting_urls, twitter_people_list_tracker, linkedin_people_list_tracker, \
- linkedin_links_tracker, all_urls, all_ip, all_emails, all_hosts
+ return (
+ total_asns,
+ interesting_urls,
+ twitter_people_list_tracker,
+ linkedin_people_list_tracker,
+ linkedin_links_tracker,
+ all_urls,
+ all_ip,
+ all_emails,
+ all_hosts,
+ )
# Check to see if all_emails and all_hosts are defined.
try:
all_emails
except NameError:
- print('\n\n[!] No emails found because all_emails is not defined.\n\n ')
+ print("\n\n[!] No emails found because all_emails is not defined.\n\n ")
sys.exit(1)
try:
all_hosts
except NameError:
- print('\n\n[!] No hosts found because all_hosts is not defined.\n\n ')
+ print("\n\n[!] No hosts found because all_hosts is not defined.\n\n ")
sys.exit(1)
# Results
if len(total_asns) > 0:
- print(f'\n[*] ASNS found: {len(total_asns)}')
- print('--------------------')
+ print(f"\n[*] ASNS found: {len(total_asns)}")
+ print("--------------------")
total_asns = list(sorted(set(total_asns)))
for asn in total_asns:
print(asn)
if len(interesting_urls) > 0:
- print(f'\n[*] Interesting Urls found: {len(interesting_urls)}')
- print('--------------------')
+ print(f"\n[*] Interesting Urls found: {len(interesting_urls)}")
+ print("--------------------")
interesting_urls = list(sorted(set(interesting_urls)))
for iurl in interesting_urls:
print(iurl)
- if len(twitter_people_list_tracker) == 0 and 'twitter' in engines:
- print('\n[*] No Twitter users found.\n\n')
+ if len(twitter_people_list_tracker) == 0 and "twitter" in engines:
+ print("\n[*] No Twitter users found.\n\n")
else:
if len(twitter_people_list_tracker) >= 1:
- print('\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker)))
- print('---------------------')
+ print("\n[*] Twitter Users found: " + str(len(twitter_people_list_tracker)))
+ print("---------------------")
twitter_people_list_tracker = list(sorted(set(twitter_people_list_tracker)))
for usr in twitter_people_list_tracker:
print(usr)
- if len(linkedin_people_list_tracker) == 0 and 'linkedin' in engines:
- print('\n[*] No LinkedIn users found.\n\n')
+ if len(linkedin_people_list_tracker) == 0 and "linkedin" in engines:
+ print("\n[*] No LinkedIn users found.\n\n")
else:
if len(linkedin_people_list_tracker) >= 1:
- print('\n[*] LinkedIn Users found: ' + str(len(linkedin_people_list_tracker)))
- print('---------------------')
- linkedin_people_list_tracker = list(sorted(set(linkedin_people_list_tracker)))
+ print(
+ "\n[*] LinkedIn Users found: " + str(len(linkedin_people_list_tracker))
+ )
+ print("---------------------")
+ linkedin_people_list_tracker = list(
+ sorted(set(linkedin_people_list_tracker))
+ )
for usr in linkedin_people_list_tracker:
print(usr)
- if len(linkedin_links_tracker) == 0 and ('linkedin' in engines or 'rocketreach' in engines):
- print(f'\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}')
+ if len(linkedin_links_tracker) == 0 and (
+ "linkedin" in engines or "rocketreach" in engines
+ ):
+ print(f"\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}")
linkedin_links_tracker = list(sorted(set(linkedin_links_tracker)))
- print('---------------------')
+ print("---------------------")
for link in linkedin_people_list_tracker:
print(link)
length_urls = len(all_urls)
if length_urls == 0:
- if len(engines) >= 1 and 'trello' in engines:
- print('\n[*] No Trello URLs found.')
+ if len(engines) >= 1 and "trello" in engines:
+ print("\n[*] No Trello URLs found.")
else:
total = length_urls
- print('\n[*] Trello URLs found: ' + str(total))
- print('--------------------')
+ print("\n[*] Trello URLs found: " + str(total))
+ print("--------------------")
all_urls = list(sorted(set(all_urls)))
for url in sorted(all_urls):
print(url)
if len(all_ip) == 0:
- print('\n[*] No IPs found.')
+ print("\n[*] No IPs found.")
else:
- print('\n[*] IPs found: ' + str(len(all_ip)))
- print('-------------------')
+ print("\n[*] IPs found: " + str(len(all_ip)))
+ print("-------------------")
# use netaddr as the list may contain ipv4 and ipv6 addresses
ip_list = []
for ip in set(all_ip):
try:
ip = ip.strip()
if len(ip) > 0:
- if '/' in ip:
+ if "/" in ip:
ip_list.append(str(netaddr.IPNetwork(ip)))
else:
ip_list.append(str(netaddr.IPAddress(ip)))
except Exception as e:
- print(f'An exception has occurred while adding: {ip} to ip_list: {e}')
+ print(f"An exception has occurred while adding: {ip} to ip_list: {e}")
continue
ip_list = list(sorted(ip_list))
- print('\n'.join(map(str, ip_list)))
+ print("\n".join(map(str, ip_list)))
if len(all_emails) == 0:
- print('\n[*] No emails found.')
+ print("\n[*] No emails found.")
else:
- print('\n[*] Emails found: ' + str(len(all_emails)))
- print('----------------------')
+ print("\n[*] Emails found: " + str(len(all_emails)))
+ print("----------------------")
all_emails = sorted(list(set(all_emails)))
- print(('\n'.join(all_emails)))
+ print(("\n".join(all_emails)))
if len(all_hosts) == 0:
- print('\n[*] No hosts found.\n\n')
+ print("\n[*] No hosts found.\n\n")
else:
db = stash.StashManager()
if dnsresolve is None or len(final_dns_resolver_list) > 0:
temp = set()
for host in full:
- if ':' in host:
+ if ":" in host:
# TODO parse addresses and sort them as they are IPs
- subdomain, addr = host.split(':')
+ subdomain, addr = host.split(":")
if subdomain.endswith(word):
- temp.add(subdomain + ':' + addr)
+ temp.add(subdomain + ":" + addr)
continue
if host.endswith(word):
- if host[:4] == 'www.':
+ if host[:4] == "www.":
if host[4:] in all_hosts or host[4:] in full:
temp.add(host[4:])
continue
temp.add(host)
full = list(sorted(temp))
- full.sort(key=lambda el: el.split(':')[0])
- print('\n[*] Hosts found: ' + str(len(full)))
- print('---------------------')
+ full.sort(key=lambda el: el.split(":")[0])
+ print("\n[*] Hosts found: " + str(len(full)))
+ print("---------------------")
for host in full:
print(host)
try:
- if ':' in host:
- _, addr = host.split(':')
- await db.store(word, addr, 'ip', 'DNS-resolver')
+ if ":" in host:
+ _, addr = host.split(":")
+ await db.store(word, addr, "ip", "DNS-resolver")
except Exception as e:
- print(f'An exception has occurred while attempting to insert: {host} IP into DB: {e}')
+ print(
+ f"An exception has occurred while attempting to insert: {host} IP into DB: {e}"
+ )
continue
else:
- all_hosts = [host.replace('www.', '') for host in all_hosts if host.replace('www.', '') in all_hosts]
+ all_hosts = [
+ host.replace("www.", "")
+ for host in all_hosts
+ if host.replace("www.", "") in all_hosts
+ ]
all_hosts = list(sorted(set(all_hosts)))
- print('\n[*] Hosts found: ' + str(len(all_hosts)))
- print('---------------------')
+ print("\n[*] Hosts found: " + str(len(all_hosts)))
+ print("---------------------")
for host in all_hosts:
print(host)
# DNS brute force
if dnsbrute and dnsbrute[0] is True:
- print('\n[*] Starting DNS brute force.')
+ print("\n[*] Starting DNS brute force.")
dns_force = dnssearch.DnsForce(word, final_dns_resolver_list, verbose=True)
resolved_pair, hosts, ips = await dns_force.run()
# hosts = list({host for host in hosts if ':' in host})
@@ -778,19 +1143,19 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
db = stash.StashManager()
temp = set()
for host in resolved_pair:
- if ':' in host:
+ if ":" in host:
# TODO parse addresses and sort them as they are IPs
- subdomain, addr = host.split(':')
+ subdomain, addr = host.split(":")
if subdomain.endswith(word):
# Append to full so it's within JSON/XML at the end if output file is requested
if host not in full:
full.append(host)
- temp.add(subdomain + ':' + addr)
+ temp.add(subdomain + ":" + addr)
if host not in all_hosts:
all_hosts.append(host)
continue
if host.endswith(word):
- if host[:4] == 'www.':
+ if host[:4] == "www.":
if host[4:] in all_hosts or host[4:] in full:
continue
if host not in full:
@@ -798,16 +1163,16 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
temp.add(host)
if host not in all_hosts:
all_hosts.append(host)
- print('\n[*] Hosts found after DNS brute force:')
+ print("\n[*] Hosts found after DNS brute force:")
for sub in temp:
print(sub)
- await db.store_all(word, list(sorted(temp)), 'host', 'dns_bruteforce')
+ await db.store_all(word, list(sorted(temp)), "host", "dns_bruteforce")
takeover_results = dict()
# TakeOver Checking
if takeover_status:
- print('\n[*] Performing subdomain takeover check')
- print('\n[*] Subdomain Takeover checking IS ACTIVE RECON')
+ print("\n[*] Performing subdomain takeover check")
+ print("\n[*] Subdomain Takeover checking IS ACTIVE RECON")
search_take = takeover.TakeOver(all_hosts)
await search_take.populate_fingerprints()
await search_take.process(proxy=use_proxy)
@@ -816,51 +1181,56 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
dnsrev: List = []
# print(f'DNSlookup: {dnslookup}')
if dnslookup is True:
- print('\n[*] Starting active queries for DNSLookup.')
+ print("\n[*] Starting active queries for DNSLookup.")
# load the reverse dns tools
from theHarvester.discovery.dnssearch import (
generate_postprocessing_callback,
reverse_all_ips_in_range,
- serialize_ip_range)
+ serialize_ip_range,
+ )
# reverse each iprange in a separate task
__reverse_dns_tasks: Dict = {}
for entry in host_ip:
- __ip_range = serialize_ip_range(ip=entry, netmask='24')
+ __ip_range = serialize_ip_range(ip=entry, netmask="24")
if __ip_range and __ip_range not in set(__reverse_dns_tasks.keys()):
- print('\n[*] Performing reverse lookup on ' + __ip_range)
- __reverse_dns_tasks[__ip_range] = asyncio.create_task(reverse_all_ips_in_range(
- iprange=__ip_range,
- callback=generate_postprocessing_callback(
- target=word,
- local_results=dnsrev,
- overall_results=full),
- nameservers=final_dns_resolver_list if len(final_dns_resolver_list) > 0 else None))
+ print("\n[*] Performing reverse lookup on " + __ip_range)
+ __reverse_dns_tasks[__ip_range] = asyncio.create_task(
+ reverse_all_ips_in_range(
+ iprange=__ip_range,
+ callback=generate_postprocessing_callback(
+ target=word, local_results=dnsrev, overall_results=full
+ ),
+ nameservers=final_dns_resolver_list
+ if len(final_dns_resolver_list) > 0
+ else None,
+ )
+ )
# nameservers=list(map(str, dnsserver.split(','))) if dnsserver else None))
# run all the reversing tasks concurrently
await asyncio.gather(*__reverse_dns_tasks.values())
# Display the newly found hosts
- print('\n[*] Hosts found after reverse lookup (in target domain):')
- print('--------------------------------------------------------')
+ print("\n[*] Hosts found after reverse lookup (in target domain):")
+ print("--------------------------------------------------------")
for xh in dnsrev:
print(xh)
# Virtual hosts search
- if virtual == 'basic':
- print('\n[*] Virtual hosts:')
- print('------------------')
+ if virtual == "basic":
+ print("\n[*] Virtual hosts:")
+ print("------------------")
for data in host_ip:
basic_search = bingsearch.SearchBing(data, limit, start)
await basic_search.process_vhost()
results = await basic_search.get_allhostnames()
for result in results:
- result = re.sub(r'[[]*', '', result)
- result = re.sub('<', '', result)
- result = re.sub('>', '', result)
- print((data + '\t' + result))
- vhost.append(data + ':' + result)
- full.append(data + ':' + result)
+ result = re.sub(r"[[]*", "", result)
+ result = re.sub("<", "", result)
+ result = re.sub(">", "", result)
+ print((data + "\t" + result))
+ vhost.append(data + ":" + result)
+ full.append(data + ":" + result)
vhost = sorted(set(vhost))
else:
pass
@@ -869,115 +1239,152 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
screenshot_tups = []
if len(args.screenshot) > 0:
import time
+
from aiomultiprocess import Pool
+
from theHarvester.screenshot.screenshot import ScreenShotter
+
screen_shotter = ScreenShotter(args.screenshot)
path_exists = screen_shotter.verify_path()
# Verify path exists, if not create it or if user does not create it skips screenshot
if path_exists:
await screen_shotter.verify_installation()
- print(f'\nScreenshots can be found in: {screen_shotter.output}{screen_shotter.slash}')
+ print(
+ f"\nScreenshots can be found in: {screen_shotter.output}{screen_shotter.slash}"
+ )
start_time = time.perf_counter()
- print('Filtering domains for ones we can reach')
+ print("Filtering domains for ones we can reach")
if dnsresolve is None or len(final_dns_resolver_list) > 0:
- unique_resolved_domains = {url.split(':')[0] for url in full if ':' in url and 'www.' not in url}
+ unique_resolved_domains = {
+ url.split(":")[0]
+ for url in full
+ if ":" in url and "www." not in url
+ }
else:
# Technically not resolved in this case which is not ideal
# You should always use dns resolve when doing screenshotting
- print('NOTE for future use cases you should only use screenshotting in tandem with DNS resolving')
+ print(
+ "NOTE for future use cases you should only use screenshotting in tandem with DNS resolving"
+ )
unique_resolved_domains = set(all_hosts)
if len(unique_resolved_domains) > 0:
# First filter out ones that didn't resolve
- print('Attempting to visit unique resolved domains, this is ACTIVE RECON')
+ print(
+ "Attempting to visit unique resolved domains, this is ACTIVE RECON"
+ )
async with Pool(10) as pool:
- results = await pool.map(screen_shotter.visit, list(unique_resolved_domains))
+ results = await pool.map(
+ screen_shotter.visit, list(unique_resolved_domains)
+ )
# Filter out domains that we couldn't connect to
- unique_resolved_domains_list = list(sorted({tup[0] for tup in results if len(tup[1]) > 0}))
+ unique_resolved_domains_list = list(
+ sorted({tup[0] for tup in results if len(tup[1]) > 0})
+ )
async with Pool(3) as pool:
- print(f'Length of unique resolved domains: {len(unique_resolved_domains_list)} chunking now!\n')
+ print(
+ f"Length of unique resolved domains: {len(unique_resolved_domains_list)} chunking now!\n"
+ )
# If you have the resources, you could make the function faster by increasing the chunk number
chunk_number = 14
- for chunk in screen_shotter.chunk_list(unique_resolved_domains_list, chunk_number):
+ for chunk in screen_shotter.chunk_list(
+ unique_resolved_domains_list, chunk_number
+ ):
try:
- screenshot_tups.extend(await pool.map(screen_shotter.take_screenshot, chunk))
+ screenshot_tups.extend(
+ await pool.map(screen_shotter.take_screenshot, chunk)
+ )
except Exception as ee:
- print(f'An exception has occurred while mapping: {ee}')
+ print(f"An exception has occurred while mapping: {ee}")
end = time.perf_counter()
# There is probably an easier way to do this
total = int(end - start_time)
mon, sec = divmod(total, 60)
hr, mon = divmod(mon, 60)
total_time = "%02d:%02d" % (mon, sec)
- print(f'Finished taking screenshots in {total_time} seconds')
- print('[+] Note there may be leftover chrome processes you may have to kill manually\n')
+ print(f"Finished taking screenshots in {total_time} seconds")
+ print(
+ "[+] Note there may be leftover chrome processes you may have to kill manually\n"
+ )
# Shodan
shodanres = []
if shodan is True:
- print('\033[94m[*] Searching Shodan. ')
+ print("\033[94m[*] Searching Shodan. ")
try:
for ip in host_ip:
# TODO fix shodan
- print(('\tSearching for ' + ip))
+ print(("\tSearching for " + ip))
shodan = shodansearch.SearchShodan()
shodandict = await shodan.search_ip(ip)
await asyncio.sleep(5)
rowdata = []
for key, value in shodandict[ip].items():
- if str(value) == 'Not in Shodan' or 'Error occurred in the Shodan IP search module' in str(value):
+ if str(
+ value
+ ) == "Not in Shodan" or "Error occurred in the Shodan IP search module" in str(
+ value
+ ):
break
if isinstance(value, int):
value = str(value)
if isinstance(value, list):
- value = ', '.join(map(str, value))
+ value = ", ".join(map(str, value))
rowdata.append(value)
shodanres.append(rowdata)
print(ujson.dumps(shodandict[ip], indent=4, sort_keys=True))
- print('\n')
+ print("\n")
except Exception as e:
- print(f'[!] An error occurred with Shodan: {e} ')
+ print(f"[!] An error occurred with Shodan: {e} ")
else:
pass
- if filename != '':
- print('\n[*] Reporting started.')
+ if filename != "":
+ print("\n[*] Reporting started.")
try:
if len(rest_filename) == 0:
- filename = filename.rsplit('.', 1)[0] + '.xml'
+ filename = filename.rsplit(".", 1)[0] + ".xml"
else:
- filename = 'theHarvester/app/static/' + rest_filename.rsplit('.', 1)[0] + '.xml'
+ filename = (
+ "theHarvester/app/static/"
+ + rest_filename.rsplit(".", 1)[0]
+ + ".xml"
+ )
# TODO use aiofiles if user is using rest api
# XML REPORT SECTION
- with open(filename, 'w+') as file:
+ with open(filename, "w+") as file:
file.write('')
for x in all_emails:
- file.write('' + x + '')
+ file.write("" + x + "")
for x in full:
- host, ip = x.split(':', 1) if ':' in x else (x, '')
+ host, ip = x.split(":", 1) if ":" in x else (x, "")
if ip and len(ip) > 3:
- file.write(f'{ip}{host}')
+ file.write(
+ f"{ip}{host}"
+ )
else:
- file.write(f'{host}')
+ file.write(f"{host}")
for x in vhost:
- host, ip = x.split(':', 1) if ':' in x else (x, '')
+ host, ip = x.split(":", 1) if ":" in x else (x, "")
if ip and len(ip) > 3:
- file.write(f'{ip} {host}')
+ file.write(
+ f"{ip} {host}"
+ )
else:
- file.write(f'{host}')
+ file.write(f"{host}")
# TODO add Shodan output into XML report
- file.write('')
- print('[*] XML File saved.')
+ file.write("")
+ print("[*] XML File saved.")
except Exception as error:
- print(f'[!] An error occurred while saving the XML file: {error}')
+ print(f"[!] An error occurred while saving the XML file: {error}")
try:
# JSON REPORT SECTION
- filename = filename.rsplit('.', 1)[0] + '.json'
+ filename = filename.rsplit(".", 1)[0] + ".json"
# create dict with values for json output
json_dict: Dict = dict()
# determine if variable exists
# it should but just a validation check
- if 'ip_list' in locals():
+ if "ip_list" in locals():
if all_ip and len(all_ip) >= 1 and ip_list and len(ip_list) > 0:
json_dict["ips"] = ip_list
@@ -1013,16 +1420,16 @@ async def start(rest_args: Optional[argparse.Namespace] = None):
json_dict["linkedin_links"] = linkedin_links_tracker
if takeover_status and len(takeover_results) > 0:
- json_dict['takeover_results'] = takeover_results
+ json_dict["takeover_results"] = takeover_results
json_dict["shodan"] = shodanres
- with open(filename, 'w+') as fp:
+ with open(filename, "w+") as fp:
dumped_json = ujson.dumps(json_dict, sort_keys=True)
fp.write(dumped_json)
- print('[*] JSON File saved.')
+ print("[*] JSON File saved.")
except Exception as er:
- print(f'[!] An error occurred while saving the JSON file: {er} ')
- print('\n\n')
+ print(f"[!] An error occurred while saving the JSON file: {er} ")
+ print("\n\n")
sys.exit(0)
@@ -1031,7 +1438,7 @@ async def entry_point() -> None:
Core.banner()
await start()
except KeyboardInterrupt:
- print('\n\n[!] ctrl+c detected from user, quitting.\n\n ')
+ print("\n\n[!] ctrl+c detected from user, quitting.\n\n ")
except Exception as error_entry_point:
print(error_entry_point)
sys.exit(1)
diff --git a/theHarvester/discovery/anubis.py b/theHarvester/discovery/anubis.py
index f59fb849..672cd256 100644
--- a/theHarvester/discovery/anubis.py
+++ b/theHarvester/discovery/anubis.py
@@ -1,16 +1,16 @@
from typing import Type
+
from theHarvester.lib.core import *
class SearchAnubis:
-
def __init__(self, word) -> None:
self.word = word
self.totalhosts: List = []
self.proxy = False
async def do_search(self) -> None:
- url = f'https://jldc.me/anubis/subdomains/{self.word}'
+ url = f"https://jldc.me/anubis/subdomains/{self.word}"
response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy)
self.totalhosts = response[0]
diff --git a/theHarvester/discovery/baidusearch.py b/theHarvester/discovery/baidusearch.py
index 60ca1319..d714c33f 100644
--- a/theHarvester/discovery/baidusearch.py
+++ b/theHarvester/discovery/baidusearch.py
@@ -3,23 +3,25 @@ from theHarvester.parsers import myparser
class SearchBaidu:
-
def __init__(self, word, limit) -> None:
self.word = word
self.total_results = ""
- self.server = 'www.baidu.com'
- self.hostname = 'www.baidu.com'
+ self.server = "www.baidu.com"
+ self.hostname = "www.baidu.com"
self.limit = limit
self.proxy = False
async def do_search(self) -> None:
- headers = {
- 'Host': self.hostname,
- 'User-agent': Core.get_user_agent()
- }
- base_url = f'https://{self.server}/s?wd=%40{self.word}&pn=xx&oq={self.word}'
- urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 10) if num <= self.limit]
- responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy)
+ headers = {"Host": self.hostname, "User-agent": Core.get_user_agent()}
+ base_url = f"https://{self.server}/s?wd=%40{self.word}&pn=xx&oq={self.word}"
+ urls = [
+ base_url.replace("xx", str(num))
+ for num in range(0, self.limit, 10)
+ if num <= self.limit
+ ]
+ responses = await AsyncFetcher.fetch_all(
+ urls, headers=headers, proxy=self.proxy
+ )
for response in responses:
self.total_results += response
diff --git a/theHarvester/discovery/bevigil.py b/theHarvester/discovery/bevigil.py
index 757aed62..e21b5aeb 100644
--- a/theHarvester/discovery/bevigil.py
+++ b/theHarvester/discovery/bevigil.py
@@ -1,10 +1,10 @@
+from typing import Set
+
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import *
-from typing import Set
class SearchBeVigil:
-
def __init__(self, word) -> None:
self.word = word
self.totalhosts: Set = set()
@@ -12,20 +12,24 @@ class SearchBeVigil:
self.key = Core.bevigil_key()
if self.key is None:
self.key = ""
- raise MissingKey('bevigil')
+ raise MissingKey("bevigil")
self.proxy = False
async def do_search(self) -> None:
subdomain_endpoint = f"https://osint.bevigil.com/api/{self.word}/subdomains/"
url_endpoint = f"https://osint.bevigil.com/api/{self.word}/urls/"
- headers = {'X-Access-Token': self.key}
+ headers = {"X-Access-Token": self.key}
- responses = await AsyncFetcher.fetch_all([subdomain_endpoint], json=True, proxy=self.proxy, headers=headers)
+ responses = await AsyncFetcher.fetch_all(
+ [subdomain_endpoint], json=True, proxy=self.proxy, headers=headers
+ )
response = responses[0]
for subdomain in response["subdomains"]:
self.totalhosts.add(subdomain)
- responses = await AsyncFetcher.fetch_all([url_endpoint], json=True, proxy=self.proxy, headers=headers)
+ responses = await AsyncFetcher.fetch_all(
+ [url_endpoint], json=True, proxy=self.proxy, headers=headers
+ )
response = responses[0]
for url in response["urls"]:
self.interestingurls.add(url)
diff --git a/theHarvester/discovery/binaryedgesearch.py b/theHarvester/discovery/binaryedgesearch.py
index 036f9e36..95c4815d 100644
--- a/theHarvester/discovery/binaryedgesearch.py
+++ b/theHarvester/discovery/binaryedgesearch.py
@@ -1,10 +1,10 @@
-from theHarvester.discovery.constants import *
-from typing import Set
import asyncio
+from typing import Set
+
+from theHarvester.discovery.constants import *
class SearchBinaryEdge:
-
def __init__(self, word, limit) -> None:
self.word = word
self.totalhosts: Set = set()
@@ -13,24 +13,29 @@ class SearchBinaryEdge:
self.limit = 501 if limit >= 501 else limit
self.limit = 2 if self.limit == 1 else self.limit
if self.key is None:
- raise MissingKey('binaryedge')
+ raise MissingKey("binaryedge")
async def do_search(self) -> None:
- base_url = f'https://api.binaryedge.io/v2/query/domains/subdomain/{self.word}'
- headers = {'X-KEY': self.key, 'User-Agent': Core.get_user_agent()}
+ base_url = f"https://api.binaryedge.io/v2/query/domains/subdomain/{self.word}"
+ headers = {"X-KEY": self.key, "User-Agent": Core.get_user_agent()}
for page in range(1, self.limit):
- params = {'page': page}
- response = await AsyncFetcher.fetch_all([base_url], json=True, proxy=self.proxy, params=params, headers=headers)
+ params = {"page": page}
+ response = await AsyncFetcher.fetch_all(
+ [base_url], json=True, proxy=self.proxy, params=params, headers=headers
+ )
responses = response[0]
dct = responses
- if ('status' in dct.keys() and 'message' in dct.keys()) and \
- (dct['status'] == 400 or 'Bad Parameter' in dct['message'] or 'Error' in dct['message']):
+ if ("status" in dct.keys() and "message" in dct.keys()) and (
+ dct["status"] == 400
+ or "Bad Parameter" in dct["message"]
+ or "Error" in dct["message"]
+ ):
# 400 status code means no more results
break
- if 'events' in dct.keys():
- if len(dct['events']) == 0:
+ if "events" in dct.keys():
+ if len(dct["events"]) == 0:
break
- self.totalhosts.update({host for host in dct['events']})
+ self.totalhosts.update({host for host in dct["events"]})
await asyncio.sleep(get_delay())
async def get_hostnames(self) -> set:
diff --git a/theHarvester/discovery/bravesearch.py b/theHarvester/discovery/bravesearch.py
index 5f191d41..e65b5969 100644
--- a/theHarvester/discovery/bravesearch.py
+++ b/theHarvester/discovery/bravesearch.py
@@ -1,37 +1,44 @@
+import asyncio
+
from theHarvester.discovery.constants import *
from theHarvester.parsers import myparser
-import asyncio
class SearchBrave:
-
def __init__(self, word, limit):
self.word = word
self.results = ""
self.totalresults = ""
- self.server = 'https://search.brave.com/search?q='
+ self.server = "https://search.brave.com/search?q="
self.limit = limit
self.proxy = False
async def do_search(self):
- headers = {'User-Agent': Core.get_user_agent()}
- for query in [f'"{self.word}"', f'site:{self.word}']:
+ headers = {"User-Agent": Core.get_user_agent()}
+ for query in [f'"{self.word}"', f"site:{self.word}"]:
try:
for offset in range(0, 50):
# To reduce total number of requests only two queries are made "self.word" and site:self.word
- current_url = f'{self.server}{query}&offset={offset}&source=web&show_local=0&spellcheck=0'
- resp = await AsyncFetcher.fetch_all([current_url], headers=headers, proxy=self.proxy)
+ current_url = f"{self.server}{query}&offset={offset}&source=web&show_local=0&spellcheck=0"
+ resp = await AsyncFetcher.fetch_all(
+ [current_url], headers=headers, proxy=self.proxy
+ )
self.results = resp[0]
self.totalresults += self.results
# if 'Results from Microsoft Bing.' in resp[0] \
- if 'Not many great matches came back for your search' in resp[0] \
- or 'Your request has been flagged as being suspicious and Brave Search' in resp[0] \
- or 'Prove' in resp[0] and 'robot' in resp[0] or 'Robot' in resp[0]:
+ if (
+ "Not many great matches came back for your search" in resp[0]
+ or "Your request has been flagged as being suspicious and Brave Search"
+ in resp[0]
+ or "Prove" in resp[0]
+ and "robot" in resp[0]
+ or "Robot" in resp[0]
+ ):
await asyncio.sleep(get_delay() + 80)
break
await asyncio.sleep(get_delay() + 10)
except Exception as e:
- print(f'An exception has occurred in bravesearch: {e}')
+ print(f"An exception has occurred in bravesearch: {e}")
await asyncio.sleep(get_delay() + 80)
continue
diff --git a/theHarvester/discovery/bufferoverun.py b/theHarvester/discovery/bufferoverun.py
index 84256d8c..a953de2d 100644
--- a/theHarvester/discovery/bufferoverun.py
+++ b/theHarvester/discovery/bufferoverun.py
@@ -1,8 +1,9 @@
import re
-from theHarvester.lib.core import *
-from theHarvester.discovery.constants import MissingKey
from typing import Set
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import *
+
class SearchBufferover:
def __init__(self, word) -> None:
@@ -11,20 +12,32 @@ class SearchBufferover:
self.totalips: Set = set()
self.key = Core.bufferoverun_key()
if self.key is None:
- raise MissingKey('bufferoverun')
+ raise MissingKey("bufferoverun")
self.proxy = False
async def do_search(self) -> None:
- url = f'https://tls.bufferover.run/dns?q={self.word}'
- response = await AsyncFetcher.fetch_all([url], json=True, headers={'User-Agent': Core.get_user_agent(),
- 'x-api-key': f'{self.key}'}, proxy=self.proxy)
+ url = f"https://tls.bufferover.run/dns?q={self.word}"
+ response = await AsyncFetcher.fetch_all(
+ [url],
+ json=True,
+ headers={"User-Agent": Core.get_user_agent(), "x-api-key": f"{self.key}"},
+ proxy=self.proxy,
+ )
dct = response[0]
- if dct['Results']:
+ if dct["Results"]:
self.totalhosts = {
- host.split(',')if ',' in host and self.word.replace('www.', '') in host.split(',')[0] in host else
- host.split(',')[4] for host in dct['Results']}
+ host.split(",")
+ if "," in host
+ and self.word.replace("www.", "") in host.split(",")[0] in host
+ else host.split(",")[4]
+ for host in dct["Results"]
+ }
- self.totalips = {ip.split(',')[0] for ip in dct['Results'] if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip.split(',')[0])}
+ self.totalips = {
+ ip.split(",")[0]
+ for ip in dct["Results"]
+ if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip.split(",")[0])
+ }
async def get_hostnames(self) -> set:
return self.totalhosts
diff --git a/theHarvester/discovery/certspottersearch.py b/theHarvester/discovery/certspottersearch.py
index b4efe40d..93619da8 100644
--- a/theHarvester/discovery/certspottersearch.py
+++ b/theHarvester/discovery/certspottersearch.py
@@ -1,28 +1,30 @@
-from theHarvester.lib.core import *
from typing import Set
+from theHarvester.lib.core import *
+
class SearchCertspoter:
-
def __init__(self, word) -> None:
self.word = word
self.totalhosts: Set = set()
self.proxy = False
async def do_search(self) -> None:
- base_url = f'https://api.certspotter.com/v1/issuances?domain={self.word}&expand=dns_names'
+ base_url = f"https://api.certspotter.com/v1/issuances?domain={self.word}&expand=dns_names"
try:
- response = await AsyncFetcher.fetch_all([base_url], json=True, proxy=self.proxy)
+ response = await AsyncFetcher.fetch_all(
+ [base_url], json=True, proxy=self.proxy
+ )
response = response[0]
if isinstance(response, list):
for dct in response:
for key, value in dct.items():
- if key == 'dns_names':
+ if key == "dns_names":
self.totalhosts.update({name for name in value if name})
elif isinstance(response, dict):
- self.totalhosts.update({response['dns_names'] if 'dns_names' in response.keys() else ''}) # type: ignore
+ self.totalhosts.update({response["dns_names"] if "dns_names" in response.keys() else ""}) # type: ignore
else:
- self.totalhosts.update({''})
+ self.totalhosts.update({""})
except Exception as e:
print(e)
@@ -32,4 +34,4 @@ class SearchCertspoter:
async def process(self, proxy: bool = False) -> None:
self.proxy = proxy
await self.do_search()
- print('\tSearching results.')
+ print("\tSearching results.")
diff --git a/theHarvester/discovery/criminalip.py b/theHarvester/discovery/criminalip.py
index 65230f97..be5af679 100644
--- a/theHarvester/discovery/criminalip.py
+++ b/theHarvester/discovery/criminalip.py
@@ -1,9 +1,11 @@
-from theHarvester.lib.core import *
-from theHarvester.discovery.constants import MissingKey, get_delay
from typing import Set
-import ujson
from urllib.parse import urlparse
+import ujson
+
+from theHarvester.discovery.constants import MissingKey, get_delay
+from theHarvester.lib.core import *
+
class SearchCriminalIP:
def __init__(self, word) -> None:
@@ -13,50 +15,64 @@ class SearchCriminalIP:
self.asns: Set = set()
self.key = Core.criminalip_key()
if self.key is None:
- raise MissingKey('criminalip')
+ raise MissingKey("criminalip")
self.proxy = False
async def do_search(self) -> None:
# https://www.criminalip.io/developer/api/post-domain-scan
# https://www.criminalip.io/developer/api/get-domain-status-id
# https://www.criminalip.io/developer/api/get-domain-report-id
- url = 'https://api.criminalip.io/v1/domain/scan'
+ url = "https://api.criminalip.io/v1/domain/scan"
data = f'{{"query": "{self.word}"}}'
# print(f'Current key: {self.key}')
user_agent = Core.get_user_agent()
- response = await AsyncFetcher.post_fetch(url, json=True, headers={'User-Agent': user_agent,
- 'x-api-key': f"{self.key}"}, data=data,
- proxy=self.proxy)
+ response = await AsyncFetcher.post_fetch(
+ url,
+ json=True,
+ headers={"User-Agent": user_agent, "x-api-key": f"{self.key}"},
+ data=data,
+ proxy=self.proxy,
+ )
# print(f'My response: {response}')
# Expected response format:
# {'data': {'scan_id': scan_id}, 'message': 'api success', 'status': 200}
- if 'status' in response.keys():
- status = response['status']
+ if "status" in response.keys():
+ status = response["status"]
if status != 200:
- print(f'An error has occurred searching criminalip dumping response: {response}')
+ print(
+ f"An error has occurred searching criminalip dumping response: {response}"
+ )
else:
- scan_id = response['data']['scan_id']
+ scan_id = response["data"]["scan_id"]
scan_percentage = 0
counter = 0
while scan_percentage != 100:
status_url = f"https://api.criminalip.io/v1/domain/status/{scan_id}"
- status_response = await AsyncFetcher.fetch_all([status_url], json=True,
- headers={'User-Agent': user_agent,
- 'x-api-key': f"{self.key}"},
- proxy=self.proxy)
+ status_response = await AsyncFetcher.fetch_all(
+ [status_url],
+ json=True,
+ headers={"User-Agent": user_agent, "x-api-key": f"{self.key}"},
+ proxy=self.proxy,
+ )
status = status_response[0]
# print(f'Status response: {status}')
# Expected format:
# {"data": {"scan_percentage": 100}, "message": "api success", "status": 200}
- scan_percentage = status['data']['scan_percentage']
+ scan_percentage = status["data"]["scan_percentage"]
if scan_percentage == 100:
break
if scan_percentage == -2:
- print(f'CriminalIP failed to scan: {self.word} does not exist, verify manually')
- print(f'Dumping data: scan_response: {response} status_response: {status}')
+ print(
+ f"CriminalIP failed to scan: {self.word} does not exist, verify manually"
+ )
+ print(
+ f"Dumping data: scan_response: {response} status_response: {status}"
+ )
return
if scan_percentage == -1:
- print(f'CriminalIP scan failed dumping data: scan_response: {response} status_response: {status}')
+ print(
+ f"CriminalIP scan failed dumping data: scan_response: {response} status_response: {status}"
+ )
return
# Wait for scan to finish
if counter >= 5:
@@ -65,127 +81,146 @@ class SearchCriminalIP:
await asyncio.sleep(10 * get_delay())
counter += 1
if counter == 10:
- print('Ten iterations have occurred in CriminalIP waiting for scan to finish, returning to prevent infinite loop.')
- print(f'Verify results manually on CriminalIP dumping data: scan_response: {response} status_response: {status}')
+ print(
+ "Ten iterations have occurred in CriminalIP waiting for scan to finish, returning to prevent infinite loop."
+ )
+ print(
+ f"Verify results manually on CriminalIP dumping data: scan_response: {response} status_response: {status}"
+ )
return
report_url = f"https://api.criminalip.io/v1/domain/report/{scan_id}"
- scan_response = await AsyncFetcher.fetch_all([report_url], json=True, headers={'User-Agent': user_agent,
- 'x-api-key': f"{self.key}"},
- proxy=self.proxy)
+ scan_response = await AsyncFetcher.fetch_all(
+ [report_url],
+ json=True,
+ headers={"User-Agent": user_agent, "x-api-key": f"{self.key}"},
+ proxy=self.proxy,
+ )
scan = scan_response[0]
# json_formatted_str = json.dumps(scan, indent=2)
# print(json_formatted_str)
try:
await self.parser(scan)
except Exception as e:
- print(f'An exception occurred while parsing criminalip result: {e}')
- print('Dumping json: ')
+ print(f"An exception occurred while parsing criminalip result: {e}")
+ print("Dumping json: ")
print(scan)
async def parser(self, jlines):
# TODO when new scope field is added to parse lines for potential new scope!
# TODO map as_name to asn for asn data
# TODO determine if worth storing interesting urls
- if 'data' not in jlines.keys():
- print(f'Error with criminalip data, dumping: {jlines}')
+ if "data" not in jlines.keys():
+ print(f"Error with criminalip data, dumping: {jlines}")
return
- data = jlines['data']
- for cert in data['certificates']:
+ data = jlines["data"]
+ for cert in data["certificates"]:
# print(f'Current cert: {cert}')
- if cert['subject'].endswith('.' + self.word):
- self.totalhosts.add(cert['subject'])
+ if cert["subject"].endswith("." + self.word):
+ self.totalhosts.add(cert["subject"])
- for connected_domain in data['connected_domain_subdomain']:
+ for connected_domain in data["connected_domain_subdomain"]:
try:
- main_domain = connected_domain['main_domain']['domain']
- subdomains = [sub['domain'] for sub in connected_domain['subdomains']]
- if main_domain.endswith('.' + self.word):
+ main_domain = connected_domain["main_domain"]["domain"]
+ subdomains = [sub["domain"] for sub in connected_domain["subdomains"]]
+ if main_domain.endswith("." + self.word):
self.totalhosts.add(main_domain)
for sub in subdomains:
# print(f'Current sub: {sub}')
- if sub.endswith('.' + self.word):
+ if sub.endswith("." + self.word):
self.totalhosts.add(sub)
except Exception as e:
- print(f'An exception has occurred: {e}')
- print(f'Main line: {connected_domain}')
+ print(f"An exception has occurred: {e}")
+ print(f"Main line: {connected_domain}")
- for ip_info in data['connected_ip_info']:
- self.asns.add(str(ip_info['asn']))
- domains = [sub['domain'] for sub in ip_info['domain_list']]
+ for ip_info in data["connected_ip_info"]:
+ self.asns.add(str(ip_info["asn"]))
+ domains = [sub["domain"] for sub in ip_info["domain_list"]]
for sub in domains:
- if sub.endswith('.' + self.word):
+ if sub.endswith("." + self.word):
self.totalhosts.add(sub)
- self.totalips.add(ip_info['ip'])
+ self.totalips.add(ip_info["ip"])
- for cookie in data['cookies']:
- if cookie['domain'] != '.' + self.word and cookie['domain'].endswith('.' + self.word):
- self.totalhosts.add(cookie['domain'])
+ for cookie in data["cookies"]:
+ if cookie["domain"] != "." + self.word and cookie["domain"].endswith(
+ "." + self.word
+ ):
+ self.totalhosts.add(cookie["domain"])
- for country in data['country']:
- if country['domain'].endswith('.' + self.word):
- self.totalhosts.add(country['domain'])
- for ip in country['mapped_ips']:
- self.totalips.add(ip['ip'])
+ for country in data["country"]:
+ if country["domain"].endswith("." + self.word):
+ self.totalhosts.add(country["domain"])
+ for ip in country["mapped_ips"]:
+ self.totalips.add(ip["ip"])
- for k, v in data['dns_record'].items():
- if k == 'dns_record_type_a':
- for ip in data['dns_record'][k]['ipv4']:
- self.totalips.add(ip['ip'])
+ for k, v in data["dns_record"].items():
+ if k == "dns_record_type_a":
+ for ip in data["dns_record"][k]["ipv4"]:
+ self.totalips.add(ip["ip"])
else:
if isinstance(v, list):
for item in v:
if isinstance(item, list):
for subitem in item:
- if subitem.endswith('.' + self.word):
+ if subitem.endswith("." + self.word):
self.totalhosts.add(subitem)
else:
- if item.endswith('.' + self.word):
+ if item.endswith("." + self.word):
self.totalhosts.add(item)
- for domain_list in data['domain_list']:
- self.asns.add(str(domain_list['asn']))
- domains = [sub['domain'] for sub in domain_list['domain_list']]
+ for domain_list in data["domain_list"]:
+ self.asns.add(str(domain_list["asn"]))
+ domains = [sub["domain"] for sub in domain_list["domain_list"]]
for sub in domains:
- if sub.endswith('.' + self.word):
+ if sub.endswith("." + self.word):
self.totalhosts.add(sub)
- self.totalips.add(domain_list['ip'])
+ self.totalips.add(domain_list["ip"])
- for html_page_links in data['html_page_link_domains']:
- domain = html_page_links['domain']
- if domain.endswith('.' + self.word):
+ for html_page_links in data["html_page_link_domains"]:
+ domain = html_page_links["domain"]
+ if domain.endswith("." + self.word):
self.totalhosts.add(domain)
- for ip in html_page_links['mapped_ips']:
- self.totalips.add(ip['ip'])
+ for ip in html_page_links["mapped_ips"]:
+ self.totalips.add(ip["ip"])
# TODO combine data['links'] and data['network_logs'] urls into one list for one run through
- for link in data['links']:
- url = link['url']
+ for link in data["links"]:
+ url = link["url"]
parsed_url = urlparse(url)
netloc = parsed_url.netloc
if self.word in netloc:
- if (':' in netloc and netloc.split(':')[0].endswith(self.word)) or netloc.endswith(self.word):
+ if (
+ ":" in netloc and netloc.split(":")[0].endswith(self.word)
+ ) or netloc.endswith(self.word):
self.totalhosts.add(netloc)
- for log in data['network_logs']:
- url = log['url']
+ for log in data["network_logs"]:
+ url = log["url"]
parsed_url = urlparse(url)
netloc = parsed_url.netloc
if self.word in netloc:
- if (':' in netloc and netloc.split(':')[0].endswith(self.word)) or netloc.endswith(self.word):
+ if (
+ ":" in netloc and netloc.split(":")[0].endswith(self.word)
+ ) or netloc.endswith(self.word):
self.totalhosts.add(netloc)
- self.asns.add(str(log['as_number']))
+ self.asns.add(str(log["as_number"]))
- for redirects in data['page_redirections']:
+ for redirects in data["page_redirections"]:
for redirect in redirects:
- url = redirect['url']
+ url = redirect["url"]
parsed_url = urlparse(url)
netloc = parsed_url.netloc
if self.word in netloc:
- if (':' in netloc and netloc.split(':')[0].endswith(self.word)) or netloc.endswith(self.word):
+ if (
+ ":" in netloc and netloc.split(":")[0].endswith(self.word)
+ ) or netloc.endswith(self.word):
self.totalhosts.add(netloc)
- self.totalhosts = {host.replace('www.', '') for host in self.totalhosts if '*.' + self.word != host}
+ self.totalhosts = {
+ host.replace("www.", "")
+ for host in self.totalhosts
+ if "*." + self.word != host
+ }
# print(f'hostnames: {self.totalhosts}')
# print(f'asns: {self.asns}')
diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py
index c5b83a50..8322f5b7 100644
--- a/theHarvester/discovery/crtsh.py
+++ b/theHarvester/discovery/crtsh.py
@@ -1,9 +1,9 @@
-from theHarvester.lib.core import *
from typing import List, Set
+from theHarvester.lib.core import *
+
class SearchCrtsh:
-
def __init__(self, word) -> None:
self.word = word
self.data: List = []
@@ -12,13 +12,22 @@ class SearchCrtsh:
async def do_search(self) -> List:
data: Set = set()
try:
- url = f'https://crt.sh/?q=%25.{self.word}&output=json'
+ url = f"https://crt.sh/?q=%25.{self.word}&output=json"
response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy)
response = response[0]
data = set(
- [dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value']
- for dct in response])
- data = {domain for domain in data if (domain[0] != '*' and str(domain[0:4]).isnumeric() is False)}
+ [
+ dct["name_value"][2:]
+ if "*." == dct["name_value"][:2]
+ else dct["name_value"]
+ for dct in response
+ ]
+ )
+ data = {
+ domain
+ for domain in data
+ if (domain[0] != "*" and str(domain[0:4]).isnumeric() is False)
+ }
except Exception as e:
print(e)
clean: List = []
diff --git a/theHarvester/discovery/dnssearch.py b/theHarvester/discovery/dnssearch.py
index aca06c58..6db6f8fb 100644
--- a/theHarvester/discovery/dnssearch.py
+++ b/theHarvester/discovery/dnssearch.py
@@ -8,15 +8,15 @@ DNS Browsing
Explore the space around known hosts & ips for extra catches.
"""
+import asyncio
import re
import sys
-
-import asyncio
-from aiodns import DNSResolver
from ipaddress import IPv4Network
from typing import Callable, List, Optional
-from theHarvester.lib import hostchecker
+from aiodns import DNSResolver
+
+from theHarvester.lib import hostchecker
#####################################################################
# DNS FORCE
@@ -24,7 +24,6 @@ from theHarvester.lib import hostchecker
class DnsForce:
-
def __init__(self, domain, dnsserver, verbose: bool = False) -> None:
self.domain = domain
self.subdo = False
@@ -33,20 +32,22 @@ class DnsForce:
# self.dnsserver = list(map(str, dnsserver.split(','))) if isinstance(dnsserver, str) else dnsserver
self.dnsserver = dnsserver
try:
- with open('/etc/theHarvester/wordlists/dns-names.txt', 'r') as file:
+ with open("/etc/theHarvester/wordlists/dns-names.txt", "r") as file:
self.list = file.readlines()
except FileNotFoundError:
try:
- with open('/usr/local/etc/theHarvester/wordlists/dns-names.txt', 'r') as file:
+ with open(
+ "/usr/local/etc/theHarvester/wordlists/dns-names.txt", "r"
+ ) as file:
self.list = file.readlines()
except FileNotFoundError:
- with open('wordlists/dns-names.txt', 'r') as file:
+ with open("wordlists/dns-names.txt", "r") as file:
self.list = file.readlines()
- self.domain = domain.replace('www.', '')
- self.list = [f'{word.strip()}.{self.domain}' for word in self.list]
+ self.domain = domain.replace("www.", "")
+ self.list = [f"{word.strip()}.{self.domain}" for word in self.list]
async def run(self):
- print(f'Starting DNS brute forcing with {len(self.list)} words')
+ print(f"Starting DNS brute forcing with {len(self.list)} words")
checker = hostchecker.Checker(self.list, nameserver=self.dnsserver)
resolved_pair, hosts, ips = await checker.check()
return resolved_pair, hosts, ips
@@ -57,16 +58,15 @@ class DnsForce:
#####################################################################
-IP_REGEX = r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}'
-PORT_REGEX = r'\d{1,5}'
-NETMASK_REGEX: str = r'\d{1,2}|' + IP_REGEX
-NETWORK_REGEX: str = r'\b({})(?:\:({}))?(?:\/({}))?\b'.format(
- IP_REGEX,
- PORT_REGEX,
- NETMASK_REGEX)
+IP_REGEX = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}"
+PORT_REGEX = r"\d{1,5}"
+NETMASK_REGEX: str = r"\d{1,2}|" + IP_REGEX
+NETWORK_REGEX: str = r"\b({})(?:\:({}))?(?:\/({}))?\b".format(
+ IP_REGEX, PORT_REGEX, NETMASK_REGEX
+)
-def serialize_ip_range(ip: str, netmask: str = '24') -> str:
+def serialize_ip_range(ip: str, netmask: str = "24") -> str:
"""
Serialize a network range in a constant format, 'x.x.x.x/y'.
@@ -89,12 +89,12 @@ def serialize_ip_range(ip: str, netmask: str = '24') -> str:
__ip = __ip_matches.group(1)
__netmask = netmask if netmask else __ip_matches.group(3)
if __ip and __netmask:
- return str(IPv4Network('{}/{}'.format(__ip, __netmask), strict=False))
+ return str(IPv4Network("{}/{}".format(__ip, __netmask), strict=False))
elif __ip:
- return str(IPv4Network('{}/{}'.format(__ip, '24'), strict=False))
+ return str(IPv4Network("{}/{}".format(__ip, "24"), strict=False))
# invalid input ip
- return ''
+ return ""
def list_ips_in_network_range(iprange: str) -> List[str]:
@@ -133,12 +133,14 @@ async def reverse_single_ip(ip: str, resolver: DNSResolver) -> str:
"""
try:
__host = await resolver.gethostbyaddr(ip)
- return __host.name if __host else ''
+ return __host.name if __host else ""
except Exception:
- return ''
+ return ""
-async def reverse_all_ips_in_range(iprange: str, callback: Callable, nameservers: Optional[List[str]] = None) -> None:
+async def reverse_all_ips_in_range(
+ iprange: str, callback: Callable, nameservers: Optional[List[str]] = None
+) -> None:
"""
Reverse all the IPs stored in a network range.
All the queries are made concurrently.
@@ -185,8 +187,8 @@ def log_query(ip: str) -> None:
-------
out: None.
"""
- sys.stdout.write(chr(27) + '[2K' + chr(27) + '[G')
- sys.stdout.write('\r' + ip + ' - ')
+ sys.stdout.write(chr(27) + "[2K" + chr(27) + "[G")
+ sys.stdout.write("\r" + ip + " - ")
sys.stdout.flush()
diff --git a/theHarvester/discovery/duckduckgosearch.py b/theHarvester/discovery/duckduckgosearch.py
index 3539fca3..49ab9e1c 100644
--- a/theHarvester/discovery/duckduckgosearch.py
+++ b/theHarvester/discovery/duckduckgosearch.py
@@ -1,34 +1,36 @@
+import ujson
+
from theHarvester.discovery.constants import *
from theHarvester.lib.core import *
from theHarvester.parsers import myparser
-import ujson
class SearchDuckDuckGo:
-
def __init__(self, word, limit) -> None:
self.word = word
self.results = ""
self.totalresults = ""
self.dorks: List = []
self.links: List = []
- self.database = 'https://duckduckgo.com/?q='
- self.api = 'https://api.duckduckgo.com/?q=x&format=json&pretty=1' # Currently using API.
- self.quantity = '100'
+ self.database = "https://duckduckgo.com/?q="
+ self.api = "https://api.duckduckgo.com/?q=x&format=json&pretty=1" # Currently using API.
+ self.quantity = "100"
self.limit = limit
self.proxy = False
async def do_search(self) -> None:
# Do normal scraping.
- url = self.api.replace('x', self.word)
- headers = {'User-Agent': Core.get_user_agent()}
- first_resp = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy)
+ url = self.api.replace("x", self.word)
+ headers = {"User-Agent": Core.get_user_agent()}
+ first_resp = await AsyncFetcher.fetch_all(
+ [url], headers=headers, proxy=self.proxy
+ )
self.results = first_resp[0]
self.totalresults += self.results
urls = await self.crawl(self.results)
urls = {url for url in urls if len(url) > 5}
all_resps = await AsyncFetcher.fetch_all(urls)
- self.totalresults += ''.join(all_resps)
+ self.totalresults += "".join(all_resps)
async def crawl(self, text):
"""
@@ -53,27 +55,39 @@ class SearchDuckDuckGo:
if isinstance(val, dict): # Validation check.
for key in val.keys():
value = val.get(key)
- if isinstance(value, str) and value != '' and 'https://' in value or 'http://' in value:
+ if (
+ isinstance(value, str)
+ and value != ""
+ and "https://" in value
+ or "http://" in value
+ ):
urls.add(value)
- if isinstance(val, str) and val != '' and 'https://' in val or 'http://' in val:
+ if (
+ isinstance(val, str)
+ and val != ""
+ and "https://" in val
+ or "http://" in val
+ ):
urls.add(val)
tmp = set()
for url in urls:
- if '<' in url and 'href=' in url: # Format is
- equal_index = url.index('=')
- true_url = ''
- for ch in url[equal_index + 1:]:
+ if (
+ "<" in url and "href=" in url
+ ): # Format is
+ equal_index = url.index("=")
+ true_url = ""
+ for ch in url[equal_index + 1 :]:
if ch == '"':
tmp.add(true_url)
break
true_url += ch
else:
- if url != '':
+ if url != "":
tmp.add(url)
return tmp
except Exception as e:
- print(f'Exception occurred: {e}')
+ print(f"Exception occurred: {e}")
return []
async def get_emails(self):
diff --git a/theHarvester/discovery/fullhuntsearch.py b/theHarvester/discovery/fullhuntsearch.py
index 911a5ac0..610f59c0 100644
--- a/theHarvester/discovery/fullhuntsearch.py
+++ b/theHarvester/discovery/fullhuntsearch.py
@@ -3,21 +3,23 @@ from theHarvester.lib.core import *
class SearchFullHunt:
-
def __init__(self, word) -> None:
self.word = word
self.key = Core.fullhunt_key()
if self.key is None:
- raise MissingKey('fullhunt')
+ raise MissingKey("fullhunt")
self.total_results = None
self.proxy = False
async def do_search(self) -> None:
- url = f'https://fullhunt.io/api/v1/domain/{self.word}/subdomains'
- response = await AsyncFetcher.fetch_all([url], json=True, headers={'User-Agent': Core.get_user_agent(),
- 'X-API-KEY': self.key},
- proxy=self.proxy)
- self.total_results = response[0]['hosts']
+ url = f"https://fullhunt.io/api/v1/domain/{self.word}/subdomains"
+ response = await AsyncFetcher.fetch_all(
+ [url],
+ json=True,
+ headers={"User-Agent": Core.get_user_agent(), "X-API-KEY": self.key},
+ proxy=self.proxy,
+ )
+ self.total_results = response[0]["hosts"]
async def get_hostnames(self):
return self.total_results
diff --git a/theHarvester/discovery/hackertarget.py b/theHarvester/discovery/hackertarget.py
index e9971858..96ae45b1 100644
--- a/theHarvester/discovery/hackertarget.py
+++ b/theHarvester/discovery/hackertarget.py
@@ -9,14 +9,19 @@ class SearchHackerTarget:
def __init__(self, word) -> None:
self.word = word
self.total_results = ""
- self.hostname = 'https://api.hackertarget.com'
+ self.hostname = "https://api.hackertarget.com"
self.proxy = False
self.results = None
async def do_search(self) -> None:
- headers = {'User-agent': Core.get_user_agent()}
- urls = [f'{self.hostname}/hostsearch/?q={self.word}', f'{self.hostname}/reversedns/?q={self.word}']
- responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy)
+ headers = {"User-agent": Core.get_user_agent()}
+ urls = [
+ f"{self.hostname}/hostsearch/?q={self.word}",
+ f"{self.hostname}/reversedns/?q={self.word}",
+ ]
+ responses = await AsyncFetcher.fetch_all(
+ urls, headers=headers, proxy=self.proxy
+ )
for response in responses:
self.total_results += response.replace(",", ":")
@@ -25,4 +30,8 @@ class SearchHackerTarget:
await self.do_search()
async def get_hostnames(self) -> list:
- return [result for result in self.total_results.splitlines() if 'No PTR records found' not in result]
+ return [
+ result
+ for result in self.total_results.splitlines()
+ if "No PTR records found" not in result
+ ]
diff --git a/theHarvester/discovery/huntersearch.py b/theHarvester/discovery/huntersearch.py
index bae95bde..46027bc9 100644
--- a/theHarvester/discovery/huntersearch.py
+++ b/theHarvester/discovery/huntersearch.py
@@ -1,10 +1,10 @@
+from typing import List
+
from theHarvester.discovery.constants import *
from theHarvester.lib.core import *
-from typing import List
class SearchHunter:
-
def __init__(self, word, limit, start) -> None:
self.word = word
self.limit = limit
@@ -12,10 +12,10 @@ class SearchHunter:
self.start = start
self.key = Core.hunter_key()
if self.key is None:
- raise MissingKey('Hunter')
+ raise MissingKey("Hunter")
self.total_results = ""
self.counter = start
- self.database = f'https://api.hunter.io/v2/domain-search?domain={self.word}&api_key={self.key}&limit=10'
+ self.database = f"https://api.hunter.io/v2/domain-search?domain={self.word}&api_key={self.key}&limit=10"
self.proxy = False
self.hostnames: List = []
self.emails: List = []
@@ -23,48 +23,79 @@ class SearchHunter:
async def do_search(self) -> None:
# First determine if a user account is not a free account, this call is free
is_free = True
- headers = {'User-Agent': Core.get_user_agent()}
- acc_info_url = f'https://api.hunter.io/v2/account?api_key={self.key}'
- response = await AsyncFetcher.fetch_all([acc_info_url], headers=headers, json=True)
- is_free = is_free if 'plan_name' in response[0]['data'].keys() and response[0]['data']['plan_name'].lower() \
- == 'free' else False
+ headers = {"User-Agent": Core.get_user_agent()}
+ acc_info_url = f"https://api.hunter.io/v2/account?api_key={self.key}"
+ response = await AsyncFetcher.fetch_all(
+ [acc_info_url], headers=headers, json=True
+ )
+ is_free = (
+ is_free
+ if "plan_name" in response[0]["data"].keys()
+ and response[0]["data"]["plan_name"].lower() == "free"
+ else False
+ )
# Extract the total number of requests that are available for an account
- total_requests_avail = response[0]['data']['requests']['searches']['available'] - response[0]['data']['requests']['searches']['used']
+ total_requests_avail = (
+ response[0]["data"]["requests"]["searches"]["available"]
+ - response[0]["data"]["requests"]["searches"]["used"]
+ )
if is_free:
- response = await AsyncFetcher.fetch_all([self.database], headers=headers, proxy=self.proxy, json=True)
+ response = await AsyncFetcher.fetch_all(
+ [self.database], headers=headers, proxy=self.proxy, json=True
+ )
self.emails, self.hostnames = await self.parse_resp(json_resp=response[0])
else:
# Determine the total number of emails that are available
# As the most emails you can get within one query are 100
# This is only done where paid accounts are in play
- hunter_dinfo_url = f'https://api.hunter.io/v2/email-count?domain={self.word}'
- response = await AsyncFetcher.fetch_all([hunter_dinfo_url], headers=headers, proxy=self.proxy, json=True)
- total_number_reqs = response[0]['data']['total'] // 100
+ hunter_dinfo_url = (
+ f"https://api.hunter.io/v2/email-count?domain={self.word}"
+ )
+ response = await AsyncFetcher.fetch_all(
+ [hunter_dinfo_url], headers=headers, proxy=self.proxy, json=True
+ )
+ total_number_reqs = response[0]["data"]["total"] // 100
# Parse out meta field within initial JSON response to determine the total number of results
if total_requests_avail < total_number_reqs:
- print('WARNING: account does not have enough requests to gather all emails')
- print(f'Total requests available: {total_requests_avail}, total requests '
- f'needed to be made: {total_number_reqs}')
- print('RETURNING current results, if you would still like to '
- 'run this module comment out the if request')
+ print(
+ "WARNING: account does not have enough requests to gather all emails"
+ )
+ print(
+ f"Total requests available: {total_requests_avail}, total requests "
+ f"needed to be made: {total_number_reqs}"
+ )
+ print(
+ "RETURNING current results, if you would still like to "
+ "run this module comment out the if request"
+ )
return
self.limit = 100
# max number of emails you can get per request is 100
# increments of 100 with offset determining where to start
# See docs for more details: https://hunter.io/api-documentation/v2#domain-search
for offset in range(0, 100 * total_number_reqs, 100):
- req_url = f'https://api.hunter.io/v2/domain-search?domain={self.word}&api_key={self.key}&limit{self.limit}&offset={offset}'
- response = await AsyncFetcher.fetch_all([req_url], headers=headers, proxy=self.proxy, json=True)
+ req_url = f"https://api.hunter.io/v2/domain-search?domain={self.word}&api_key={self.key}&limit{self.limit}&offset={offset}"
+ response = await AsyncFetcher.fetch_all(
+ [req_url], headers=headers, proxy=self.proxy, json=True
+ )
temp_emails, temp_hostnames = await self.parse_resp(response[0])
self.emails.extend(temp_emails)
self.hostnames.extend(temp_hostnames)
await asyncio.sleep(1)
async def parse_resp(self, json_resp):
- emails = list(sorted({email['value'] for email in json_resp['data']['emails']}))
- domains = list(sorted({source['domain'] for email in json_resp['data']['emails'] for source in email['sources']
- if self.word in source['domain']}))
+ emails = list(sorted({email["value"] for email in json_resp["data"]["emails"]}))
+ domains = list(
+ sorted(
+ {
+ source["domain"]
+ for email in json_resp["data"]["emails"]
+ for source in email["sources"]
+ if self.word in source["domain"]
+ }
+ )
+ )
return emails, domains
async def process(self, proxy: bool = False) -> None:
diff --git a/theHarvester/discovery/netlas.py b/theHarvester/discovery/netlas.py
index 697a0269..dd6c62fe 100644
--- a/theHarvester/discovery/netlas.py
+++ b/theHarvester/discovery/netlas.py
@@ -1,25 +1,27 @@
+from typing import Set
+
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import *
-from typing import Set
class SearchNetlas:
-
def __init__(self, word) -> None:
self.word = word
self.totalhosts: List = []
self.totalips: List = []
self.key = Core.netlas_key()
if self.key is None:
- raise MissingKey('netlas')
+ raise MissingKey("netlas")
self.proxy = False
async def do_search(self) -> None:
- api = f'https://app.netlas.io/api/domains/?q=*.{self.word}&source_type=include&start=0&fields=*'
- headers = {'X-API-Key': self.key}
- response = await AsyncFetcher.fetch_all([api], json=True, headers=headers, proxy=self.proxy)
- for domain in response[0]['items']:
- self.totalhosts.append(domain['data']['domain'])
+ api = f"https://app.netlas.io/api/domains/?q=*.{self.word}&source_type=include&start=0&fields=*"
+ headers = {"X-API-Key": self.key}
+ response = await AsyncFetcher.fetch_all(
+ [api], json=True, headers=headers, proxy=self.proxy
+ )
+ for domain in response[0]["items"]:
+ self.totalhosts.append(domain["data"]["domain"])
async def get_hostnames(self) -> List:
return self.totalhosts
diff --git a/theHarvester/discovery/onyphe.py b/theHarvester/discovery/onyphe.py
index 6f93a5e7..8960becd 100644
--- a/theHarvester/discovery/onyphe.py
+++ b/theHarvester/discovery/onyphe.py
@@ -1,33 +1,37 @@
-from theHarvester.lib.core import *
-from theHarvester.discovery.constants import MissingKey
from typing import Set
from urllib.parse import urlparse
+
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import *
+
# from theHarvester.parsers import myparser
class SearchOnyphe:
def __init__(self, word) -> None:
self.word = word
- self.response = ''
+ self.response = ""
self.totalhosts: Set = set()
self.totalips: Set = set()
self.asns: Set = set()
self.key = Core.onyphe_key()
if self.key is None:
- raise MissingKey('onyphe')
+ raise MissingKey("onyphe")
self.proxy = False
async def do_search(self) -> None:
# https://www.onyphe.io/docs/apis/search
# https://www.onyphe.io/search?q=domain%3Acharter.com&captcharesponse=j5cGT
# base_url = f'https://www.onyphe.io/api/v2/search/?q=domain:domain:{self.word}'
- base_url = f'https://www.onyphe.io/api/v2/search/?q=domain:{self.word}'
+ base_url = f"https://www.onyphe.io/api/v2/search/?q=domain:{self.word}"
headers = {
- 'User-Agent': Core.get_user_agent(),
+ "User-Agent": Core.get_user_agent(),
"Content-Type": "application/json",
- "Authorization": f'bearer {self.key}',
+ "Authorization": f"bearer {self.key}",
}
- response = await AsyncFetcher.fetch_all([base_url], json=True, headers=headers, proxy=self.proxy)
+ response = await AsyncFetcher.fetch_all(
+ [base_url], json=True, headers=headers, proxy=self.proxy
+ )
self.response = response[0]
await self.parse_onyphe_resp_json()
@@ -35,41 +39,74 @@ class SearchOnyphe:
if isinstance(self.response, list):
self.response = self.response[0]
if not isinstance(self.response, dict):
- raise Exception(f'An exception has occurred {self.response} is not a dict')
- if 'Success' == self.response['text']:
- if 'results' in self.response.keys():
- for result in self.response['results']:
+ raise Exception(f"An exception has occurred {self.response} is not a dict")
+ if "Success" == self.response["text"]:
+ if "results" in self.response.keys():
+ for result in self.response["results"]:
try:
- if 'alternativeip' in result.keys():
- self.totalips.update({altip for altip in result['alternativeip']})
- if 'url' in result.keys() and isinstance(result['url'], list):
- self.totalhosts.update(urlparse(url).netloc for url in result['url']
- if urlparse(url).netloc.endswith(self.word))
- self.asns.add(result['asn'])
- self.asns.add(result['geolocus']['asn'])
- self.totalips.add(result['geolocus']['subnet'])
- self.totalips.add(result['ip'])
- self.totalips.add(result['subnet'])
+ if "alternativeip" in result.keys():
+ self.totalips.update(
+ {altip for altip in result["alternativeip"]}
+ )
+ if "url" in result.keys() and isinstance(result["url"], list):
+ self.totalhosts.update(
+ urlparse(url).netloc
+ for url in result["url"]
+ if urlparse(url).netloc.endswith(self.word)
+ )
+ self.asns.add(result["asn"])
+ self.asns.add(result["geolocus"]["asn"])
+ self.totalips.add(result["geolocus"]["subnet"])
+ self.totalips.add(result["ip"])
+ self.totalips.add(result["subnet"])
# Shouldn't be needed as API autoparses urls from html raw data
# rawres = myparser.Parser(result['data'], self.word)
# if await rawres.hostnames():
# self.totalhosts.update(set(await rawres.hostnames()))
- for subdomain_key in ["domain", "hostname", "subdomains", "subject", "reverse", "geolocus"]:
+ for subdomain_key in [
+ "domain",
+ "hostname",
+ "subdomains",
+ "subject",
+ "reverse",
+ "geolocus",
+ ]:
if subdomain_key in result.keys():
- if subdomain_key == 'subject':
- self.totalhosts.update({domain for domain in result[subdomain_key]['altname']
- if domain.endswith(self.word)})
- elif subdomain_key == 'geolocus':
- self.totalhosts.update({domain for domain in result[subdomain_key]['domain']
- if domain.endswith(self.word)})
+ if subdomain_key == "subject":
+ self.totalhosts.update(
+ {
+ domain
+ for domain in result[subdomain_key][
+ "altname"
+ ]
+ if domain.endswith(self.word)
+ }
+ )
+ elif subdomain_key == "geolocus":
+ self.totalhosts.update(
+ {
+ domain
+ for domain in result[subdomain_key][
+ "domain"
+ ]
+ if domain.endswith(self.word)
+ }
+ )
else:
- self.totalhosts.update({domain for domain in result[subdomain_key]
- if domain.endswith(self.word)})
+ self.totalhosts.update(
+ {
+ domain
+ for domain in result[subdomain_key]
+ if domain.endswith(self.word)
+ }
+ )
except Exception as e:
- print(f'An exception has occurred on result: {result}: {e}')
+ print(f"An exception has occurred on result: {result}: {e}")
continue
else:
- print(f'Onhyphe API query did not succeed dumping current response: {self.response}')
+ print(
+ f"Onhyphe API query did not succeed dumping current response: {self.response}"
+ )
async def get_asns(self) -> set:
return self.asns
diff --git a/theHarvester/discovery/otxsearch.py b/theHarvester/discovery/otxsearch.py
index db44ae24..dbe0610d 100644
--- a/theHarvester/discovery/otxsearch.py
+++ b/theHarvester/discovery/otxsearch.py
@@ -1,10 +1,10 @@
-from typing import Set
-from theHarvester.lib.core import *
import re
+from typing import Set
+
+from theHarvester.lib.core import *
class SearchOtx:
-
def __init__(self, word) -> None:
self.word = word
self.totalhosts: Set = set()
@@ -12,14 +12,17 @@ class SearchOtx:
self.proxy = False
async def do_search(self) -> None:
- url = f'https://otx.alienvault.com/api/v1/indicators/domain/{self.word}/passive_dns'
+ url = f"https://otx.alienvault.com/api/v1/indicators/domain/{self.word}/passive_dns"
response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy)
responses = response[0]
dct = responses
- self.totalhosts = {host['hostname'] for host in dct['passive_dns']}
+ self.totalhosts = {host["hostname"] for host in dct["passive_dns"]}
# filter out ips that are just called NXDOMAIN
- self.totalips = {ip['address'] for ip in dct['passive_dns'] if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$",
- ip['address'])}
+ self.totalips = {
+ ip["address"]
+ for ip in dct["passive_dns"]
+ if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip["address"])
+ }
async def get_hostnames(self) -> set:
return self.totalhosts
diff --git a/theHarvester/discovery/pentesttools.py b/theHarvester/discovery/pentesttools.py
index 700043a6..35f780c0 100644
--- a/theHarvester/discovery/pentesttools.py
+++ b/theHarvester/discovery/pentesttools.py
@@ -1,57 +1,66 @@
+import time
+from typing import List
+
+import ujson
+
from theHarvester.discovery.constants import *
from theHarvester.lib.core import *
-from typing import List
-import ujson
-import time
class SearchPentestTools:
-
def __init__(self, word) -> None:
# Script is largely based off https://pentest-tools.com/public/api_client.py.txt
self.word = word
self.key = Core.pentest_tools_key()
if self.key is None:
- raise MissingKey('PentestTools')
+ raise MissingKey("PentestTools")
self.total_results: List = []
- self.api = f'https://pentest-tools.com/api?key={self.key}'
+ self.api = f"https://pentest-tools.com/api?key={self.key}"
self.proxy = False
async def poll(self, scan_id):
while True:
time.sleep(3)
# Get the status of our scan
- scan_status_data = {
- 'op': 'get_scan_status',
- 'scan_id': scan_id
- }
- responses = await AsyncFetcher.post_fetch(url=self.api, data=ujson.dumps(scan_status_data), proxy=self.proxy)
+ scan_status_data = {"op": "get_scan_status", "scan_id": scan_id}
+ responses = await AsyncFetcher.post_fetch(
+ url=self.api, data=ujson.dumps(scan_status_data), proxy=self.proxy
+ )
res_json = ujson.loads(responses.strip())
- if res_json['op_status'] == 'success':
- if res_json['scan_status'] != 'waiting' and res_json['scan_status'] != 'running':
+ if res_json["op_status"] == "success":
+ if (
+ res_json["scan_status"] != "waiting"
+ and res_json["scan_status"] != "running"
+ ):
getoutput_data = {
- 'op': 'get_output',
- 'scan_id': scan_id,
- 'output_format': 'json'
+ "op": "get_output",
+ "scan_id": scan_id,
+ "output_format": "json",
}
- responses = await AsyncFetcher.post_fetch(url=self.api,
- data=ujson.dumps(getoutput_data),
- proxy=self.proxy)
+ responses = await AsyncFetcher.post_fetch(
+ url=self.api, data=ujson.dumps(getoutput_data), proxy=self.proxy
+ )
- res_json = ujson.loads(responses.strip('\n'))
+ res_json = ujson.loads(responses.strip("\n"))
self.total_results = await self.parse_json(res_json)
break
else:
- print(f"Operation get_scan_status failed because: {res_json['error']}. {res_json['details']}")
+ print(
+ f"Operation get_scan_status failed because: {res_json['error']}. {res_json['details']}"
+ )
break
@staticmethod
async def parse_json(json_results):
- status = json_results['op_status']
- if status == 'success':
- scan_tests = json_results['scan_output']['output_json']
- output_data = scan_tests[0]['output_data']
- host_to_ip = [f'{subdomain[0]}:{subdomain[1]}' for subdomain in output_data if len(subdomain) > 0]
+ status = json_results["op_status"]
+ if status == "success":
+ scan_tests = json_results["scan_output"]["output_json"]
+ output_data = scan_tests[0]["output_data"]
+ host_to_ip = [
+ f"{subdomain[0]}:{subdomain[1]}"
+ for subdomain in output_data
+ if len(subdomain) > 0
+ ]
return host_to_ip
return []
@@ -60,18 +69,20 @@ class SearchPentestTools:
async def do_search(self) -> None:
subdomain_payload = {
- 'op': 'start_scan',
- 'tool_id': 20,
- 'tool_params': {
- 'target': f'{self.word}',
- 'web_details': 'off',
- 'do_smart_search': 'off'
- }
+ "op": "start_scan",
+ "tool_id": 20,
+ "tool_params": {
+ "target": f"{self.word}",
+ "web_details": "off",
+ "do_smart_search": "off",
+ },
}
- responses = await AsyncFetcher.post_fetch(url=self.api, data=ujson.dumps(subdomain_payload), proxy=self.proxy)
+ responses = await AsyncFetcher.post_fetch(
+ url=self.api, data=ujson.dumps(subdomain_payload), proxy=self.proxy
+ )
res_json = ujson.loads(responses.strip())
- if res_json['op_status'] == 'success':
- scan_id = res_json['scan_id']
+ if res_json["op_status"] == "success":
+ scan_id = res_json["scan_id"]
await self.poll(scan_id)
async def process(self, proxy: bool = False) -> None:
diff --git a/theHarvester/discovery/projectdiscovery.py b/theHarvester/discovery/projectdiscovery.py
index a2a40542..dd28851c 100644
--- a/theHarvester/discovery/projectdiscovery.py
+++ b/theHarvester/discovery/projectdiscovery.py
@@ -3,21 +3,25 @@ from theHarvester.lib.core import *
class SearchDiscovery:
-
def __init__(self, word) -> None:
self.word = word
self.key = Core.projectdiscovery_key()
if self.key is None:
- raise MissingKey('ProjectDiscovery')
+ raise MissingKey("ProjectDiscovery")
self.total_results = None
self.proxy = False
async def do_search(self):
- url = f'https://dns.projectdiscovery.io/dns/{self.word}/subdomains'
- response = await AsyncFetcher.fetch_all([url], json=True, headers={'User-Agent': Core.get_user_agent(),
- 'Authorization': self.key},
- proxy=self.proxy)
- self.total_results = [f'{domains}.{self.word}' for domains in response[0]['subdomains']]
+ url = f"https://dns.projectdiscovery.io/dns/{self.word}/subdomains"
+ response = await AsyncFetcher.fetch_all(
+ [url],
+ json=True,
+ headers={"User-Agent": Core.get_user_agent(), "Authorization": self.key},
+ proxy=self.proxy,
+ )
+ self.total_results = [
+ f"{domains}.{self.word}" for domains in response[0]["subdomains"]
+ ]
async def get_hostnames(self):
return self.total_results
diff --git a/theHarvester/discovery/rapiddns.py b/theHarvester/discovery/rapiddns.py
index e2bd1a7a..1e2245af 100644
--- a/theHarvester/discovery/rapiddns.py
+++ b/theHarvester/discovery/rapiddns.py
@@ -1,9 +1,9 @@
from bs4 import BeautifulSoup
+
from theHarvester.lib.core import *
class SearchRapidDns:
-
def __init__(self, word) -> None:
self.word = word
self.total_results: List = []
@@ -11,14 +11,16 @@ class SearchRapidDns:
async def do_search(self):
try:
- headers = {'User-agent': Core.get_user_agent()}
+ headers = {"User-agent": Core.get_user_agent()}
# TODO see if it's worth adding sameip searches
# f'{self.hostname}/sameip/{self.word}?full=1#result'
- urls = [f'https://rapiddns.io/subdomain/{self.word}?full=1#result']
- responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy)
+ urls = [f"https://rapiddns.io/subdomain/{self.word}?full=1#result"]
+ responses = await AsyncFetcher.fetch_all(
+ urls, headers=headers, proxy=self.proxy
+ )
if len(responses[0]) <= 1:
return self.total_results
- soup = BeautifulSoup(responses[0], 'html.parser')
+ soup = BeautifulSoup(responses[0], "html.parser")
rows = soup.find("table").find("tbody").find_all("tr")
if rows:
# Validation check
@@ -27,13 +29,15 @@ class SearchRapidDns:
if len(cells) >= 0:
# sanity check
subdomain = str(cells[0].get_text())
- if cells[-1].get_text() == 'CNAME':
- self.total_results.append(f'{subdomain}')
+ if cells[-1].get_text() == "CNAME":
+ self.total_results.append(f"{subdomain}")
else:
- self.total_results.append(f'{subdomain}:{str(cells[1].get_text()).strip()}')
+ self.total_results.append(
+ f"{subdomain}:{str(cells[1].get_text()).strip()}"
+ )
self.total_results = list({domain for domain in self.total_results})
except Exception as e:
- print(f'An exception has occurred: {str(e)}')
+ print(f"An exception has occurred: {str(e)}")
async def process(self, proxy: bool = False) -> None:
self.proxy = proxy
diff --git a/theHarvester/discovery/rocketreach.py b/theHarvester/discovery/rocketreach.py
index 3928fc55..9f0b2ee0 100644
--- a/theHarvester/discovery/rocketreach.py
+++ b/theHarvester/discovery/rocketreach.py
@@ -1,58 +1,69 @@
+import asyncio
+from typing import Set
+
from theHarvester.discovery.constants import *
from theHarvester.lib.core import *
-from typing import Set
-import asyncio
class SearchRocketReach:
-
def __init__(self, word, limit) -> None:
self.ips: Set = set()
self.word = word
self.key = Core.rocketreach_key()
if self.key is None:
- raise MissingKey('RocketReach')
+ raise MissingKey("RocketReach")
self.hosts: Set = set()
self.proxy = False
- self.baseurl = 'https://rocketreach.co/api/v2/person/search'
+ self.baseurl = "https://rocketreach.co/api/v2/person/search"
self.links: Set = set()
self.limit = limit
async def do_search(self) -> None:
try:
headers = {
- 'Api-Key': self.key,
- 'Content-Type': 'application/json',
- 'User-Agent': Core.get_user_agent()
+ "Api-Key": self.key,
+ "Content-Type": "application/json",
+ "User-Agent": Core.get_user_agent(),
}
next_page = 1 # track pagination
for count in range(1, self.limit):
data = f'{{"query":{{"company_domain": ["{self.word}"]}}, "start": {next_page}, "page_size": 100}}'
- result = await AsyncFetcher.post_fetch(self.baseurl, headers=headers, data=data, json=True)
- if 'detail' in result.keys() and 'error' in result.keys() and 'Subscribe to a plan to access' in result['detail']:
+ result = await AsyncFetcher.post_fetch(
+ self.baseurl, headers=headers, data=data, json=True
+ )
+ if (
+ "detail" in result.keys()
+ and "error" in result.keys()
+ and "Subscribe to a plan to access" in result["detail"]
+ ):
# No more results can be fetched
break
- if 'detail' in result.keys() and 'Request was throttled.' in result['detail']:
+ if (
+ "detail" in result.keys()
+ and "Request was throttled." in result["detail"]
+ ):
# Rate limit has been triggered need to sleep extra
- print(f'RocketReach requests have been throttled; '
- f'{result["detail"].split(" ", 3)[-1].replace("available", "availability")}')
+ print(
+ f"RocketReach requests have been throttled; "
+ f'{result["detail"].split(" ", 3)[-1].replace("available", "availability")}'
+ )
break
- if 'profiles' in dict(result).keys():
- if len(result['profiles']) == 0:
+ if "profiles" in dict(result).keys():
+ if len(result["profiles"]) == 0:
break
- for profile in result['profiles']:
- if 'linkedin_url' in dict(profile).keys():
- self.links.add(profile['linkedin_url'])
- if 'pagination' in dict(result).keys():
- next_page = int(result['pagination']['next'])
- if next_page > int(result['pagination']['total']):
+ for profile in result["profiles"]:
+ if "linkedin_url" in dict(profile).keys():
+ self.links.add(profile["linkedin_url"])
+ if "pagination" in dict(result).keys():
+ next_page = int(result["pagination"]["next"])
+ if next_page > int(result["pagination"]["total"]):
break
await asyncio.sleep(get_delay() + 5)
except Exception as e:
- print(f'An exception has occurred: {e}')
+ print(f"An exception has occurred: {e}")
async def get_links(self):
return self.links
diff --git a/theHarvester/discovery/searchhunterhow.py b/theHarvester/discovery/searchhunterhow.py
index 0bfa7f7f..8618b71e 100644
--- a/theHarvester/discovery/searchhunterhow.py
+++ b/theHarvester/discovery/searchhunterhow.py
@@ -1,10 +1,12 @@
-from theHarvester.lib.core import *
-from theHarvester.discovery.constants import MissingKey
-from typing import Set
import base64
from datetime import datetime
+from typing import Set
+
from dateutil.relativedelta import relativedelta
+from theHarvester.discovery.constants import MissingKey
+from theHarvester.lib.core import *
+
class SearchHunterHow:
def __init__(self, word) -> None:
@@ -12,44 +14,55 @@ class SearchHunterHow:
self.total_hostnames: Set = set()
self.key = Core.hunterhow_key()
if self.key is None:
- raise MissingKey('hunterhow')
+ raise MissingKey("hunterhow")
self.proxy = False
async def do_search(self) -> None:
# https://hunter.how/search-api
query = f'domain.suffix="{self.word}"'
# second_query = f'domain="{self.word}"'
- encoded_query = base64.urlsafe_b64encode(query.encode("utf-8")).decode('ascii')
+ encoded_query = base64.urlsafe_b64encode(query.encode("utf-8")).decode("ascii")
page = 1
page_size = 100 # can be either: 10,20,50,100)
# The interval between the start time and the end time cannot exceed one year
# Can not exceed one year, but years=1 does not work due to their backend, 364 will suffice
today = datetime.today()
one_year_ago = today - relativedelta(days=364)
- start_time = one_year_ago.strftime('%Y-%m-%d')
- end_time = today.strftime('%Y-%m-%d')
+ start_time = one_year_ago.strftime("%Y-%m-%d")
+ end_time = today.strftime("%Y-%m-%d")
# two_years_ago = one_year_ago - relativedelta(days=364)
# start_time = two_years_ago.strftime('%Y-%m-%d')
# end_time = one_year_ago.strftime('%Y-%m-%d')
- url = "https://api.hunter.how/search?api-key=%s&query=%s&page=%d&page_size=%d&start_time=%s&end_time=%s" % (
- # self.key, encoded_query, page, page_size, start_time, end_time
- self.key, encoded_query, page, page_size, start_time, end_time
+ url = (
+ "https://api.hunter.how/search?api-key=%s&query=%s&page=%d&page_size=%d&start_time=%s&end_time=%s"
+ % (
+ # self.key, encoded_query, page, page_size, start_time, end_time
+ self.key,
+ encoded_query,
+ page,
+ page_size,
+ start_time,
+ end_time,
+ )
)
# print(f'Sending url: {url}')
- response = await AsyncFetcher.fetch_all([url], json=True, headers={'User-Agent': Core.get_user_agent(),
- 'x-api-key': f'{self.key}'},
- proxy=self.proxy)
+ response = await AsyncFetcher.fetch_all(
+ [url],
+ json=True,
+ headers={"User-Agent": Core.get_user_agent(), "x-api-key": f"{self.key}"},
+ proxy=self.proxy,
+ )
dct = response[0]
# print(f'json response: ')
# print(dct)
- if 'code' in dct.keys():
- if dct['code'] == 40001:
+ if "code" in dct.keys():
+ if dct["code"] == 40001:
print(f'Code 40001 indicates for searchhunterhow: {dct["message"]}')
return
# total = dct['data']['total']
# TODO determine if total is ever 100 how to get more subdomains?
- for sub in dct['data']['list']:
- self.total_hostnames.add(sub['domain'])
+ for sub in dct["data"]["list"]:
+ self.total_hostnames.add(sub["domain"])
async def get_hostnames(self) -> set:
return self.total_hostnames
diff --git a/theHarvester/discovery/shodansearch.py b/theHarvester/discovery/shodansearch.py
index b30adcc6..dd549e4e 100644
--- a/theHarvester/discovery/shodansearch.py
+++ b/theHarvester/discovery/shodansearch.py
@@ -1,17 +1,17 @@
-from theHarvester.discovery.constants import *
-from theHarvester.lib.core import *
-from shodan import exception
-from shodan import Shodan
from collections import OrderedDict
from typing import List
+from shodan import Shodan, exception
+
+from theHarvester.discovery.constants import *
+from theHarvester.lib.core import *
+
class SearchShodan:
-
def __init__(self) -> None:
self.key = Core.shodan_key()
if self.key is None:
- raise MissingKey('Shodan')
+ raise MissingKey("Shodan")
self.api = Shodan(self.key)
self.hostdatarow: List = []
self.tracker: OrderedDict = OrderedDict()
@@ -20,72 +20,81 @@ class SearchShodan:
try:
ipaddress = ip
results = self.api.host(ipaddress)
- asn = ''
+ asn = ""
domains: List = list()
hostnames: List = list()
- ip_str = ''
- isp = ''
- org = ''
+ ip_str = ""
+ isp = ""
+ org = ""
ports: List = list()
- title = ''
- server = ''
- product = ''
+ title = ""
+ server = ""
+ product = ""
technologies: List = list()
- data_first_dict = dict(results['data'][0])
+ data_first_dict = dict(results["data"][0])
- if 'ip_str' in data_first_dict.keys():
- ip_str += data_first_dict['ip_str']
+ if "ip_str" in data_first_dict.keys():
+ ip_str += data_first_dict["ip_str"]
- if 'http' in data_first_dict.keys():
- http_results_dict = dict(data_first_dict['http'])
- if 'title' in http_results_dict.keys():
- title_val = str(http_results_dict['title']).strip()
- if title_val != 'None':
+ if "http" in data_first_dict.keys():
+ http_results_dict = dict(data_first_dict["http"])
+ if "title" in http_results_dict.keys():
+ title_val = str(http_results_dict["title"]).strip()
+ if title_val != "None":
title += title_val
- if 'components' in http_results_dict.keys():
- for key in http_results_dict['components'].keys():
+ if "components" in http_results_dict.keys():
+ for key in http_results_dict["components"].keys():
technologies.append(key)
- if 'server' in http_results_dict.keys():
- server_val = str(http_results_dict['server']).strip()
- if server_val != 'None':
+ if "server" in http_results_dict.keys():
+ server_val = str(http_results_dict["server"]).strip()
+ if server_val != "None":
server += server_val
for key, value in results.items():
- if key == 'asn':
+ if key == "asn":
asn += value
- if key == 'domains':
+ if key == "domains":
value = list(value)
value.sort()
domains.extend(value)
- if key == 'hostnames':
+ if key == "hostnames":
value = [host.strip() for host in list(value)]
value.sort()
hostnames.extend(value)
- if key == 'isp':
+ if key == "isp":
isp += value
- if key == 'org':
+ if key == "org":
org += str(value)
- if key == 'ports':
+ if key == "ports":
value = list(value)
value.sort()
ports.extend(value)
- if key == 'product':
+ if key == "product":
product += value
technologies = list(set(technologies))
- self.tracker[ip] = {'asn': asn.strip(), 'domains': domains, 'hostnames': hostnames,
- 'ip_str': ip_str.strip(), 'isp': isp.strip(), 'org': org.strip(),
- 'ports': ports, 'product': product.strip(),
- 'server': server.strip(), 'technologies': technologies, 'title': title.strip()}
+ self.tracker[ip] = {
+ "asn": asn.strip(),
+ "domains": domains,
+ "hostnames": hostnames,
+ "ip_str": ip_str.strip(),
+ "isp": isp.strip(),
+ "org": org.strip(),
+ "ports": ports,
+ "product": product.strip(),
+ "server": server.strip(),
+ "technologies": technologies,
+ "title": title.strip(),
+ }
return self.tracker
except exception.APIError:
- print(f'{ip}: Not in Shodan')
- self.tracker[ip] = 'Not in Shodan'
+ print(f"{ip}: Not in Shodan")
+ self.tracker[ip] = "Not in Shodan"
except Exception as e:
# print(f'Error occurred in the Shodan IP search module: {e}')
- self.tracker[ip] = f'Error occurred in the Shodan IP search module: {e}'
+ self.tracker[ip] = f"Error occurred in the Shodan IP search module: {e}"
finally:
return self.tracker
diff --git a/theHarvester/discovery/sitedossier.py b/theHarvester/discovery/sitedossier.py
index c5f82f8f..245fb19a 100644
--- a/theHarvester/discovery/sitedossier.py
+++ b/theHarvester/discovery/sitedossier.py
@@ -1,9 +1,9 @@
-from theHarvester.discovery.constants import *
from bs4 import BeautifulSoup
+from theHarvester.discovery.constants import *
+
class SearchSitedossier:
-
def __init__(self, word):
self.word = word
self.totalhosts = set()
@@ -16,66 +16,91 @@ class SearchSitedossier:
# Hence the need for delays after each request to get the most results
# Feel free to tweak the delays as needed
url = f"http://{self.server}/parentdomain/{self.word}"
- headers = {'User-Agent': Core.get_user_agent()}
- response = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy)
+ headers = {"User-Agent": Core.get_user_agent()}
+ response = await AsyncFetcher.fetch_all(
+ [url], headers=headers, proxy=self.proxy
+ )
base_response = response[0]
- soup = BeautifulSoup(base_response, 'html.parser')
+ soup = BeautifulSoup(base_response, "html.parser")
# iter_counter = 1
# iterations_needed = total_number // 100
# iterations_needed += 1
flagged_counter = 0
- stop_conditions = ['End of list.', 'No data currently available.']
- bot_string = 'Our web servers have detected unusual or excessive requests ' \
- 'from your computer or network. Please enter the unique "word"' \
- ' below to confirm that you are a human interactively using this site.'
- if (stop_conditions[0] not in base_response and stop_conditions[1] not in base_response) \
- and bot_string not in base_response:
- total_number = soup.find('i')
- total_number = int(total_number.text.strip().split(' ')[-1].replace(',', ''))
- hrefs = soup.find_all('a', href=True)
+ stop_conditions = ["End of list.", "No data currently available."]
+ bot_string = (
+ "Our web servers have detected unusual or excessive requests "
+ 'from your computer or network. Please enter the unique "word"'
+ " below to confirm that you are a human interactively using this site."
+ )
+ if (
+ stop_conditions[0] not in base_response
+ and stop_conditions[1] not in base_response
+ ) and bot_string not in base_response:
+ total_number = soup.find("i")
+ total_number = int(
+ total_number.text.strip().split(" ")[-1].replace(",", "")
+ )
+ hrefs = soup.find_all("a", href=True)
for a in hrefs:
- unparsed = a['href']
- if '/site/' in unparsed:
- subdomain = str(unparsed.split('/')[-1]).lower()
+ unparsed = a["href"]
+ if "/site/" in unparsed:
+ subdomain = str(unparsed.split("/")[-1]).lower()
self.totalhosts.add(subdomain)
await asyncio.sleep(get_delay() + 15 + get_delay())
for i in range(101, total_number, 100):
- headers = {'User-Agent': Core.get_user_agent()}
+ headers = {"User-Agent": Core.get_user_agent()}
iter_url = f"http://{self.server}/parentdomain/{self.word}/{i}"
- print(f'My current iter_url: {iter_url}')
- response = await AsyncFetcher.fetch_all([iter_url], headers=headers, proxy=self.proxy)
+ print(f"My current iter_url: {iter_url}")
+ response = await AsyncFetcher.fetch_all(
+ [iter_url], headers=headers, proxy=self.proxy
+ )
response = response[0]
- if stop_conditions[0] in response or stop_conditions[1] in response or flagged_counter >= 3:
+ if (
+ stop_conditions[0] in response
+ or stop_conditions[1] in response
+ or flagged_counter >= 3
+ ):
break
if bot_string in response:
new_sleep_time = get_delay() * 30
- print(f'Triggered a captcha for sitedossier sleeping for: {new_sleep_time} seconds')
+ print(
+ f"Triggered a captcha for sitedossier sleeping for: {new_sleep_time} seconds"
+ )
flagged_counter += 1
await asyncio.sleep(new_sleep_time)
- response = await AsyncFetcher.fetch_all([iter_url], headers={'User-Agent': Core.get_user_agent()},
- proxy=self.proxy)
+ response = await AsyncFetcher.fetch_all(
+ [iter_url],
+ headers={"User-Agent": Core.get_user_agent()},
+ proxy=self.proxy,
+ )
response = response[0]
if bot_string in response:
new_sleep_time = get_delay() * 30 * get_delay()
- print(f'Still triggering a captcha, sleeping longer for: {new_sleep_time}'
- f' and skipping this batch: {iter_url}')
+ print(
+ f"Still triggering a captcha, sleeping longer for: {new_sleep_time}"
+ f" and skipping this batch: {iter_url}"
+ )
await asyncio.sleep(new_sleep_time)
flagged_counter += 1
if flagged_counter >= 3:
break
- soup = BeautifulSoup(response, 'html.parser')
- hrefs = soup.find_all('a', href=True)
+ soup = BeautifulSoup(response, "html.parser")
+ hrefs = soup.find_all("a", href=True)
for a in hrefs:
- unparsed = a['href']
- if '/site/' in unparsed:
- subdomain = str(unparsed.split('/')[-1]).lower()
+ unparsed = a["href"]
+ if "/site/" in unparsed:
+ subdomain = str(unparsed.split("/")[-1]).lower()
self.totalhosts.add(subdomain)
await asyncio.sleep(get_delay() + 15 + get_delay())
- print(f'In total found: {len(self.totalhosts)}')
+ print(f"In total found: {len(self.totalhosts)}")
print(self.totalhosts)
else:
- print('Sitedossier module has triggered a captcha on first iteration, no results can be found.')
- print('Change IPs, manually solve the captcha, or wait before rerunning Sitedossier module')
+ print(
+ "Sitedossier module has triggered a captcha on first iteration, no results can be found."
+ )
+ print(
+ "Change IPs, manually solve the captcha, or wait before rerunning Sitedossier module"
+ )
async def get_hostnames(self):
return self.totalhosts
diff --git a/theHarvester/discovery/subdomaincenter.py b/theHarvester/discovery/subdomaincenter.py
index b70ede5d..7e64b7ec 100644
--- a/theHarvester/discovery/subdomaincenter.py
+++ b/theHarvester/discovery/subdomaincenter.py
@@ -2,22 +2,26 @@ from theHarvester.discovery.constants import *
class SubdomainCenter:
-
def __init__(self, word):
self.word = word
self.results = set()
- self.server = 'https://api.subdomain.center/?domain='
+ self.server = "https://api.subdomain.center/?domain="
self.proxy = False
async def do_search(self):
- headers = {'User-Agent': Core.get_user_agent()}
+ headers = {"User-Agent": Core.get_user_agent()}
try:
- current_url = f'{self.server}{self.word}'
- resp = await AsyncFetcher.fetch_all([current_url], headers=headers, proxy=self.proxy, json=True)
+ current_url = f"{self.server}{self.word}"
+ resp = await AsyncFetcher.fetch_all(
+ [current_url], headers=headers, proxy=self.proxy, json=True
+ )
self.results = resp[0]
- self.results = {sub[4:] if sub[:4] == 'www.' and sub[4:] else sub for sub in self.results}
+ self.results = {
+ sub[4:] if sub[:4] == "www." and sub[4:] else sub
+ for sub in self.results
+ }
except Exception as e:
- print(f'An exception has occurred in SubdomainCenter on : {e}')
+ print(f"An exception has occurred in SubdomainCenter on : {e}")
async def get_hostnames(self):
return self.results
diff --git a/theHarvester/discovery/subdomainfinderc99.py b/theHarvester/discovery/subdomainfinderc99.py
index d10f6e5d..3b1082f4 100644
--- a/theHarvester/discovery/subdomainfinderc99.py
+++ b/theHarvester/discovery/subdomainfinderc99.py
@@ -1,9 +1,11 @@
-from theHarvester.lib.core import *
-from theHarvester.discovery.constants import get_delay
-from theHarvester.parsers import myparser
from typing import Set
-from bs4 import BeautifulSoup
+
import ujson
+from bs4 import BeautifulSoup
+
+from theHarvester.discovery.constants import get_delay
+from theHarvester.lib.core import *
+from theHarvester.parsers import myparser
class SearchSubdomainfinderc99:
@@ -12,21 +14,24 @@ class SearchSubdomainfinderc99:
self.total_results: Set = set()
self.proxy = False
# TODO add api support
- self.server = 'https://subdomainfinder.c99.nl/'
- self.totalresults = ''
+ self.server = "https://subdomainfinder.c99.nl/"
+ self.totalresults = ""
async def do_search(self) -> None:
# Based on https://gist.github.com/th3gundy/bc83580cbe04031e9164362b33600962
- headers = {'User-Agent': Core.get_user_agent()}
- resp = await AsyncFetcher.fetch_all([self.server], headers=headers, proxy=self.proxy)
+ headers = {"User-Agent": Core.get_user_agent()}
+ resp = await AsyncFetcher.fetch_all(
+ [self.server], headers=headers, proxy=self.proxy
+ )
data = await self.get_csrf_params(resp[0])
- data['scan_subdomains'] = ''
- data['domain'] = self.word
- data['privatequery'] = 'on'
+ data["scan_subdomains"] = ""
+ data["domain"] = self.word
+ data["privatequery"] = "on"
await asyncio.sleep(get_delay())
- second_resp = await AsyncFetcher.post_fetch(self.server, headers=headers, proxy=self.proxy,
- data=ujson.dumps(data))
+ second_resp = await AsyncFetcher.post_fetch(
+ self.server, headers=headers, proxy=self.proxy, data=ujson.dumps(data)
+ )
# print(second_resp)
self.totalresults += second_resp
@@ -50,10 +55,10 @@ class SearchSubdomainfinderc99:
@staticmethod
async def get_csrf_params(data):
csrf_params = {}
- html = BeautifulSoup(data, 'html.parser').find('div', {'class': 'input-group'})
- for c in html.find_all('input'):
+ html = BeautifulSoup(data, "html.parser").find("div", {"class": "input-group"})
+ for c in html.find_all("input"):
try:
- csrf_params[c.get('name')] = c.get('value')
+ csrf_params[c.get("name")] = c.get("value")
except Exception:
continue
diff --git a/theHarvester/discovery/threatminer.py b/theHarvester/discovery/threatminer.py
index a196d4cb..a162a9fd 100644
--- a/theHarvester/discovery/threatminer.py
+++ b/theHarvester/discovery/threatminer.py
@@ -1,9 +1,9 @@
-from typing import Type, List, Set
+from typing import List, Set, Type
+
from theHarvester.lib.core import *
class SearchThreatminer:
-
def __init__(self, word) -> None:
self.word = word
self.totalhosts: Set = set()
@@ -11,13 +11,15 @@ class SearchThreatminer:
self.proxy = False
async def do_search(self) -> None:
- url = f'https://api.threatminer.org/v2/domain.php?q={self.word}&rt=5'
+ url = f"https://api.threatminer.org/v2/domain.php?q={self.word}&rt=5"
response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy)
- self.totalhosts = {host for host in response[0]['results']}
- second_url = f'https://api.threatminer.org/v2/domain.php?q={self.word}&rt=2'
- secondresp = await AsyncFetcher.fetch_all([second_url], json=True, proxy=self.proxy)
+ self.totalhosts = {host for host in response[0]["results"]}
+ second_url = f"https://api.threatminer.org/v2/domain.php?q={self.word}&rt=2"
+ secondresp = await AsyncFetcher.fetch_all(
+ [second_url], json=True, proxy=self.proxy
+ )
try:
- self.totalips = {resp['ip'] for resp in secondresp[0]['results']}
+ self.totalips = {resp["ip"] for resp in secondresp[0]["results"]}
except TypeError:
pass
diff --git a/theHarvester/discovery/tombasearch.py b/theHarvester/discovery/tombasearch.py
index 3f4993d5..9f464c20 100644
--- a/theHarvester/discovery/tombasearch.py
+++ b/theHarvester/discovery/tombasearch.py
@@ -1,10 +1,10 @@
+from typing import List
+
from theHarvester.discovery.constants import *
from theHarvester.lib.core import *
-from typing import List
class SearchTomba:
-
def __init__(self, word, limit, start) -> None:
self.word = word
self.limit = limit
@@ -12,10 +12,12 @@ class SearchTomba:
self.start = start
self.key = Core.tomba_key()
if self.key[0] is None or self.key[1] is None:
- raise MissingKey('Tomba Key and/or Secret')
+ raise MissingKey("Tomba Key and/or Secret")
self.total_results = ""
self.counter = start
- self.database = f'https://api.tomba.io/v1/domain-search?domain={self.word}&limit=10'
+ self.database = (
+ f"https://api.tomba.io/v1/domain-search?domain={self.word}&limit=10"
+ )
self.proxy = False
self.hostnames: List = []
self.emails: List = []
@@ -23,48 +25,81 @@ class SearchTomba:
async def do_search(self) -> None:
# First determine if a user account is not a free account, this call is free
is_free = True
- headers = {'User-Agent': Core.get_user_agent(), 'X-Tomba-Key': self.key[0], 'X-Tomba-Secret': self.key[1]}
- acc_info_url = 'https://api.tomba.io/v1/me'
- response = await AsyncFetcher.fetch_all([acc_info_url], headers=headers, json=True)
- is_free = is_free if 'name' in response[0]['data']['pricing'].keys() and response[0]['data']['pricing']['name'].lower() \
- == 'free' else False
+ headers = {
+ "User-Agent": Core.get_user_agent(),
+ "X-Tomba-Key": self.key[0],
+ "X-Tomba-Secret": self.key[1],
+ }
+ acc_info_url = "https://api.tomba.io/v1/me"
+ response = await AsyncFetcher.fetch_all(
+ [acc_info_url], headers=headers, json=True
+ )
+ is_free = (
+ is_free
+ if "name" in response[0]["data"]["pricing"].keys()
+ and response[0]["data"]["pricing"]["name"].lower() == "free"
+ else False
+ )
# Extract the total number of requests that are available for an account
- total_requests_avail = response[0]['data']['requests']['domains']['available'] - response[0]['data']['requests']['domains']['used']
+ total_requests_avail = (
+ response[0]["data"]["requests"]["domains"]["available"]
+ - response[0]["data"]["requests"]["domains"]["used"]
+ )
if is_free:
- response = await AsyncFetcher.fetch_all([self.database], headers=headers, proxy=self.proxy, json=True)
+ response = await AsyncFetcher.fetch_all(
+ [self.database], headers=headers, proxy=self.proxy, json=True
+ )
self.emails, self.hostnames = await self.parse_resp(json_resp=response[0])
else:
# Determine the total number of emails that are available
# As the most emails you can get within one query are 100
# This is only done where paid accounts are in play
- tomba_counter = f'https://api.tomba.io/v1/email-count?domain={self.word}'
- response = await AsyncFetcher.fetch_all([tomba_counter], headers=headers, proxy=self.proxy, json=True)
- total_number_reqs = response[0]['data']['total'] // 100
+ tomba_counter = f"https://api.tomba.io/v1/email-count?domain={self.word}"
+ response = await AsyncFetcher.fetch_all(
+ [tomba_counter], headers=headers, proxy=self.proxy, json=True
+ )
+ total_number_reqs = response[0]["data"]["total"] // 100
# Parse out meta field within initial JSON response to determine the total number of results
if total_requests_avail < total_number_reqs:
- print('WARNING: The account does not have enough requests to gather all the emails.')
- print(f'Total requests available: {total_requests_avail}, total requests '
- f'needed to be made: {total_number_reqs}')
- print('RETURNING current results, If you still wish to run this module despite the current results, please comment out the "if request" line.')
+ print(
+ "WARNING: The account does not have enough requests to gather all the emails."
+ )
+ print(
+ f"Total requests available: {total_requests_avail}, total requests "
+ f"needed to be made: {total_number_reqs}"
+ )
+ print(
+ 'RETURNING current results, If you still wish to run this module despite the current results, please comment out the "if request" line.'
+ )
return
self.limit = 100
# max number of emails you can get per request
# increments of max number with page determining where to start
# See docs for more details: https://developer.tomba.io/#domain-search
for page in range(0, total_number_reqs + 1):
- req_url = f'https://api.tomba.io/v1/domain-search?domain={self.word}&limit={self.limit}&page={page}'
- response = await AsyncFetcher.fetch_all([req_url], headers=headers, proxy=self.proxy, json=True)
+ req_url = f"https://api.tomba.io/v1/domain-search?domain={self.word}&limit={self.limit}&page={page}"
+ response = await AsyncFetcher.fetch_all(
+ [req_url], headers=headers, proxy=self.proxy, json=True
+ )
temp_emails, temp_hostnames = await self.parse_resp(response[0])
self.emails.extend(temp_emails)
self.hostnames.extend(temp_hostnames)
await asyncio.sleep(1)
async def parse_resp(self, json_resp):
- emails = list(sorted({email['email'] for email in json_resp['data']['emails']}))
- domains = list(sorted({source['website_url'] for email in json_resp['data']['emails'] for source in email['sources']
- if self.word in source['website_url']}))
+ emails = list(sorted({email["email"] for email in json_resp["data"]["emails"]}))
+ domains = list(
+ sorted(
+ {
+ source["website_url"]
+ for email in json_resp["data"]["emails"]
+ for source in email["sources"]
+ if self.word in source["website_url"]
+ }
+ )
+ )
return emails, domains
async def process(self, proxy: bool = False) -> None:
diff --git a/theHarvester/discovery/urlscan.py b/theHarvester/discovery/urlscan.py
index c5b1e7c4..c3e2371b 100644
--- a/theHarvester/discovery/urlscan.py
+++ b/theHarvester/discovery/urlscan.py
@@ -1,4 +1,5 @@
from typing import List, Set
+
from theHarvester.lib.core import *
@@ -12,13 +13,25 @@ class SearchUrlscan:
self.proxy = False
async def do_search(self) -> None:
- url = f'https://urlscan.io/api/v1/search/?q=domain:{self.word}'
+ url = f"https://urlscan.io/api/v1/search/?q=domain:{self.word}"
response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy)
resp = response[0]
- self.totalhosts = {f"{page['page']['domain']}" for page in resp['results']}
- self.totalips = {f"{page['page']['ip']}" for page in resp['results'] if 'ip' in page['page'].keys()}
- self.interestingurls = {f"{page['page']['url']}" for page in resp['results'] if self.word in page['page']['url'] and 'url' in page['page'].keys()}
- self.totalasns = {f"{page['page']['asn']}" for page in resp['results'] if 'asn' in page['page'].keys()}
+ self.totalhosts = {f"{page['page']['domain']}" for page in resp["results"]}
+ self.totalips = {
+ f"{page['page']['ip']}"
+ for page in resp["results"]
+ if "ip" in page["page"].keys()
+ }
+ self.interestingurls = {
+ f"{page['page']['url']}"
+ for page in resp["results"]
+ if self.word in page["page"]["url"] and "url" in page["page"].keys()
+ }
+ self.totalasns = {
+ f"{page['page']['asn']}"
+ for page in resp["results"]
+ if "asn" in page["page"].keys()
+ }
async def get_hostnames(self) -> Set:
return self.totalhosts
diff --git a/theHarvester/discovery/virustotal.py b/theHarvester/discovery/virustotal.py
index edc4d82d..5a849429 100644
--- a/theHarvester/discovery/virustotal.py
+++ b/theHarvester/discovery/virustotal.py
@@ -3,11 +3,10 @@ from theHarvester.lib.core import *
class SearchVirustotal:
-
def __init__(self, word) -> None:
self.key = Core.virustotal_key()
if self.key is None:
- raise MissingKey('virustotal')
+ raise MissingKey("virustotal")
self.word = word
self.proxy = False
self.hostnames: List = []
@@ -17,12 +16,14 @@ class SearchVirustotal:
# based on: https://developers.virustotal.com/reference/domains-relationships
# base_url = "https://www.virustotal.com/api/v3/domains/domain/subdomains?limit=40"
headers = {
- 'User-Agent': Core.get_user_agent(),
+ "User-Agent": Core.get_user_agent(),
"Accept": "application/json",
- "x-apikey": self.key
+ "x-apikey": self.key,
}
- base_url = f"https://www.virustotal.com/api/v3/domains/{self.word}/subdomains?limit=40"
- cursor = ''
+ base_url = (
+ f"https://www.virustotal.com/api/v3/domains/{self.word}/subdomains?limit=40"
+ )
+ cursor = ""
count = 0
fail_counter = 0
counter = 0
@@ -34,28 +35,43 @@ class SearchVirustotal:
# TODO add timer logic if proven to be needed
# in the meantime sleeping 16 seconds should eliminate hitting the rate limit
# in case rate limit is hit, fail counter exists and sleep for 65 seconds
- send_url = base_url + "&cursor=" + cursor if cursor != '' and len(cursor) > 2 else base_url
- responses = await AsyncFetcher.fetch_all([send_url], headers=headers, proxy=self.proxy, json=True)
+ send_url = (
+ base_url + "&cursor=" + cursor
+ if cursor != "" and len(cursor) > 2
+ else base_url
+ )
+ responses = await AsyncFetcher.fetch_all(
+ [send_url], headers=headers, proxy=self.proxy, json=True
+ )
jdata = responses[0]
- if 'data' not in jdata.keys():
+ if "data" not in jdata.keys():
await asyncio.sleep(60 + 5)
fail_counter += 1
- if 'meta' in jdata.keys():
- cursor = jdata['meta']['cursor'] if 'cursor' in jdata['meta'].keys() else ''
- if len(cursor) == 0 and 'data' in jdata.keys():
+ if "meta" in jdata.keys():
+ cursor = (
+ jdata["meta"]["cursor"] if "cursor" in jdata["meta"].keys() else ""
+ )
+ if len(cursor) == 0 and "data" in jdata.keys():
# if cursor no longer is within the meta field have hit last entry
breakcon = True
- count += jdata['meta']['count']
+ count += jdata["meta"]["count"]
if count == 0 or fail_counter >= 2:
break
- if 'data' in jdata.keys():
- data = jdata['data']
+ if "data" in jdata.keys():
+ data = jdata["data"]
self.hostnames.extend(await self.parse_hostnames(data, self.word))
counter += 1
await asyncio.sleep(16)
self.hostnames = list(sorted(set(self.hostnames)))
# verify domains such as x.x.com.multicdn.x.com are parsed properly
- self.hostnames = [host for host in self.hostnames if ((len(host.split('.')) >= 3) and host.split('.')[-2] == self.word.split('.')[-2])]
+ self.hostnames = [
+ host
+ for host in self.hostnames
+ if (
+ (len(host.split(".")) >= 3)
+ and host.split(".")[-2] == self.word.split(".")[-2]
+ )
+ ]
async def get_hostnames(self) -> list:
return self.hostnames
@@ -64,21 +80,36 @@ class SearchVirustotal:
async def parse_hostnames(data, word):
total_subdomains = set()
for attribute in data:
- total_subdomains.add(attribute['id'].replace('"', '').replace('www.', ''))
- attributes = attribute['attributes']
+ total_subdomains.add(attribute["id"].replace('"', "").replace("www.", ""))
+ attributes = attribute["attributes"]
total_subdomains.update(
- {value['value'].replace('"', '').replace('www.', '') for value in attributes['last_dns_records'] if
- word in value['value']})
- if 'last_https_certificate' in attributes.keys():
- total_subdomains.update({value.replace('"', '').replace('www.', '') for value in
- attributes['last_https_certificate']['extensions']['subject_alternative_name']
- if word in value})
+ {
+ value["value"].replace('"', "").replace("www.", "")
+ for value in attributes["last_dns_records"]
+ if word in value["value"]
+ }
+ )
+ if "last_https_certificate" in attributes.keys():
+ total_subdomains.update(
+ {
+ value.replace('"', "").replace("www.", "")
+ for value in attributes["last_https_certificate"]["extensions"][
+ "subject_alternative_name"
+ ]
+ if word in value
+ }
+ )
total_subdomains = list(sorted(total_subdomains))
# Other false positives may occur over time and yes there are other ways to parse this, feel free to implement
# them and submit a PR or raise an issue if you run into this filtering not being enough
# TODO determine if parsing 'v=spf1 include:_spf-x.acme.com include:_spf-x.acme.com' is worth parsing
- total_subdomains = [x for x in total_subdomains if not str(x).endswith('edgekey.net') and not str(x).endswith(
- 'akadns.net') and 'include:_spf' not in str(x)]
+ total_subdomains = [
+ x
+ for x in total_subdomains
+ if not str(x).endswith("edgekey.net")
+ and not str(x).endswith("akadns.net")
+ and "include:_spf" not in str(x)
+ ]
total_subdomains.sort()
return total_subdomains
diff --git a/theHarvester/discovery/yahoosearch.py b/theHarvester/discovery/yahoosearch.py
index a71ed604..d0c069b0 100644
--- a/theHarvester/discovery/yahoosearch.py
+++ b/theHarvester/discovery/yahoosearch.py
@@ -3,22 +3,24 @@ from theHarvester.parsers import myparser
class SearchYahoo:
-
def __init__(self, word, limit) -> None:
self.word = word
self.total_results = ""
- self.server = 'search.yahoo.com'
+ self.server = "search.yahoo.com"
self.limit = limit
self.proxy = False
async def do_search(self) -> None:
- base_url = f'https://{self.server}/search?p=%40{self.word}&b=xx&pz=10'
- headers = {
- 'Host': self.server,
- 'User-agent': Core.get_user_agent()
- }
- urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 10) if num <= self.limit]
- responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy)
+ base_url = f"https://{self.server}/search?p=%40{self.word}&b=xx&pz=10"
+ headers = {"Host": self.server, "User-agent": Core.get_user_agent()}
+ urls = [
+ base_url.replace("xx", str(num))
+ for num in range(0, self.limit, 10)
+ if num <= self.limit
+ ]
+ responses = await AsyncFetcher.fetch_all(
+ urls, headers=headers, proxy=self.proxy
+ )
for response in responses:
self.total_results += response
@@ -33,8 +35,8 @@ class SearchYahoo:
# strip out numbers and dashes for emails that look like xxx-xxx-xxxemail@host.tld
for email in toparse_emails:
email = str(email)
- if '-' in email and email[0].isdigit() and email.index('-') <= 9:
- while email[0] == '-' or email[0].isdigit():
+ if "-" in email and email[0].isdigit() and email.index("-") <= 9:
+ while email[0] == "-" or email[0].isdigit():
email = email[1:]
emails.add(email)
return list(emails)
diff --git a/theHarvester/lib/__init__.py b/theHarvester/lib/__init__.py
index 8fc0aea3..7145285d 100644
--- a/theHarvester/lib/__init__.py
+++ b/theHarvester/lib/__init__.py
@@ -1 +1 @@
-__all__ = ['hostchecker']
+__all__ = ["hostchecker"]
diff --git a/theHarvester/lib/api/api.py b/theHarvester/lib/api/api.py
index 0849896c..508b5cd5 100644
--- a/theHarvester/lib/api/api.py
+++ b/theHarvester/lib/api/api.py
@@ -1,6 +1,7 @@
import argparse
-from typing import Any, Dict, Union, List
import os
+from typing import Any, Dict, List, Union
+
from fastapi import FastAPI, Header, Query, Request
from fastapi.responses import HTMLResponse, UJSONResponse
from slowapi import Limiter, _rate_limit_exceeded_handler
@@ -12,25 +13,37 @@ from starlette.staticfiles import StaticFiles
from theHarvester import __main__
limiter = Limiter(key_func=get_remote_address)
-app = FastAPI(title='Restful Harvest', description='Rest API for theHarvester powered by FastAPI', version='0.0.2')
+app = FastAPI(
+ title="Restful Harvest",
+ description="Rest API for theHarvester powered by FastAPI",
+ version="0.0.2",
+)
app.state.limiter = limiter
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
# This is where we will host files that arise if the user specifies a filename
try:
- app.mount('/static', StaticFiles(directory='theHarvester/lib/api/static/'), name='static')
+ app.mount(
+ "/static", StaticFiles(directory="theHarvester/lib/api/static/"), name="static"
+ )
except RuntimeError:
- static_path = os.path.expanduser('~/.local/share/theHarvester/static/')
+ static_path = os.path.expanduser("~/.local/share/theHarvester/static/")
if not os.path.isdir(static_path):
os.makedirs(static_path)
- app.mount('/static', StaticFiles(directory='~/.local/share/theHarvester/static/'), name='static')
+ app.mount(
+ "/static",
+ StaticFiles(directory="~/.local/share/theHarvester/static/"),
+ name="static",
+ )
-@app.get('/', response_class=HTMLResponse)
+@app.get("/", response_class=HTMLResponse)
async def root(*, user_agent: str = Header(None)) -> Union[RedirectResponse, str]:
# very basic user agent filtering
- if user_agent and ('gobuster' in user_agent or 'sqlmap' in user_agent or 'rustbuster' in user_agent):
- response = RedirectResponse(app.url_path_for('bot'))
+ if user_agent and (
+ "gobuster" in user_agent or "sqlmap" in user_agent or "rustbuster" in user_agent
+ ):
+ response = RedirectResponse(app.url_path_for("bot"))
return response
html = """
@@ -58,93 +71,130 @@ async def root(*, user_agent: str = Header(None)) -> Union[RedirectResponse, str
return html
-@app.get('/nicebot')
+@app.get("/nicebot")
async def bot() -> Dict[str, str]:
# nice bot
- string = {'bot': 'These are not the droids you are looking for'}
+ string = {"bot": "These are not the droids you are looking for"}
return string
-@app.get('/sources', response_class=UJSONResponse)
-@limiter.limit('5/minute')
+@app.get("/sources", response_class=UJSONResponse)
+@limiter.limit("5/minute")
async def getsources(request: Request):
# Endpoint for user to query for available sources theHarvester supports
# Rate limit of 5 requests per minute
sources = __main__.Core.get_supportedengines()
- return {'sources': sources}
+ return {"sources": sources}
-@app.get('/dnsbrute', response_class=UJSONResponse)
-@limiter.limit('5/minute')
-async def dnsbrute(request: Request, user_agent: str = Header(None),
- domain: str = Query(..., description='Domain to be brute forced')) -> Union[Dict[str, Any], RedirectResponse]:
+@app.get("/dnsbrute", response_class=UJSONResponse)
+@limiter.limit("5/minute")
+async def dnsbrute(
+ request: Request,
+ user_agent: str = Header(None),
+ domain: str = Query(..., description="Domain to be brute forced"),
+) -> Union[Dict[str, Any], RedirectResponse]:
# Endpoint for user to signal to do DNS brute forcing
# Rate limit of 5 requests per minute
# basic user agent filtering
- if user_agent and ('gobuster' in user_agent or 'sqlmap' in user_agent or 'rustbuster' in user_agent):
- response = RedirectResponse(app.url_path_for('bot'))
+ if user_agent and (
+ "gobuster" in user_agent or "sqlmap" in user_agent or "rustbuster" in user_agent
+ ):
+ response = RedirectResponse(app.url_path_for("bot"))
return response
- dns_bruteforce = await __main__.start(argparse.Namespace(dns_brute=True,
- dns_lookup=False,
- dns_server=False,
- dns_tld=False,
- domain=domain,
- filename='',
- google_dork=False,
- limit=500,
- proxies=False,
- shodan=False,
- source=','.join([]),
- start=0,
- take_over=False,
- virtual_host=False))
- return {'dns_bruteforce': dns_bruteforce}
+ dns_bruteforce = await __main__.start(
+ argparse.Namespace(
+ dns_brute=True,
+ dns_lookup=False,
+ dns_server=False,
+ dns_tld=False,
+ domain=domain,
+ filename="",
+ google_dork=False,
+ limit=500,
+ proxies=False,
+ shodan=False,
+ source=",".join([]),
+ start=0,
+ take_over=False,
+ virtual_host=False,
+ )
+ )
+ return {"dns_bruteforce": dns_bruteforce}
-@app.get('/query', response_class=UJSONResponse)
-@limiter.limit('2/minute')
-async def query(request: Request, dns_server: str = Query(""), user_agent: str = Header(None),
- dns_brute: bool = Query(False),
- dns_lookup: bool = Query(False),
- dns_tld: bool = Query(False),
- filename: str = Query(""),
- google_dork: bool = Query(False), proxies: bool = Query(False), shodan: bool = Query(False),
- take_over: bool = Query(False), virtual_host: bool = Query(False),
- source: List[str] = Query(..., description='Data sources to query comma separated with no space'),
- limit: int = Query(500), start: int = Query(0),
- domain: str = Query(..., description='Domain to be harvested')) -> Union[Dict[str, Any], RedirectResponse]:
-
+@app.get("/query", response_class=UJSONResponse)
+@limiter.limit("2/minute")
+async def query(
+ request: Request,
+ dns_server: str = Query(""),
+ user_agent: str = Header(None),
+ dns_brute: bool = Query(False),
+ dns_lookup: bool = Query(False),
+ dns_tld: bool = Query(False),
+ filename: str = Query(""),
+ google_dork: bool = Query(False),
+ proxies: bool = Query(False),
+ shodan: bool = Query(False),
+ take_over: bool = Query(False),
+ virtual_host: bool = Query(False),
+ source: List[str] = Query(
+ ..., description="Data sources to query comma separated with no space"
+ ),
+ limit: int = Query(500),
+ start: int = Query(0),
+ domain: str = Query(..., description="Domain to be harvested"),
+) -> Union[Dict[str, Any], RedirectResponse]:
# Query function that allows user to query theHarvester rest API
# Rate limit of 2 requests per minute
# basic user agent filtering
- if user_agent and ('gobuster' in user_agent or 'sqlmap' in user_agent or 'rustbuster' in user_agent):
- response = RedirectResponse(app.url_path_for('bot'))
+ if user_agent and (
+ "gobuster" in user_agent or "sqlmap" in user_agent or "rustbuster" in user_agent
+ ):
+ response = RedirectResponse(app.url_path_for("bot"))
return response
try:
- asns, iurls, twitter_people_list, \
- linkedin_people_list, linkedin_links, \
- aurls, aips, aemails, ahosts = await __main__.start(argparse.Namespace(dns_brute=dns_brute,
- dns_lookup=dns_lookup,
- dns_server=dns_server,
- dns_tld=dns_tld,
- domain=domain,
- filename=filename,
- google_dork=google_dork,
- limit=limit,
- proxies=proxies,
- shodan=shodan,
- source=','.join(source),
- start=start,
- take_over=take_over,
- virtual_host=virtual_host))
+ (
+ asns,
+ iurls,
+ twitter_people_list,
+ linkedin_people_list,
+ linkedin_links,
+ aurls,
+ aips,
+ aemails,
+ ahosts,
+ ) = await __main__.start(
+ argparse.Namespace(
+ dns_brute=dns_brute,
+ dns_lookup=dns_lookup,
+ dns_server=dns_server,
+ dns_tld=dns_tld,
+ domain=domain,
+ filename=filename,
+ google_dork=google_dork,
+ limit=limit,
+ proxies=proxies,
+ shodan=shodan,
+ source=",".join(source),
+ start=start,
+ take_over=take_over,
+ virtual_host=virtual_host,
+ )
+ )
- return {'asns': asns, 'interesting_urls': iurls,
- 'twitter_people': twitter_people_list,
- 'linkedin_people': linkedin_people_list,
- 'linkedin_links': linkedin_links,
- 'trello_urls': aurls,
- 'ips': aips,
- 'emails': aemails,
- 'hosts': ahosts}
+ return {
+ "asns": asns,
+ "interesting_urls": iurls,
+ "twitter_people": twitter_people_list,
+ "linkedin_people": linkedin_people_list,
+ "linkedin_links": linkedin_links,
+ "trello_urls": aurls,
+ "ips": aips,
+ "emails": aemails,
+ "hosts": ahosts,
+ }
except Exception:
- return {'exception': 'Please contact the server administrator to check the issue'}
+ return {
+ "exception": "Please contact the server administrator to check the issue"
+ }
diff --git a/theHarvester/lib/api/api_example.py b/theHarvester/lib/api/api_example.py
index e9799d77..b81ba7e9 100644
--- a/theHarvester/lib/api/api_example.py
+++ b/theHarvester/lib/api/api_example.py
@@ -3,6 +3,7 @@ Example script to query theHarvester rest API, obtain results, and write out to
"""
import asyncio
+
import aiohttp
import netaddr
@@ -24,92 +25,98 @@ async def main() -> None:
"""
url = "http://127.0.0.1:5000"
domain = "netflix.com"
- query_url = f'{url}/query?limit=300&source=bing,baidu,duckduckgo,dogpile&domain={domain}'
+ query_url = (
+ f"{url}/query?limit=300&source=bing,baidu,duckduckgo,dogpile&domain={domain}"
+ )
async with aiohttp.ClientSession() as session:
fetched_json = await fetch_json(session, query_url)
- total_asns = fetched_json['asns']
- interesting_urls = fetched_json['interesting_urls']
- twitter_people_list_tracker = fetched_json['twitter_people']
- linkedin_people_list_tracker = fetched_json['linkedin_people']
- linkedin_links_tracker = fetched_json['linkedin_links']
- trello_urls = fetched_json['trello_urls']
- ips = fetched_json['ips']
- emails = fetched_json['emails']
- hosts = fetched_json['hosts']
+ total_asns = fetched_json["asns"]
+ interesting_urls = fetched_json["interesting_urls"]
+ twitter_people_list_tracker = fetched_json["twitter_people"]
+ linkedin_people_list_tracker = fetched_json["linkedin_people"]
+ linkedin_links_tracker = fetched_json["linkedin_links"]
+ trello_urls = fetched_json["trello_urls"]
+ ips = fetched_json["ips"]
+ emails = fetched_json["emails"]
+ hosts = fetched_json["hosts"]
if len(total_asns) > 0:
- print(f'\n[*] ASNS found: {len(total_asns)}')
- print('--------------------')
+ print(f"\n[*] ASNS found: {len(total_asns)}")
+ print("--------------------")
total_asns = list(sorted(set(total_asns)))
for asn in total_asns:
print(asn)
if len(interesting_urls) > 0:
- print(f'\n[*] Interesting Urls found: {len(interesting_urls)}')
- print('--------------------')
+ print(f"\n[*] Interesting Urls found: {len(interesting_urls)}")
+ print("--------------------")
interesting_urls = list(sorted(set(interesting_urls)))
for iurl in interesting_urls:
print(iurl)
if len(twitter_people_list_tracker) == 0:
- print('\n[*] No Twitter users found.\n\n')
+ print("\n[*] No Twitter users found.\n\n")
else:
if len(twitter_people_list_tracker) >= 1:
- print('\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker)))
- print('---------------------')
+ print("\n[*] Twitter Users found: " + str(len(twitter_people_list_tracker)))
+ print("---------------------")
twitter_people_list_tracker = list(sorted(set(twitter_people_list_tracker)))
for usr in twitter_people_list_tracker:
print(usr)
if len(linkedin_people_list_tracker) == 0:
- print('\n[*] No LinkedIn users found.\n\n')
+ print("\n[*] No LinkedIn users found.\n\n")
else:
if len(linkedin_people_list_tracker) >= 1:
- print('\n[*] LinkedIn Users found: ' + str(len(linkedin_people_list_tracker)))
- print('---------------------')
- linkedin_people_list_tracker = list(sorted(set(linkedin_people_list_tracker)))
+ print(
+ "\n[*] LinkedIn Users found: " + str(len(linkedin_people_list_tracker))
+ )
+ print("---------------------")
+ linkedin_people_list_tracker = list(
+ sorted(set(linkedin_people_list_tracker))
+ )
for usr in linkedin_people_list_tracker:
print(usr)
if len(linkedin_links_tracker) == 0:
- print(f'\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}')
+ print(f"\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}")
linkedin_links_tracker = list(sorted(set(linkedin_links_tracker)))
- print('---------------------')
+ print("---------------------")
for link in linkedin_links_tracker:
print(link)
length_urls = len(trello_urls)
total = length_urls
- print('\n[*] Trello URLs found: ' + str(total))
- print('--------------------')
+ print("\n[*] Trello URLs found: " + str(total))
+ print("--------------------")
all_urls = list(sorted(set(trello_urls)))
for url in sorted(all_urls):
print(url)
if len(ips) == 0:
- print('\n[*] No IPs found.')
+ print("\n[*] No IPs found.")
else:
- print('\n[*] IPs found: ' + str(len(ips)))
- print('-------------------')
+ print("\n[*] IPs found: " + str(len(ips)))
+ print("-------------------")
# use netaddr as the list may contain ipv4 and ipv6 addresses
ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(ips)])
- print('\n'.join(map(str, ip_list)))
+ print("\n".join(map(str, ip_list)))
if len(emails) == 0:
- print('\n[*] No emails found.')
+ print("\n[*] No emails found.")
else:
- print('\n[*] Emails found: ' + str(len(emails)))
- print('----------------------')
+ print("\n[*] Emails found: " + str(len(emails)))
+ print("----------------------")
all_emails = sorted(list(set(emails)))
- print(('\n'.join(all_emails)))
+ print(("\n".join(all_emails)))
if len(hosts) == 0:
- print('\n[*] No hosts found.\n\n')
+ print("\n[*] No hosts found.\n\n")
else:
- print('\n[*] Hosts found: ' + str(len(hosts)))
- print('---------------------')
- print('\n'.join(hosts))
+ print("\n[*] Hosts found: " + str(len(hosts)))
+ print("---------------------")
+ print("\n".join(hosts))
-if __name__ == '__main__':
+if __name__ == "__main__":
asyncio.run(main())
diff --git a/theHarvester/lib/hostchecker.py b/theHarvester/lib/hostchecker.py
index 735bbaac..219841a0 100644
--- a/theHarvester/lib/hostchecker.py
+++ b/theHarvester/lib/hostchecker.py
@@ -7,14 +7,14 @@ Revised to use aiodns & asyncio on 2019-09-23
# Support for Python3.9
from __future__ import annotations
-import aiodns
import asyncio
import socket
-from typing import Tuple, Any, List, Set
+from typing import Any, List, Set, Tuple
+
+import aiodns
class Checker:
-
def __init__(self, hosts: list, nameserver: list) -> None:
self.hosts = hosts
self.realhosts: List = []
@@ -42,7 +42,7 @@ class Checker:
if addresses == [] or addresses is None or result is None:
return f"{host}:"
else:
- addresses = ','.join(map(str, list(sorted(set(addresses)))))
+ addresses = ",".join(map(str, list(sorted(set(addresses)))))
# addresses = list(sorted(addresses))
return f"{host}:{addresses}"
except Exception:
@@ -53,27 +53,31 @@ class Checker:
def chunks(lst, n):
"""Yield successive n-sized chunks from lst."""
for i in range(0, len(lst), n):
- yield lst[i:i + n]
+ yield lst[i : i + n]
async def query_all(self, resolver, hosts) -> list[Any]:
# TODO chunk list into 50 pieces regardless of IPs and subnets
- results = await asyncio.gather(*[asyncio.create_task(self.resolve_host(host, resolver))
- for host in hosts])
+ results = await asyncio.gather(
+ *[asyncio.create_task(self.resolve_host(host, resolver)) for host in hosts]
+ )
return results
async def check(self):
loop = asyncio.get_event_loop()
- resolver = aiodns.DNSResolver(loop=loop, timeout=8) if len(self.nameserver) == 0 \
+ resolver = (
+ aiodns.DNSResolver(loop=loop, timeout=8)
+ if len(self.nameserver) == 0
else aiodns.DNSResolver(loop=loop, timeout=8, nameservers=self.nameserver)
+ )
all_results = set()
for chunk in self.chunks(self.hosts, 50):
# TODO split this to get IPs added total ips
results = await self.query_all(resolver, chunk)
all_results.update(results)
for pair in results:
- host, addresses = pair.split(':')
+ host, addresses = pair.split(":")
self.realhosts.append(host)
- self.addresses.update({addr for addr in addresses.split(',')})
+ self.addresses.update({addr for addr in addresses.split(",")})
# address may be a list of ips
# and do a set comprehension to remove duplicates
self.realhosts.sort()
diff --git a/theHarvester/lib/version.py b/theHarvester/lib/version.py
index e5ea2a6a..99e39bb2 100644
--- a/theHarvester/lib/version.py
+++ b/theHarvester/lib/version.py
@@ -1,4 +1,5 @@
# coding=utf-8
+
def version() -> str:
- return '4.4.1'
+ return "4.4.1"
diff --git a/theHarvester/parsers/intelxparser.py b/theHarvester/parsers/intelxparser.py
index d61de739..030c451e 100644
--- a/theHarvester/parsers/intelxparser.py
+++ b/theHarvester/parsers/intelxparser.py
@@ -2,7 +2,6 @@ from typing import Set
class Parser:
-
def __init__(self) -> None:
self.emails: Set = set()
self.hosts: Set = set()
@@ -15,16 +14,16 @@ class Parser:
"""
if results is not None:
for dictionary in results["selectors"]:
- field = dictionary['selectorvalue']
- if '@' in field:
+ field = dictionary["selectorvalue"]
+ if "@" in field:
self.emails.add(field)
else:
field = str(field)
- if 'http' in field or 'https' in field:
- if field[:5] == 'https':
+ if "http" in field or "https" in field:
+ if field[:5] == "https":
field = field[8:]
else:
field = field[7:]
- self.hosts.add(field.replace(')', '').replace(',', ''))
+ self.hosts.add(field.replace(")", "").replace(",", ""))
return self.emails, self.hosts
return None, None
diff --git a/theHarvester/parsers/myparser.py b/theHarvester/parsers/myparser.py
index 5de16a24..62b88437 100644
--- a/theHarvester/parsers/myparser.py
+++ b/theHarvester/parsers/myparser.py
@@ -1,36 +1,70 @@
import re
-from typing import Set, List
+from typing import List, Set
class Parser:
-
def __init__(self, results, word) -> None:
self.results = results
self.word = word
self.temp: List = []
async def genericClean(self) -> None:
- self.results = self.results.replace('', '').replace('', '').replace('', '').replace('', '') \
- .replace('%3a', '').replace('', '').replace('', '') \
- .replace('', '').replace('', '')
+ self.results = (
+ self.results.replace("", "")
+ .replace("", "")
+ .replace("", "")
+ .replace("", "")
+ .replace("%3a", "")
+ .replace("", "")
+ .replace("", "")
+ .replace("", "")
+ .replace("", "")
+ )
- for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '%2f', '/', '\\'):
- self.results = self.results.replace(search, ' ')
+ for search in (
+ "<",
+ ">",
+ ":",
+ "=",
+ ";",
+ "&",
+ "%3A",
+ "%3D",
+ "%3C",
+ "%2f",
+ "/",
+ "\\",
+ ):
+ self.results = self.results.replace(search, " ")
async def urlClean(self) -> None:
- self.results = self.results.replace('', '').replace('', '').replace('%2f', '').replace('%3a', '')
- for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C'):
- self.results = self.results.replace(search, ' ')
+ self.results = (
+ self.results.replace("", "")
+ .replace("", "")
+ .replace("%2f", "")
+ .replace("%3a", "")
+ )
+ for search in ("<", ">", ":", "=", ";", "&", "%3A", "%3D", "%3C"):
+ self.results = self.results.replace(search, " ")
async def emails(self):
await self.genericClean()
# Local part is required, charset is flexible.
# https://tools.ietf.org/html/rfc6531 (removed * and () as they provide FP mostly)
- reg_emails = re.compile(r'[a-zA-Z0-9.\-_+#~!$&\',;=:]+' + '@' + '[a-zA-Z0-9.-]*' + self.word.replace('www.', ''))
+ reg_emails = re.compile(
+ r"[a-zA-Z0-9.\-_+#~!$&\',;=:]+"
+ + "@"
+ + "[a-zA-Z0-9.-]*"
+ + self.word.replace("www.", "")
+ )
self.temp = reg_emails.findall(self.results)
emails = await self.unique()
- true_emails = {str(email)[1:].lower().strip() if len(str(email)) > 1 and str(email)[0] == '.'
- else len(str(email)) > 1 and str(email).lower().strip() for email in emails}
+ true_emails = {
+ str(email)[1:].lower().strip()
+ if len(str(email)) > 1 and str(email)[0] == "."
+ else len(str(email)) > 1 and str(email).lower().strip()
+ for email in emails
+ }
# if email starts with dot shift email string and make sure all emails are lowercase
return true_emails
@@ -40,7 +74,11 @@ class Parser:
self.temp = reg_urls.findall(self.results)
allurls = await self.unique()
for iteration in allurls:
- if iteration.count('webcache') or iteration.count('google.com') or iteration.count('search?hl'):
+ if (
+ iteration.count("webcache")
+ or iteration.count("google.com")
+ or iteration.count("search?hl")
+ ):
pass
else:
urls.append(iteration)
@@ -50,11 +88,11 @@ class Parser:
# should check both www. and not www.
hostnames = []
await self.genericClean()
- reg_hosts = re.compile(r'[a-zA-Z0-9.-]*\.' + self.word)
+ reg_hosts = re.compile(r"[a-zA-Z0-9.-]*\." + self.word)
first_hostnames = reg_hosts.findall(self.results)
hostnames.extend(first_hostnames)
# TODO determine if necessary below or if only pass through is fine
- reg_hosts = re.compile(r'[a-zA-Z0-9.-]*\.' + self.word.replace('www.', ''))
+ reg_hosts = re.compile(r"[a-zA-Z0-9.-]*\." + self.word.replace("www.", ""))
# reg_hosts = re.compile(r'www\.[a-zA-Z0-9.-]*\.' + 'www.' + self.word)
# reg_hosts = re.compile(r'www\.[a-zA-Z0-9.-]*\.(?:' + 'www.' + self.word + ')?')
second_hostnames = reg_hosts.findall(self.results)
@@ -62,29 +100,31 @@ class Parser:
return list(set(hostnames))
async def hostnames_all(self):
- reg_hosts = re.compile('(.*?)')
+ reg_hosts = re.compile("(.*?)")
temp = reg_hosts.findall(self.results)
for iteration in temp:
- if iteration.count(':'):
- res = iteration.split(':')[1].split('/')[2]
+ if iteration.count(":"):
+ res = iteration.split(":")[1].split("/")[2]
else:
- res = iteration.split('/')[0]
+ res = iteration.split("/")[0]
self.temp.append(res)
hostnames = await self.unique()
return hostnames
async def set(self):
- reg_sets = re.compile(r'>[a-zA-Z\d]*')
+ reg_sets = re.compile(r">[a-zA-Z\d]*")
self.temp = reg_sets.findall(self.results)
sets = []
for iteration in self.temp:
- delete = iteration.replace('>', '')
- delete = delete.replace('", "")
+ delete = delete.replace(" Set[str]:
- found = re.finditer(r'(http|https)://(www\.)?trello.com/([a-zA-Z\d\-_\.]+/?)*', self.results)
+ found = re.finditer(
+ r"(http|https)://(www\.)?trello.com/([a-zA-Z\d\-_\.]+/?)*", self.results
+ )
urls = {match.group().strip() for match in found}
return urls