From 98fd8db3bcb5e781b2fb53f5689479ff1fd4d421 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Mar 2025 00:45:19 +0000 Subject: [PATCH 01/41] Bump fastapi from 0.115.11 to 0.115.12 Bumps [fastapi](https://github.com/fastapi/fastapi) from 0.115.11 to 0.115.12. - [Release notes](https://github.com/fastapi/fastapi/releases) - [Commits](https://github.com/fastapi/fastapi/compare/0.115.11...0.115.12) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 5a394afa..048d54ff 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -7,7 +7,7 @@ beautifulsoup4==4.13.3 censys==2.2.16 certifi==2025.1.31 dnspython==2.7.0 -fastapi==0.115.11 +fastapi==0.115.12 lxml==5.3.1 netaddr==1.3.0 ujson==5.10.0 From c8c9b012651f00eda310a3ccbf0ba0f72fa6412d Mon Sep 17 00:00:00 2001 From: NotoriousRebel <36310667+NotoriousRebel@users.noreply.github.com> Date: Sun, 30 Mar 2025 22:47:51 -0400 Subject: [PATCH 02/41] Added whoisxml subdomain discovery with API module. --- theHarvester.py | 4 +- theHarvester/__main__.py | 13 +++- theHarvester/data/api-keys.yaml | 3 + theHarvester/discovery/whoisxml.py | 42 ++++++++++++ theHarvester/lib/core.py | 104 ++++++++++++++--------------- 5 files changed, 110 insertions(+), 56 deletions(-) create mode 100644 theHarvester/discovery/whoisxml.py diff --git a/theHarvester.py b/theHarvester.py index 97efe6a8..bc1af5e2 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -4,8 +4,8 @@ import sys from theHarvester.theHarvester import main -if sys.version_info.major < 3 or sys.version_info.minor < 11: - print('\033[93m[!] Make sure you have Python 3.11+ installed, quitting.\n\n \033[0m') +if sys.version_info.major < 3 or sys.version_info.minor < 10: + print('\033[93m[!] Make sure you have Python 3.10+ installed, quitting.\n\n \033[0m') sys.exit(1) if __name__ == '__main__': diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index d65e93c5..3b50c03b 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -50,6 +50,7 @@ from theHarvester.discovery import ( tombasearch, urlscan, virustotal, + whoisxml, yahoosearch, zoomeyesearch, ) @@ -151,7 +152,7 @@ async def start(rest_args: argparse.Namespace | None = None): censys, certspotter, criminalip, crtsh, duckduckgo, fullhunt, github-code, hackertarget, hunter, hunterhow, intelx, netlas, onyphe, otx, pentesttools, projectdiscovery, rapiddns, rocketreach, securityTrails, sitedossier, subdomaincenter, subdomainfinderc99, threatminer, tomba, - urlscan, virustotal, yahoo, zoomeye""", + urlscan, virustotal, yahoo, whoisxml, zoomeye""", ) # determines if filename is coming from rest api or user @@ -760,6 +761,16 @@ async def start(rest_args: argparse.Namespace | None = None): if isinstance(e, MissingKey): print(e) + elif engineitem == 'whoisxml': + try: + whoisxml_search = whoisxml.SearchWhoisXML(word) + stor_lst.append(store(whoisxml_search, engineitem, store_host=True)) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(f'An exception has occurred in WhoisXML search: {e}') + elif engineitem == 'yahoo': try: yahoo_search = yahoosearch.SearchYahoo(word, limit) diff --git a/theHarvester/data/api-keys.yaml b/theHarvester/data/api-keys.yaml index c0098ad5..0fe98b55 100644 --- a/theHarvester/data/api-keys.yaml +++ b/theHarvester/data/api-keys.yaml @@ -61,5 +61,8 @@ apikeys: virustotal: key: + whoisxml: + key: + zoomeye: key: diff --git a/theHarvester/discovery/whoisxml.py b/theHarvester/discovery/whoisxml.py new file mode 100644 index 00000000..edc0b481 --- /dev/null +++ b/theHarvester/discovery/whoisxml.py @@ -0,0 +1,42 @@ +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import AsyncFetcher, Core + + +class SearchWhoisXML: + def __init__(self, word) -> None: + self.word = word + self.key = Core.whoisxml_key() + if self.key is None: + raise MissingKey('whoisxml') + self.total_results = None + self.proxy = False + + async def do_search(self): + # https://subdomains.whoisxmlapi.com/api/documentation/making-requests + url = f'https://subdomains.whoisxmlapi.com/api/v1' + params = { + 'apiKey': self.key, + 'domainName': self.word + } + response = await AsyncFetcher.fetch_all( + [url], + json=True, + params=params, + headers={'User-Agent': Core.get_user_agent()}, + proxy=self.proxy, + ) + # Parse the response according to the example JSON structure: + # {"search":"example.com.com","result":{"count":10000,"records":[{"domain":"test.example.com","firstSeen":1678169400,"lastSeen":1678169400}]}} + self.total_results = [] + print(response[0]) + if response and response[0]: + # Extract domains from the records array + if 'result' in response[0] and 'records' in response[0]['result']: + self.total_results = [record['domain'] for record in response[0]['result']['records']] + + async def get_hostnames(self): + return self.total_results + + async def process(self, proxy: bool = False) -> None: + self.proxy = proxy + await self.do_search() diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 8ff38488..a274ab68 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -134,6 +134,10 @@ class Core: def virustotal_key() -> str: return Core.api_keys()['virustotal']['key'] + @staticmethod + def whoisxml_key() -> str: + return Core.api_keys()['whoisxml']['key'] + @staticmethod def proxy_list() -> list: keys = yaml.safe_load(Core._read_config('proxies.yaml')) @@ -193,6 +197,7 @@ class Core: 'tomba', 'urlscan', 'virustotal', + 'whoisxml', 'yahoo', 'zoomeye', ] @@ -201,61 +206,54 @@ class Core: @staticmethod def get_user_agent() -> str: # User-Agents from https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ - # Lasted updated 7/2/23 + # Lasted updated 3/30/25 # TODO use bs4 to auto parse user agents user_agents = [ - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0', - 'Mozilla/5.0 (Windows NT 10.0; rv:114.0) Gecko/20100101 Firefox/114.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.43', - 'Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 OPR/99.0.0.0', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/113.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.51', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.58', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.37', - 'Mozilla/5.0 (Windows NT 10.0; rv:113.0) Gecko/20100101 Firefox/113.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5.1 Safari/605.1.15', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.57', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.41', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 OPR/98.0.0.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 YaBrowser/23.5.2.625 Yowser/2.5 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.6.1 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0', - 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0', - 'Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4590.2 Mobile Safari/537.36 Chrome-Lighthouse', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:136.0) Gecko/20100101 Firefox/136.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3.1 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Safari/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:135.0) Gecko/20100101 Firefox/135.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 YaBrowser/25.2.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 OPR/117.0.0.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 GLS/100.10.9939.100", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:137.0) Gecko/20100101 Firefox/137.0", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.3", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0" ] return random.choice(user_agents) From 5bc1a88e9418b78c362af89c3aadc624c5379cda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Apr 2025 21:35:39 +0000 Subject: [PATCH 03/41] Bump aiohttp from 3.11.13 to 3.11.16 Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.11.13 to 3.11.16. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.11.13...v3.11.16) --- updated-dependencies: - dependency-name: aiohttp dependency-version: 3.11.16 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 5a394afa..a6e714cf 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,6 +1,6 @@ aiodns==3.2.0 aiofiles==24.1.0 -aiohttp==3.11.13 +aiohttp==3.11.16 aiomultiprocess==0.9.1 aiosqlite==0.21.0 beautifulsoup4==4.13.3 From 57c1ac11458c1458dbf616c5b470e4cc0fdfa71d Mon Sep 17 00:00:00 2001 From: Lee Baird Date: Wed, 2 Apr 2025 17:00:00 -0500 Subject: [PATCH 04/41] Added WhoisXML info. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ecd5360a..e7219647 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,8 @@ Passive modules: * virustotal: Domain search (Requires an API key, see below.) - https://www.virustotal.com +* whoisxml: Subdomain search (Requires an API key, see below.) - https://subdomains.whoisxmlapi.com/api/pricing + * yahoo: Yahoo search engine * zoomeye: China's version of Shodan (Requires an API key, see below.) - https://www.zoomeye.org @@ -123,6 +125,7 @@ Documentation to setup API keys can be found at - https://github.com/laramies/th * securityTrails * shodan - $ * tomba - Free up to 50 search. +* whoisxml * zoomeye Install and dependencies: From 4cb7f50f46e36ac50708c3454b638c87a3305ad0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Apr 2025 21:31:21 +0000 Subject: [PATCH 05/41] Bump ruff from 0.10.0 to 0.11.4 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.10.0 to 0.11.4. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.10.0...0.11.4) --- updated-dependencies: - dependency-name: ruff dependency-version: 0.11.4 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index a85af242..4c1f2a95 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,5 +8,5 @@ types-ujson==5.10.0.20240515 types-PyYAML==6.0.12.20241230 types-requests==2.32.0.20250306 # 2.31.0.7 introduced a regression types-python-dateutil==2.9.0.20241206 -ruff==0.10.0 +ruff==0.11.4 wheel==0.45.1 \ No newline at end of file From b7e43302022446271c245ec03cb1f083289bbd0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Apr 2025 21:28:40 +0000 Subject: [PATCH 06/41] Bump ruff from 0.11.4 to 0.11.5 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.11.4...0.11.5) --- updated-dependencies: - dependency-name: ruff dependency-version: 0.11.5 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 4c1f2a95..fe04b24c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,5 +8,5 @@ types-ujson==5.10.0.20240515 types-PyYAML==6.0.12.20241230 types-requests==2.32.0.20250306 # 2.31.0.7 introduced a regression types-python-dateutil==2.9.0.20241206 -ruff==0.11.4 +ruff==0.11.5 wheel==0.45.1 \ No newline at end of file From 8818611301def510dc9b55d742a593d1803276a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Apr 2025 08:19:53 +0000 Subject: [PATCH 07/41] Bump lxml from 5.3.1 to 5.3.2 Bumps [lxml](https://github.com/lxml/lxml) from 5.3.1 to 5.3.2. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-5.3.1...lxml-5.3.2) --- updated-dependencies: - dependency-name: lxml dependency-version: 5.3.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 2e6b9048..88cfb8af 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,7 +8,7 @@ censys==2.2.16 certifi==2025.1.31 dnspython==2.7.0 fastapi==0.115.12 -lxml==5.3.1 +lxml==5.3.2 netaddr==1.3.0 ujson==5.10.0 playwright==1.51.0 From bf74db6a4e4af888fabc470eb15b453ee5763db4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Mar 2025 22:27:34 +0000 Subject: [PATCH 08/41] Bump pytest-asyncio from 0.25.3 to 0.26.0 Bumps [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) from 0.25.3 to 0.26.0. - [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases) - [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.25.3...v0.26.0) --- updated-dependencies: - dependency-name: pytest-asyncio dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index fe04b24c..864a0576 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,7 +1,7 @@ mypy==1.15.0 mypy-extensions==1.0.0 pytest==8.3.5 -pytest-asyncio==0.25.3 +pytest-asyncio==0.26.0 types-certifi==2021.10.8.3 types-chardet==5.0.4.6 types-ujson==5.10.0.20240515 From 63c7aeaee77ef5ebea4c6d921b8fe4ade60082bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 13 Apr 2025 17:11:41 +0000 Subject: [PATCH 09/41] Bump types-ujson from 5.10.0.20240515 to 5.10.0.20250326 Bumps [types-ujson](https://github.com/python/typeshed) from 5.10.0.20240515 to 5.10.0.20250326. - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-ujson dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 864a0576..28bfb6ff 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,7 +4,7 @@ pytest==8.3.5 pytest-asyncio==0.26.0 types-certifi==2021.10.8.3 types-chardet==5.0.4.6 -types-ujson==5.10.0.20240515 +types-ujson==5.10.0.20250326 types-PyYAML==6.0.12.20241230 types-requests==2.32.0.20250306 # 2.31.0.7 introduced a regression types-python-dateutil==2.9.0.20241206 From aa7c6deaea692f2488ab64f81abef8cdd9cb4de8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Apr 2025 08:19:18 +0000 Subject: [PATCH 10/41] Bump types-requests from 2.32.0.20250306 to 2.32.0.20250328 Bumps [types-requests](https://github.com/python/typeshed) from 2.32.0.20250306 to 2.32.0.20250328. - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 28bfb6ff..cb78400f 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,7 +6,7 @@ types-certifi==2021.10.8.3 types-chardet==5.0.4.6 types-ujson==5.10.0.20250326 types-PyYAML==6.0.12.20241230 -types-requests==2.32.0.20250306 # 2.31.0.7 introduced a regression +types-requests==2.32.0.20250328 # 2.31.0.7 introduced a regression types-python-dateutil==2.9.0.20241206 ruff==0.11.5 wheel==0.45.1 \ No newline at end of file From 9d578cb02fc73efae9d5e10b0559b499f0fea75c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 13 Apr 2025 17:27:35 +0000 Subject: [PATCH 11/41] Bump types-pyyaml from 6.0.12.20241230 to 6.0.12.20250402 Bumps [types-pyyaml](https://github.com/typeshed-internal/stub_uploader) from 6.0.12.20241230 to 6.0.12.20250402. - [Commits](https://github.com/typeshed-internal/stub_uploader/commits) --- updated-dependencies: - dependency-name: types-pyyaml dependency-version: 6.0.12.20250402 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index cb78400f..3c3a73ed 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,7 +5,7 @@ pytest-asyncio==0.26.0 types-certifi==2021.10.8.3 types-chardet==5.0.4.6 types-ujson==5.10.0.20250326 -types-PyYAML==6.0.12.20241230 +types-PyYAML==6.0.12.20250402 types-requests==2.32.0.20250328 # 2.31.0.7 introduced a regression types-python-dateutil==2.9.0.20241206 ruff==0.11.5 From 4b82e2ee9ae454f68e43e026b7d69395a2a82e0b Mon Sep 17 00:00:00 2001 From: Anis Mekacher Date: Sun, 13 Apr 2025 20:06:56 +0200 Subject: [PATCH 12/41] Added venacus data leak search engine plugin. (#1968) * Added venacus data leak search engine plugin. * Added venacus key to api-keys template. * Run ruff check fix and format. --------- Co-authored-by: Anis Mekacher --- README.md | 3 + theHarvester/__main__.py | 37 +++++++- theHarvester/data/api-keys.yaml | 3 + theHarvester/discovery/venacussearch.py | 93 +++++++++++++++++++ theHarvester/discovery/whoisxml.py | 7 +- theHarvester/lib/core.py | 95 ++++++++++---------- theHarvester/lib/stash.py | 4 + theHarvester/parsers/venacusparser.py | 114 ++++++++++++++++++++++++ 8 files changed, 304 insertions(+), 52 deletions(-) create mode 100644 theHarvester/discovery/venacussearch.py create mode 100644 theHarvester/parsers/venacusparser.py diff --git a/README.md b/README.md index e7219647..1e183fc0 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,8 @@ Passive modules: * zoomeye: China's version of Shodan (Requires an API key, see below.) - https://www.zoomeye.org +* venacus: Venacus search engine (Requires an API key, see below.) - https://venacus.com + Active modules: --------------- @@ -127,6 +129,7 @@ Documentation to setup API keys can be found at - https://github.com/laramies/th * tomba - Free up to 50 search. * whoisxml * zoomeye +* venacus - $ Install and dependencies: ------------------------- diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 3b50c03b..6b848d6c 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -49,6 +49,7 @@ from theHarvester.discovery import ( threatminer, tombasearch, urlscan, + venacussearch, virustotal, whoisxml, yahoosearch, @@ -152,7 +153,7 @@ async def start(rest_args: argparse.Namespace | None = None): censys, certspotter, criminalip, crtsh, duckduckgo, fullhunt, github-code, hackertarget, hunter, hunterhow, intelx, netlas, onyphe, otx, pentesttools, projectdiscovery, rapiddns, rocketreach, securityTrails, sitedossier, subdomaincenter, subdomainfinderc99, threatminer, tomba, - urlscan, virustotal, yahoo, whoisxml, zoomeye""", + urlscan, virustotal, yahoo, whoisxml, zoomeye, venacus""", ) # determines if filename is coming from rest api or user @@ -186,6 +187,7 @@ async def start(rest_args: argparse.Namespace | None = None): all_emails: list = [] all_hosts: list = [] all_ip: list = [] + all_people: list[dict[str, str]] = [] dnslookup = args.dns_lookup dnsserver = args.dns_server # TODO arg is not used anywhere replace with resolvers wordlist arg dnsresolve dnsresolve = args.dns_resolve @@ -338,6 +340,7 @@ async def start(rest_args: argparse.Namespace | None = None): if store_people: people_list = await search_engine.get_people() + all_people.extend(people_list) await db_stash.store_all(word, people_list, 'people', source) if store_links: @@ -802,6 +805,25 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): print(e) + + elif engineitem == 'venacus': + try: + venacus_search = venacussearch.SearchVenacus(word=word, limit=limit, offset_doc=start) + stor_lst.append( + store( + venacus_search, + engineitem, + store_emails=True, + store_ip=True, + store_people=True, + store_interestingurls=True, + ) + ) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(f'An exception has occurred in venacus search: {e}') else: if rest_args is not None: try: @@ -813,7 +835,7 @@ async def start(rest_args: argparse.Namespace | None = None): # Print which engines aren't supported unsupported_engines = set(engines) - set(Core.get_supportedengines()) if unsupported_engines: - print(f"The following engines are not supported: {unsupported_engines}") + print(f'The following engines are not supported: {unsupported_engines}') print('\n[!] Invalid source.\n') sys.exit(1) @@ -963,6 +985,14 @@ async def start(rest_args: argparse.Namespace | None = None): all_emails = sorted(list(set(all_emails))) print('\n'.join(all_emails)) + if len(all_people) == 0: + print('\n[*] No people found.') + else: + print('\n[*] People found: ' + str(len(all_people))) + print('----------------------') + for person in all_people: + print(person) + if len(all_hosts) == 0: print('\n[*] No hosts found.\n\n') else: @@ -1242,6 +1272,9 @@ async def start(rest_args: argparse.Namespace | None = None): if len(linkedin_links_tracker) > 0: json_dict['linkedin_links'] = linkedin_links_tracker + if len(all_people) > 0: + json_dict['people'] = all_people + if takeover_status and len(takeover_results) > 0: json_dict['takeover_results'] = takeover_results diff --git a/theHarvester/data/api-keys.yaml b/theHarvester/data/api-keys.yaml index 0fe98b55..bfa0e875 100644 --- a/theHarvester/data/api-keys.yaml +++ b/theHarvester/data/api-keys.yaml @@ -66,3 +66,6 @@ apikeys: zoomeye: key: + + venacus: + key: \ No newline at end of file diff --git a/theHarvester/discovery/venacussearch.py b/theHarvester/discovery/venacussearch.py new file mode 100644 index 00000000..f73703c8 --- /dev/null +++ b/theHarvester/discovery/venacussearch.py @@ -0,0 +1,93 @@ +from typing import Any + +import aiohttp + +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import Core +from theHarvester.parsers import venacusparser + + +class SearchVenacus: + def __init__(self, word: str, limit=1000, offset_doc=0) -> None: + self.word = word + self.key = Core.venacus_key() + if self.key is None: + raise MissingKey('Venacus') + self.base_url = 'https://api.venacus.com' + self.results: list[dict[str, Any]] = [] + self.parsed: dict[str, Any] = {} + self.proxy = False + self.offset_doc = offset_doc + self.offset_in_doc = 0 + self.ai = False + self.more = True + self.limit = limit + + async def do_search(self) -> None: + total_results = [] + result_count = 0 + + try: + headers = { + 'Authorization': f'Bearer {self.key}', + 'User-Agent': f'{Core.get_user_agent()}-theHarvester', + } + + async with aiohttp.ClientSession() as session: + while self.more and result_count < self.limit: + query = { + 'q': self.word, + 'offset_doc': self.offset_doc, + 'offset_in_doc': self.offset_in_doc, + 'limit': 100, + 'ai': 'true' if self.ai else 'false', + } + + async with session.get(f'{self.base_url}/v1/search/', headers=headers, params=query) as total_resp: + search_data = await total_resp.json() + current_results = search_data.get('data', []) + + if not current_results: + print('No more results found.') + break + + total_results.extend(current_results) + result_count += len(current_results) + + self.offset_doc = search_data.get('offset_doc', 0) + self.offset_in_doc = search_data.get('offset_in_doc', 0) + + self.more = search_data.get('more', False) + + self.results = total_results + if not self.results: + print('No results found.') + + except Exception as e: + print(f'An exception has occurred in Venacus: {e}') + + async def process(self, proxy: bool = False): + self.proxy = proxy + await self.do_search() + parser = venacusparser.Parser() + self.parsed = await parser.parse_text_tokens(self.results) # type: ignore + + async def get_people(self) -> list[dict[str, str]]: + if 'people' not in self.parsed: + return [] + return self.parsed['people'] + + async def get_emails(self) -> set[str]: + if 'emails' not in self.parsed: + return set() + return self.parsed['emails'] + + async def get_ips(self) -> set[str]: + if 'ips' not in self.parsed: + return set() + return self.parsed['ips'] + + async def get_interestingurls(self) -> set[str]: + if 'urls' not in self.parsed: + return set() + return self.parsed['urls'] diff --git a/theHarvester/discovery/whoisxml.py b/theHarvester/discovery/whoisxml.py index edc0b481..1faaf6bb 100644 --- a/theHarvester/discovery/whoisxml.py +++ b/theHarvester/discovery/whoisxml.py @@ -13,11 +13,8 @@ class SearchWhoisXML: async def do_search(self): # https://subdomains.whoisxmlapi.com/api/documentation/making-requests - url = f'https://subdomains.whoisxmlapi.com/api/v1' - params = { - 'apiKey': self.key, - 'domainName': self.word - } + url = 'https://subdomains.whoisxmlapi.com/api/v1' + params = {'apiKey': self.key, 'domainName': self.word} response = await AsyncFetcher.fetch_all( [url], json=True, diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index a274ab68..3fc4a64b 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -138,6 +138,10 @@ class Core: def whoisxml_key() -> str: return Core.api_keys()['whoisxml']['key'] + @staticmethod + def venacus_key() -> str: + return Core.api_keys()['venacus']['key'] + @staticmethod def proxy_list() -> list: keys = yaml.safe_load(Core._read_config('proxies.yaml')) @@ -200,6 +204,7 @@ class Core: 'whoisxml', 'yahoo', 'zoomeye', + 'venacus', ] return supportedengines @@ -209,51 +214,51 @@ class Core: # Lasted updated 3/30/25 # TODO use bs4 to auto parse user agents user_agents = [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:136.0) Gecko/20100101 Firefox/136.0", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0", - "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3.1 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0", - "Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Safari/605.1.15", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:135.0) Gecko/20100101 Firefox/135.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 YaBrowser/25.2.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 OPR/117.0.0.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 GLS/100.10.9939.100", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:137.0) Gecko/20100101 Firefox/137.0", - "Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.3", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0" + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:136.0) Gecko/20100101 Firefox/136.0', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0', + 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0', + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3.1 Safari/605.1.15', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0', + 'Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Safari/605.1.15', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:135.0) Gecko/20100101 Firefox/135.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 YaBrowser/25.2.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 OPR/117.0.0.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 GLS/100.10.9939.100', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15', + 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:137.0) Gecko/20100101 Firefox/137.0', + 'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko)', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15', + 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.3', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Safari/605.1.15', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0', ] return random.choice(user_agents) diff --git a/theHarvester/lib/stash.py b/theHarvester/lib/stash.py index 1e12f881..c1ff2b3c 100644 --- a/theHarvester/lib/stash.py +++ b/theHarvester/lib/stash.py @@ -47,6 +47,10 @@ class StashManager: print(e) async def store_all(self, domain, all, res_type, source) -> None: + # people are not stored in the database + if res_type == 'people': + return + self.domain = domain self.all = all self.type = res_type diff --git a/theHarvester/parsers/venacusparser.py b/theHarvester/parsers/venacusparser.py new file mode 100644 index 00000000..56af79dc --- /dev/null +++ b/theHarvester/parsers/venacusparser.py @@ -0,0 +1,114 @@ +import enum +from collections.abc import Mapping +from typing import Any + + +class TokenTypesEnum(str, enum.Enum): + ID = 'id' + FIRSTNAME = 'firstname' + LASTNAME = 'lastname' + EMAIL = 'email' + DOB = 'dob' + URL = 'url' + PHONE = 'phone' + DATE = 'date' + TIME = 'time' + IP = 'ip_address' + HASH = 'hash' + PASSWORD = 'password' + ADDRESS = 'address' + COMPANY = 'company' + JOB_TITLE = 'job_title' + USERNAME = 'username' + COUNTRY = 'country' + CITY = 'city' + STATE = 'state' + ZIP_CODE = 'zip_code' + CURRENCY = 'currency' + INDUSTRY = 'industry' + DEPARTMENT = 'department' + ROLE = 'role' + + +class Parser: + def __init__(self) -> None: + self.parsed_data: dict[str, set[str]] = {} + self.people: list[dict[str, str]] = [] + + async def parse_text_tokens(self, results: list[dict[str, Any]]) -> Mapping[str, set[str] | list[dict[str, str]]]: + """ + Extracts different types of information from the recognized text tokens + """ + if not results: + return {'people': set(), 'emails': set(), 'ips': set(), 'urls': set()} + + for res in results: + person: dict[str, str] | None = None + for token in res['tokens']: + if token['type'] == TokenTypesEnum.EMAIL: + if 'emails' not in self.parsed_data: + self.parsed_data['emails'] = set() + self.parsed_data['emails'].add(token['value']) + person = person or {} + person['email'] = token['value'] + elif token['type'] == TokenTypesEnum.IP: + if 'ips' not in self.parsed_data: + self.parsed_data['ips'] = set() + self.parsed_data['ips'].add(token['value']) + elif token['type'] == TokenTypesEnum.URL: + if 'urls' not in self.parsed_data: + self.parsed_data['urls'] = set() + self.parsed_data['urls'].add(token['value']) + elif token['type'] == TokenTypesEnum.FIRSTNAME: + person = person or {} + person['firstname'] = token['value'] + elif token['type'] == TokenTypesEnum.LASTNAME: + person = person or {} + person['lastname'] = token['value'] + elif token['type'] == TokenTypesEnum.COMPANY: + person = person or {} + person['company'] = token['value'] + elif token['type'] == TokenTypesEnum.CITY: + person = person or {} + person['city'] = token['value'] + elif token['type'] == TokenTypesEnum.STATE: + person = person or {} + person['state'] = token['value'] + elif token['type'] == TokenTypesEnum.COUNTRY: + person = person or {} + person['country'] = token['value'] + elif token['type'] == TokenTypesEnum.ZIP_CODE: + person = person or {} + person['zip_code'] = token['value'] + elif token['type'] == TokenTypesEnum.PHONE: + person = person or {} + person['phone'] = token['value'] + elif token['type'] == TokenTypesEnum.ADDRESS: + person = person or {} + person['address'] = token['value'] + elif token['type'] == TokenTypesEnum.ROLE: + person = person or {} + person['role'] = token['value'] + elif token['type'] == TokenTypesEnum.DOB: + person = person or {} + person['dob'] = token['value'] + elif token['type'] == TokenTypesEnum.JOB_TITLE: + person = person or {} + person['job_title'] = token['value'] + elif token['type'] == TokenTypesEnum.INDUSTRY: + person = person or {} + person['industry'] = token['value'] + elif token['type'] == TokenTypesEnum.DEPARTMENT: + person = person or {} + person['department'] = token['value'] + + if person: + for key in person: + if key != 'email': + self.people.append(person) + break + + if self.people: + self.parsed_data['people'] = self.people # type: ignore + + return self.parsed_data From 7a8bd96cec1729e17f94f65858abcc390aade028 Mon Sep 17 00:00:00 2001 From: "J.Townsend" Date: Sun, 13 Apr 2025 19:12:47 +0100 Subject: [PATCH 13/41] bump version --- theHarvester/lib/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theHarvester/lib/version.py b/theHarvester/lib/version.py index d179a573..23e97e40 100644 --- a/theHarvester/lib/version.py +++ b/theHarvester/lib/version.py @@ -1,4 +1,4 @@ -VERSION = '4.7.0' +VERSION = '4.7.1' def version() -> str: From af6dc8fdfdcd3b32ab27b2377683b9b3d698a4a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 22:11:01 +0000 Subject: [PATCH 14/41] Bump uvicorn from 0.34.0 to 0.34.1 Bumps [uvicorn](https://github.com/encode/uvicorn) from 0.34.0 to 0.34.1. - [Release notes](https://github.com/encode/uvicorn/releases) - [Changelog](https://github.com/encode/uvicorn/blob/master/docs/release-notes.md) - [Commits](https://github.com/encode/uvicorn/compare/0.34.0...0.34.1) --- updated-dependencies: - dependency-name: uvicorn dependency-version: 0.34.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 88cfb8af..3de2fc7d 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -18,6 +18,6 @@ requests==2.32.3 retrying==1.3.4 shodan==1.31.0 slowapi==0.1.9 -uvicorn==0.34.0 +uvicorn==0.34.1 uvloop==0.21.0; platform_system != "Windows" winloop==0.1.8; platform_system == "Windows" From dd6f787c4124ec194a0276c1258e1e8cc073092e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Apr 2025 21:12:51 +0000 Subject: [PATCH 15/41] Bump censys from 2.2.16 to 2.2.17 Bumps [censys](https://github.com/censys/censys-python) from 2.2.16 to 2.2.17. - [Release notes](https://github.com/censys/censys-python/releases) - [Commits](https://github.com/censys/censys-python/compare/v2.2.16...v2.2.17) --- updated-dependencies: - dependency-name: censys dependency-version: 2.2.17 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 3de2fc7d..c5f29231 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,7 +4,7 @@ aiohttp==3.11.16 aiomultiprocess==0.9.1 aiosqlite==0.21.0 beautifulsoup4==4.13.3 -censys==2.2.16 +censys==2.2.17 certifi==2025.1.31 dnspython==2.7.0 fastapi==0.115.12 From 47164c9cff8534137c34f4480f50c39cf15f7b44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Apr 2025 23:48:56 +0000 Subject: [PATCH 16/41] Bump beautifulsoup4 from 4.13.3 to 4.13.4 Bumps [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/bs4/) from 4.13.3 to 4.13.4. --- updated-dependencies: - dependency-name: beautifulsoup4 dependency-version: 4.13.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index c5f29231..5576455e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -3,7 +3,7 @@ aiofiles==24.1.0 aiohttp==3.11.16 aiomultiprocess==0.9.1 aiosqlite==0.21.0 -beautifulsoup4==4.13.3 +beautifulsoup4==4.13.4 censys==2.2.17 certifi==2025.1.31 dnspython==2.7.0 From dfccf42527a10a6c5d4024d12aef823522a2e23c Mon Sep 17 00:00:00 2001 From: Lee Baird Date: Wed, 16 Apr 2025 19:27:34 -0500 Subject: [PATCH 17/41] Merge pull request #1972 * Alphabetized new module. * Merge branch 'laramies:master' into master --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1e183fc0..c96e8e63 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,8 @@ Passive modules: * vhost: Bing virtual hosts search +* venacus: Venacus search engine (Requires an API key, see below.) - https://venacus.com + * virustotal: Domain search (Requires an API key, see below.) - https://www.virustotal.com * whoisxml: Subdomain search (Requires an API key, see below.) - https://subdomains.whoisxmlapi.com/api/pricing @@ -96,9 +98,6 @@ Passive modules: * zoomeye: China's version of Shodan (Requires an API key, see below.) - https://www.zoomeye.org -* venacus: Venacus search engine (Requires an API key, see below.) - https://venacus.com - - Active modules: --------------- * DNS brute force: dictionary brute force enumeration @@ -127,16 +126,15 @@ Documentation to setup API keys can be found at - https://github.com/laramies/th * securityTrails * shodan - $ * tomba - Free up to 50 search. +* venacus - $ * whoisxml * zoomeye -* venacus - $ Install and dependencies: ------------------------- * Python 3.11+ * https://github.com/laramies/theHarvester/wiki/Installation - Comments, bugs, and requests: ----------------------------- * [![Twitter Follow](https://img.shields.io/twitter/follow/laramies.svg?style=social&label=Follow)](https://twitter.com/laramies) Christian Martorella @laramies @@ -144,7 +142,6 @@ Comments, bugs, and requests: * [![Twitter Follow](https://img.shields.io/twitter/follow/NotoriousRebel1.svg?style=social&label=Follow)](https://twitter.com/NotoriousRebel1) Matthew Brown @NotoriousRebel1 * [![Twitter Follow](https://img.shields.io/twitter/follow/jay_townsend1.svg?style=social&label=Follow)](https://twitter.com/jay_townsend1) Jay "L1ghtn1ng" Townsend @jay_townsend1 - Main contributors: ------------------ * [![Twitter Follow](https://img.shields.io/twitter/follow/NotoriousRebel1.svg?style=social&label=Follow)](https://twitter.com/NotoriousRebel1) Matthew Brown @NotoriousRebel1 From c0486876d971568019a955dd00c62c3fe67cfcce Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Thu, 17 Apr 2025 01:42:17 +0100 Subject: [PATCH 18/41] Enhance RocketReach to extract and store email addresses. Added functionality to collect email addresses from RocketReach search results. Updated theHarvester's main process to store these emails alongside existing links, improving data extraction capabilities. --- theHarvester/__main__.py | 2 +- theHarvester/discovery/rocketreach.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 6b848d6c..b13fd3ff 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -665,7 +665,7 @@ async def start(rest_args: argparse.Namespace | None = None): elif engineitem == 'rocketreach': try: rocketreach_search = rocketreach.SearchRocketReach(word, limit) - stor_lst.append(store(rocketreach_search, engineitem, store_links=True)) + stor_lst.append(store(rocketreach_search, engineitem, store_links=True, store_emails=True)) except Exception as e: if isinstance(e, MissingKey): print(e) diff --git a/theHarvester/discovery/rocketreach.py b/theHarvester/discovery/rocketreach.py index 0b904d81..4096de35 100644 --- a/theHarvester/discovery/rocketreach.py +++ b/theHarvester/discovery/rocketreach.py @@ -15,6 +15,7 @@ class SearchRocketReach: self.proxy = False self.baseurl = 'https://rocketreach.co/api/v2/person/search' self.links: set = set() + self.emails: set = set() self.limit = limit async def do_search(self) -> None: @@ -45,6 +46,10 @@ class SearchRocketReach: for profile in result['profiles']: if 'linkedin_url' in dict(profile).keys(): self.links.add(profile['linkedin_url']) + if 'emails' in dict(profile).keys() and profile['emails']: + for email in profile['emails']: + if 'email' in email and email['email']: + self.emails.add(email['email']) if 'pagination' in dict(result).keys(): next_page = result['pagination']['page'] + 1 if next_page > result['pagination']['total_pages']: @@ -58,6 +63,9 @@ class SearchRocketReach: async def get_links(self): return self.links + async def get_emails(self): + return self.emails + async def process(self, proxy: bool = False) -> None: self.proxy = proxy await self.do_search() From 6757c510cdf176320a2ec86214dca9bd88d47ad4 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Thu, 17 Apr 2025 01:49:06 +0100 Subject: [PATCH 19/41] Remove BinaryEdge integration from theHarvester. This commit removes all code, references, and documentation related to the BinaryEdge module. The BinaryEdge API functionality is no longer supported, including its API key requirement and search capabilities. Updated dependencies, README, and source references to reflect this change. --- README.md | 5 +-- theHarvester/__main__.py | 12 ++----- theHarvester/data/api-keys.yaml | 3 -- theHarvester/discovery/binaryedgesearch.py | 42 ---------------------- theHarvester/lib/core.py | 5 --- 5 files changed, 3 insertions(+), 64 deletions(-) delete mode 100644 theHarvester/discovery/binaryedgesearch.py diff --git a/README.md b/README.md index c96e8e63..473d9982 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,6 @@ Passive modules: * baidu: Baidu search engine - www.baidu.com -* binaryedge: List of known subdomains (Requires an API key, see below.) - https://www.binaryedge.io - * bing: Microsoft search engine - https://www.bing.com * bingapi: Microsoft search engine, through the API (Requires an API key, see below.) @@ -108,9 +106,8 @@ Modules that require an API key: Documentation to setup API keys can be found at - https://github.com/laramies/theHarvester/wiki/Installation#api-keys * bevigil - Free upto 50 queries. Pricing can be found here: https://bevigil.com/pricing/osint -* binaryedge - $10/month * bing -* bufferoverun - uses the free API +* bufferoverun - uses the free binaAPI * censys - API keys are required and can be retrieved from your [Censys account](https://search.censys.io/account/api). * criminalip * fullhunt diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index b13fd3ff..88a3c498 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -17,7 +17,6 @@ from theHarvester.discovery import ( anubis, baidusearch, bevigil, - binaryedgesearch, bingsearch, bravesearch, bufferoverun, @@ -149,14 +148,14 @@ async def start(rest_args: argparse.Namespace | None = None): parser.add_argument( '-b', '--source', - help="""anubis, baidu, bevigil, binaryedge, bing, bingapi, brave, bufferoverun, + help="""anubis, baidu, bevigil, bing, bingapi, brave, bufferoverun, censys, certspotter, criminalip, crtsh, duckduckgo, fullhunt, github-code, hackertarget, hunter, hunterhow, intelx, netlas, onyphe, otx, pentesttools, projectdiscovery, rapiddns, rocketreach, securityTrails, sitedossier, subdomaincenter, subdomainfinderc99, threatminer, tomba, urlscan, virustotal, yahoo, whoisxml, zoomeye, venacus""", ) - # determines if filename is coming from rest api or user + # determines if the filename is coming from rest api or user rest_filename = '' # indicates this from the rest API if rest_args: @@ -407,13 +406,6 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: print(e) - elif engineitem == 'binaryedge': - try: - binaryedge_search = binaryedgesearch.SearchBinaryEdge(word, limit) - stor_lst.append(store(binaryedge_search, engineitem, store_host=True)) - except Exception as e: - print(e) - elif engineitem == 'bing' or engineitem == 'bingapi': try: bing_search = bingsearch.SearchBing(word, limit, start) diff --git a/theHarvester/data/api-keys.yaml b/theHarvester/data/api-keys.yaml index bfa0e875..bf14798f 100644 --- a/theHarvester/data/api-keys.yaml +++ b/theHarvester/data/api-keys.yaml @@ -2,9 +2,6 @@ apikeys: bevigil: key: - binaryedge: - key: - bing: key: diff --git a/theHarvester/discovery/binaryedgesearch.py b/theHarvester/discovery/binaryedgesearch.py deleted file mode 100644 index 895e0cd4..00000000 --- a/theHarvester/discovery/binaryedgesearch.py +++ /dev/null @@ -1,42 +0,0 @@ -import asyncio - -from theHarvester.discovery.constants import MissingKey, get_delay -from theHarvester.lib.core import AsyncFetcher, Core - - -class SearchBinaryEdge: - def __init__(self, word, limit) -> None: - self.word = word - self.totalhosts: set = set() - self.proxy = False - self.key = Core.binaryedge_key() - self.limit = 501 if limit >= 501 else limit - self.limit = 2 if self.limit == 1 else self.limit - if self.key is None: - raise MissingKey('binaryedge') - - async def do_search(self) -> None: - base_url = f'https://api.binaryedge.io/v2/query/domains/subdomain/{self.word}' - headers = {'X-KEY': self.key, 'User-Agent': Core.get_user_agent()} - for page in range(1, self.limit): - params = {'page': page} - response = await AsyncFetcher.fetch_all([base_url], json=True, proxy=self.proxy, params=params, headers=headers) - responses = response[0] - dct = responses - if ('status' in dct.keys() and 'message' in dct.keys()) and ( - dct['status'] == 400 or 'Bad Parameter' in dct['message'] or 'Error' in dct['message'] - ): - # 400 status code means no more results - break - if 'events' in dct.keys(): - if len(dct['events']) == 0: - break - self.totalhosts.update({host for host in dct['events']}) - await asyncio.sleep(get_delay()) - - async def get_hostnames(self) -> set: - return self.totalhosts - - async def process(self, proxy: bool = False) -> None: - self.proxy = proxy - await self.do_search() diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 3fc4a64b..dbbb4a09 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -54,10 +54,6 @@ class Core: def bevigil_key() -> str: return Core.api_keys()['bevigil']['key'] - @staticmethod - def binaryedge_key() -> str: - return Core.api_keys()['binaryedge']['key'] - @staticmethod def bing_key() -> str: return Core.api_keys()['bing']['key'] @@ -170,7 +166,6 @@ class Core: 'anubis', 'baidu', 'bevigil', - 'binaryedge', 'bing', 'bingapi', 'bufferoverun', From 6d87f9e0bd2ade39ae6af19b4583911b564a17eb Mon Sep 17 00:00:00 2001 From: noarche Date: Tue, 22 Apr 2025 12:10:30 -0400 Subject: [PATCH 20/41] Update Dockerfile Dockerfile update to Successfully install on server running docker io --- Dockerfile | 52 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 22541280..30326302 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,53 @@ FROM debian:testing-slim + LABEL maintainer="@jay_townsend1 & @NotoriousRebel1" -RUN apt update && apt dist-upgrade -y && apt install -y pipx git curl gcc && rm -rf /var/lib/apt/lists/* && apt clean && apt autoremove -y -RUN pipx install --python python3.12 git+https://github.com/laramies/theHarvester.git + +# Install dependencies for building Python from source +RUN apt update && apt install -y \ + curl \ + build-essential \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + wget \ + curl \ + llvm \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libffi-dev \ + liblzma-dev \ + python3-dev \ + git \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +# Install Python 3.11 from source +RUN curl -fsSL https://www.python.org/ftp/python/3.11.6/Python-3.11.6.tgz -o Python-3.11.6.tgz \ + && tar -xvf Python-3.11.6.tgz \ + && cd Python-3.11.6 \ + && ./configure --enable-optimizations \ + && make -j 2 \ + && make altinstall \ + && rm -rf /Python-3.11.6 /Python-3.11.6.tgz + +# Install pip for Python 3.11 +RUN curl https://bootstrap.pypa.io/get-pip.py | python3.11 + +# Install pipx for Python 3.11 +RUN python3.11 -m pip install --user pipx + +# Add pipx to PATH +ENV PATH=/root/.local/bin:$PATH + +# Install theHarvester via pipx +RUN pipx install --python python3.11 git+https://github.com/laramies/theHarvester.git + +# Ensure pipx path RUN pipx ensurepath + +# Set the entrypoint ENTRYPOINT ["/root/.local/bin/restfulHarvest", "-H", "0.0.0.0", "-p", "80"] -EXPOSE 80 From 3d137e2245e02ed8ed3ea7f232c576d507fc0543 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Apr 2025 21:47:04 +0000 Subject: [PATCH 21/41] Bump lxml from 5.3.2 to 5.4.0 Bumps [lxml](https://github.com/lxml/lxml) from 5.3.2 to 5.4.0. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-5.3.2...lxml-5.4.0) --- updated-dependencies: - dependency-name: lxml dependency-version: 5.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 5576455e..1fbf135d 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,7 +8,7 @@ censys==2.2.17 certifi==2025.1.31 dnspython==2.7.0 fastapi==0.115.12 -lxml==5.3.2 +lxml==5.4.0 netaddr==1.3.0 ujson==5.10.0 playwright==1.51.0 From 924c6f8c8fa4e8cfe129271e3cd42a040e1f2077 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Apr 2025 21:47:42 +0000 Subject: [PATCH 22/41] Bump ruff from 0.11.5 to 0.11.6 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.11.5...0.11.6) --- updated-dependencies: - dependency-name: ruff dependency-version: 0.11.6 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 3c3a73ed..5bfae70c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,5 +8,5 @@ types-ujson==5.10.0.20250326 types-PyYAML==6.0.12.20250402 types-requests==2.32.0.20250328 # 2.31.0.7 introduced a regression types-python-dateutil==2.9.0.20241206 -ruff==0.11.5 +ruff==0.11.6 wheel==0.45.1 \ No newline at end of file From 38e00249e2c788116d5b1ccd5948e78e9f89b6a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Apr 2025 21:37:57 +0000 Subject: [PATCH 23/41] Bump aiohttp from 3.11.16 to 3.11.18 --- updated-dependencies: - dependency-name: aiohttp dependency-version: 3.11.18 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 1fbf135d..a55979d8 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,6 +1,6 @@ aiodns==3.2.0 aiofiles==24.1.0 -aiohttp==3.11.16 +aiohttp==3.11.18 aiomultiprocess==0.9.1 aiosqlite==0.21.0 beautifulsoup4==4.13.4 From 152fdfb2be9878fce2f964112b7ae639dc03b482 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Apr 2025 21:05:21 +0000 Subject: [PATCH 24/41] Bump mypy-extensions from 1.0.0 to 1.1.0 Bumps [mypy-extensions](https://github.com/python/mypy_extensions) from 1.0.0 to 1.1.0. - [Commits](https://github.com/python/mypy_extensions/compare/1.0.0...1.1.0) --- updated-dependencies: - dependency-name: mypy-extensions dependency-version: 1.1.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 5bfae70c..d2ee0f64 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ mypy==1.15.0 -mypy-extensions==1.0.0 +mypy-extensions==1.1.0 pytest==8.3.5 pytest-asyncio==0.26.0 types-certifi==2021.10.8.3 From c87862bbadea10ad83368c580c9309405c557206 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Apr 2025 21:37:42 +0000 Subject: [PATCH 25/41] Bump uvicorn from 0.34.1 to 0.34.2 Bumps [uvicorn](https://github.com/encode/uvicorn) from 0.34.1 to 0.34.2. - [Release notes](https://github.com/encode/uvicorn/releases) - [Changelog](https://github.com/encode/uvicorn/blob/master/docs/release-notes.md) - [Commits](https://github.com/encode/uvicorn/compare/0.34.1...0.34.2) --- updated-dependencies: - dependency-name: uvicorn dependency-version: 0.34.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index a55979d8..a8c741d6 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -18,6 +18,6 @@ requests==2.32.3 retrying==1.3.4 shodan==1.31.0 slowapi==0.1.9 -uvicorn==0.34.1 +uvicorn==0.34.2 uvloop==0.21.0; platform_system != "Windows" winloop==0.1.8; platform_system == "Windows" From ca8cd8aee916bb0aa80ef99f1769745559429c3e Mon Sep 17 00:00:00 2001 From: dimeko Date: Thu, 8 May 2025 19:41:55 +0300 Subject: [PATCH 26/41] add command line arguments in exported files --- theHarvester/__main__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 88a3c498..e6f0bdc1 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -1202,6 +1202,7 @@ async def start(rest_args: argparse.Namespace | None = None): # XML REPORT SECTION with open(filename, 'w+') as file: file.write('') + file.write('' + ' '.join(['"{}"'.format(arg) if ' ' in arg else arg for arg in sys.argv[1:]]) + '') for x in all_emails: file.write('' + x + '') for x in full: @@ -1227,6 +1228,8 @@ async def start(rest_args: argparse.Namespace | None = None): filename = filename.rsplit('.', 1)[0] + '.json' # create dict with values for json output json_dict: dict = dict() + # start by adding the command line arguments + json_dict["cmd"] = ' '.join(['"{}"'.format(arg) if ' ' in arg else arg for arg in sys.argv[1:]]) # determine if a variable exists # it should but just a validation check if 'ip_list' in locals(): From 57123f4fa8041c2095801ba8a38c1a755634fac7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 May 2025 21:01:46 +0000 Subject: [PATCH 27/41] Bump ruff from 0.11.6 to 0.11.8 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.11.6 to 0.11.8. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.11.6...0.11.8) --- updated-dependencies: - dependency-name: ruff dependency-version: 0.11.8 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index d2ee0f64..d22dde08 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,5 +8,5 @@ types-ujson==5.10.0.20250326 types-PyYAML==6.0.12.20250402 types-requests==2.32.0.20250328 # 2.31.0.7 introduced a regression types-python-dateutil==2.9.0.20241206 -ruff==0.11.6 +ruff==0.11.8 wheel==0.45.1 \ No newline at end of file From 5b09846af5043c54894d22d96db3570457fee587 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Apr 2025 21:03:03 +0000 Subject: [PATCH 28/41] Bump playwright from 1.51.0 to 1.52.0 Bumps [playwright](https://github.com/microsoft/playwright-python) from 1.51.0 to 1.52.0. - [Release notes](https://github.com/microsoft/playwright-python/releases) - [Commits](https://github.com/microsoft/playwright-python/compare/v1.51.0...v1.52.0) --- updated-dependencies: - dependency-name: playwright dependency-version: 1.52.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index a8c741d6..e9f1cca0 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -11,7 +11,7 @@ fastapi==0.115.12 lxml==5.4.0 netaddr==1.3.0 ujson==5.10.0 -playwright==1.51.0 +playwright==1.52.0 PyYAML==6.0.2 python-dateutil==2.9.0.post0 requests==2.32.3 From 5821b499af02ca3a07e880a7488f6ce6afbf4037 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 21:08:04 +0000 Subject: [PATCH 29/41] Bump aiodns from 3.2.0 to 3.4.0 Bumps [aiodns](https://github.com/saghul/aiodns) from 3.2.0 to 3.4.0. - [Release notes](https://github.com/saghul/aiodns/releases) - [Changelog](https://github.com/aio-libs/aiodns/blob/master/ChangeLog) - [Commits](https://github.com/saghul/aiodns/compare/v3.2.0...v3.4.0) --- updated-dependencies: - dependency-name: aiodns dependency-version: 3.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index e9f1cca0..baf7195f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -aiodns==3.2.0 +aiodns==3.4.0 aiofiles==24.1.0 aiohttp==3.11.18 aiomultiprocess==0.9.1 From d7295af2de9fb88404aa167ee74be87646fc9f86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Apr 2025 23:00:29 +0000 Subject: [PATCH 30/41] Bump certifi from 2025.1.31 to 2025.4.26 Bumps [certifi](https://github.com/certifi/python-certifi) from 2025.1.31 to 2025.4.26. - [Commits](https://github.com/certifi/python-certifi/compare/2025.01.31...2025.04.26) --- updated-dependencies: - dependency-name: certifi dependency-version: 2025.4.26 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index baf7195f..6d162727 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -5,7 +5,7 @@ aiomultiprocess==0.9.1 aiosqlite==0.21.0 beautifulsoup4==4.13.4 censys==2.2.17 -certifi==2025.1.31 +certifi==2025.4.26 dnspython==2.7.0 fastapi==0.115.12 lxml==5.4.0 From b48e2aae9c4ca0525c25fffc4ff8d9fa6022de0a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 21:12:16 +0000 Subject: [PATCH 32/41] Bump ruff from 0.11.8 to 0.11.9 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.11.8 to 0.11.9. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.11.8...0.11.9) --- updated-dependencies: - dependency-name: ruff dependency-version: 0.11.9 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index d22dde08..c09da5e8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,5 +8,5 @@ types-ujson==5.10.0.20250326 types-PyYAML==6.0.12.20250402 types-requests==2.32.0.20250328 # 2.31.0.7 introduced a regression types-python-dateutil==2.9.0.20241206 -ruff==0.11.8 +ruff==0.11.9 wheel==0.45.1 \ No newline at end of file From 18e04b93de68103da5da8a0125b565d94b222ad1 Mon Sep 17 00:00:00 2001 From: Vladislav <126883477+p3test@users.noreply.github.com> Date: Wed, 14 May 2025 23:01:14 +0300 Subject: [PATCH 33/41] Merge pull request #1990 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added a module for domain breach checking using the Dehashed service … --- theHarvester/__main__.py | 20 ++++- theHarvester/data/api-keys.yaml | 5 +- theHarvester/discovery/search_dehashed.py | 97 +++++++++++++++++++++++ theHarvester/lib/core.py | 5 ++ 4 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 theHarvester/discovery/search_dehashed.py diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index e6f0bdc1..501dcf5c 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -24,6 +24,7 @@ from theHarvester.discovery import ( certspottersearch, criminalip, crtsh, + search_dehashed, dnssearch, duckduckgosearch, fullhuntsearch, @@ -149,7 +150,7 @@ async def start(rest_args: argparse.Namespace | None = None): '-b', '--source', help="""anubis, baidu, bevigil, bing, bingapi, brave, bufferoverun, - censys, certspotter, criminalip, crtsh, duckduckgo, fullhunt, github-code, + censys, certspotter, criminalip, crtsh, dehashed, duckduckgo, fullhunt, github-code, hackertarget, hunter, hunterhow, intelx, netlas, onyphe, otx, pentesttools, projectdiscovery, rapiddns, rocketreach, securityTrails, sitedossier, subdomaincenter, subdomainfinderc99, threatminer, tomba, urlscan, virustotal, yahoo, whoisxml, zoomeye, venacus""", @@ -504,6 +505,23 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: print(f'[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}') + elif engineitem == 'dehashed': + try: + dehashed_search = search_dehashed.SearchDehashed(word) + stor_lst.append( + store( + dehashed_search, + engineitem, + store_host=False, + store_ip=True, + ) + ) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(f'An exception has occurred in Dehashed: {e}') + elif engineitem == 'duckduckgo': duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(word, limit) stor_lst.append( diff --git a/theHarvester/data/api-keys.yaml b/theHarvester/data/api-keys.yaml index bf14798f..34ee17ce 100644 --- a/theHarvester/data/api-keys.yaml +++ b/theHarvester/data/api-keys.yaml @@ -65,4 +65,7 @@ apikeys: key: venacus: - key: \ No newline at end of file + key: + + dehashed: + key: diff --git a/theHarvester/discovery/search_dehashed.py b/theHarvester/discovery/search_dehashed.py new file mode 100644 index 00000000..5aa608f6 --- /dev/null +++ b/theHarvester/discovery/search_dehashed.py @@ -0,0 +1,97 @@ +import time +import requests +from theHarvester.discovery.constants import MissingKey +from theHarvester.lib.core import Core + + +class SearchDehashed: + def __init__(self, word) -> None: + self.word = word + self.key = Core.dehashed_key() + if self.key is None: + raise MissingKey('Dehashed') + + self.api = 'https://api.dehashed.com/v2/search' + self.headers = { + 'Content-Type': 'application/json', + 'Dehashed-Api-Key': self.key + } + self.results = '' + self.data: list[dict] = [] + + async def do_search(self) -> None: + print(f'\t[+] Performing Dehashed search for: {self.word}') + page = 1 + size = 100 + while True: + payload = { + 'query': self.word, + 'page': page, + 'size': size, + 'wildcard': False, + 'regex': False, + 'de_dupe': False + } + + try: + response = requests.post(self.api, json=payload, headers=self.headers) + if response.status_code == 401: + raise Exception('Unauthorized. Check Dehashed API key.') + if response.status_code == 403: + raise Exception('Forbidden. API key is not allowed.') + + data = response.json() + entries = data.get('entries', []) + if not entries: + break + + self.data.extend(entries) + print(f'\t[+] Page {page} - Retrieved {len(entries)} entries.') + + if len(entries) < size: + break + page += 1 + time.sleep(0.5) + except Exception as e: + print(f'\t[!] Dehashed error: {e}') + break + + async def print_csv_results(self) -> None: + if not self.data: + print("\t[!] No data found.") + return + + print("\n[Dehashed Results]") + print("Email,Username,Password,Phone,IP,Source") + + for entry in self.data: + email = entry.get('email', '') + username = entry.get('username', '') + password = entry.get('password', '') + phone = entry.get('phone', '') + ip = entry.get('ip_address', '') + source = entry.get('database_name', '') + + csv_line = f'"{email}","{username}","{password}","{phone}","{ip}","{source}"' + print(csv_line) + + async def process(self, proxy: bool = False) -> None: + await self.do_search() + await self.print_csv_results() + + async def get_emails(self) -> set: + emails = set() + for entry in self.data: + if 'email' in entry and entry['email']: + emails.add(entry['email']) + return emails + + async def get_hostnames(self) -> set: + return set() + + async def get_ips(self) -> set: + ips = set() + for entry in self.data: + if 'ip_address' in entry and entry['ip_address']: + ips.add(entry['ip_address']) + return ips diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index dbbb4a09..7aa130b6 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -70,6 +70,10 @@ class Core: def criminalip_key() -> str: return Core.api_keys()['criminalip']['key'] + @staticmethod + def dehashed_key() -> str: + return Core.api_keys()['dehashed']['key'] + @staticmethod def fullhunt_key() -> str: return Core.api_keys()['fullhunt']['key'] @@ -175,6 +179,7 @@ class Core: 'criminalip', 'crtsh', 'duckduckgo', + 'dehashed', 'fullhunt', 'github-code', 'hackertarget', From 933e39336b8a933518bc902f531091c97c8837c7 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Wed, 14 May 2025 21:07:56 +0100 Subject: [PATCH 34/41] format fixes --- theHarvester/__main__.py | 2 +- theHarvester/discovery/search_dehashed.py | 20 +++++--------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 501dcf5c..62057062 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -1247,7 +1247,7 @@ async def start(rest_args: argparse.Namespace | None = None): # create dict with values for json output json_dict: dict = dict() # start by adding the command line arguments - json_dict["cmd"] = ' '.join(['"{}"'.format(arg) if ' ' in arg else arg for arg in sys.argv[1:]]) + json_dict['cmd'] = ' '.join(['"{}"'.format(arg) if ' ' in arg else arg for arg in sys.argv[1:]]) # determine if a variable exists # it should but just a validation check if 'ip_list' in locals(): diff --git a/theHarvester/discovery/search_dehashed.py b/theHarvester/discovery/search_dehashed.py index 5aa608f6..4d60f865 100644 --- a/theHarvester/discovery/search_dehashed.py +++ b/theHarvester/discovery/search_dehashed.py @@ -12,10 +12,7 @@ class SearchDehashed: raise MissingKey('Dehashed') self.api = 'https://api.dehashed.com/v2/search' - self.headers = { - 'Content-Type': 'application/json', - 'Dehashed-Api-Key': self.key - } + self.headers = {'Content-Type': 'application/json', 'Dehashed-Api-Key': self.key} self.results = '' self.data: list[dict] = [] @@ -24,14 +21,7 @@ class SearchDehashed: page = 1 size = 100 while True: - payload = { - 'query': self.word, - 'page': page, - 'size': size, - 'wildcard': False, - 'regex': False, - 'de_dupe': False - } + payload = {'query': self.word, 'page': page, 'size': size, 'wildcard': False, 'regex': False, 'de_dupe': False} try: response = requests.post(self.api, json=payload, headers=self.headers) @@ -58,11 +48,11 @@ class SearchDehashed: async def print_csv_results(self) -> None: if not self.data: - print("\t[!] No data found.") + print('\t[!] No data found.') return - print("\n[Dehashed Results]") - print("Email,Username,Password,Phone,IP,Source") + print('\n[Dehashed Results]') + print('Email,Username,Password,Phone,IP,Source') for entry in self.data: email = entry.get('email', '') From 9055cde11ae4b736fbea37fc59784309a9d68738 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Wed, 14 May 2025 21:10:15 +0100 Subject: [PATCH 35/41] lint fixes --- theHarvester/__main__.py | 6 +++--- theHarvester/discovery/rocketreach.py | 2 +- theHarvester/discovery/search_dehashed.py | 6 ++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 62057062..4e1c0127 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -24,7 +24,6 @@ from theHarvester.discovery import ( certspottersearch, criminalip, crtsh, - search_dehashed, dnssearch, duckduckgosearch, fullhuntsearch, @@ -39,6 +38,7 @@ from theHarvester.discovery import ( projectdiscovery, rapiddns, rocketreach, + search_dehashed, searchhunterhow, securitytrailssearch, shodansearch, @@ -1220,7 +1220,7 @@ async def start(rest_args: argparse.Namespace | None = None): # XML REPORT SECTION with open(filename, 'w+') as file: file.write('') - file.write('' + ' '.join(['"{}"'.format(arg) if ' ' in arg else arg for arg in sys.argv[1:]]) + '') + file.write('' + ' '.join([f'"{arg}"' if ' ' in arg else arg for arg in sys.argv[1:]]) + '') for x in all_emails: file.write('' + x + '') for x in full: @@ -1247,7 +1247,7 @@ async def start(rest_args: argparse.Namespace | None = None): # create dict with values for json output json_dict: dict = dict() # start by adding the command line arguments - json_dict['cmd'] = ' '.join(['"{}"'.format(arg) if ' ' in arg else arg for arg in sys.argv[1:]]) + json_dict['cmd'] = ' '.join([f'"{arg}"' if ' ' in arg else arg for arg in sys.argv[1:]]) # determine if a variable exists # it should but just a validation check if 'ip_list' in locals(): diff --git a/theHarvester/discovery/rocketreach.py b/theHarvester/discovery/rocketreach.py index 4096de35..8434f51f 100644 --- a/theHarvester/discovery/rocketreach.py +++ b/theHarvester/discovery/rocketreach.py @@ -48,7 +48,7 @@ class SearchRocketReach: self.links.add(profile['linkedin_url']) if 'emails' in dict(profile).keys() and profile['emails']: for email in profile['emails']: - if 'email' in email and email['email']: + if email.get('email'): self.emails.add(email['email']) if 'pagination' in dict(result).keys(): next_page = result['pagination']['page'] + 1 diff --git a/theHarvester/discovery/search_dehashed.py b/theHarvester/discovery/search_dehashed.py index 4d60f865..506e6dba 100644 --- a/theHarvester/discovery/search_dehashed.py +++ b/theHarvester/discovery/search_dehashed.py @@ -1,5 +1,7 @@ import time + import requests + from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import Core @@ -72,7 +74,7 @@ class SearchDehashed: async def get_emails(self) -> set: emails = set() for entry in self.data: - if 'email' in entry and entry['email']: + if entry.get('email'): emails.add(entry['email']) return emails @@ -82,6 +84,6 @@ class SearchDehashed: async def get_ips(self) -> set: ips = set() for entry in self.data: - if 'ip_address' in entry and entry['ip_address']: + if entry.get('ip_address'): ips.add(entry['ip_address']) return ips From 77097bb2f0824bb6868a4216aa86b8f5cbe2969a Mon Sep 17 00:00:00 2001 From: Lee Baird Date: Wed, 14 May 2025 15:38:54 -0500 Subject: [PATCH 36/41] Alphabetized APIs --- theHarvester/data/api-keys.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/theHarvester/data/api-keys.yaml b/theHarvester/data/api-keys.yaml index 34ee17ce..64fea0db 100644 --- a/theHarvester/data/api-keys.yaml +++ b/theHarvester/data/api-keys.yaml @@ -15,6 +15,9 @@ apikeys: criminalip: key: + dehashed: + key: + fullhunt: key: @@ -55,6 +58,9 @@ apikeys: key: secret: + venacus: + key: + virustotal: key: @@ -63,9 +69,3 @@ apikeys: zoomeye: key: - - venacus: - key: - - dehashed: - key: From ef783d34f51c772d816daec42dfa4a7ac9b6b36d Mon Sep 17 00:00:00 2001 From: Lee Baird Date: Wed, 14 May 2025 17:42:51 -0500 Subject: [PATCH 37/41] Changed the corder of the CONFIG_DIRS. --- theHarvester/lib/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 7aa130b6..f2c20fed 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -21,9 +21,9 @@ if TYPE_CHECKING: DATA_DIR = Path(__file__).parents[1] / 'data' CONFIG_DIRS = [ + Path('~/.theHarvester'), Path('/etc/theHarvester/'), Path('/usr/local/etc/theHarvester/'), - Path('~/.theHarvester'), ] From 96241e1708ba2b02094f8e124d4c2d1336249868 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 May 2025 21:32:07 +0000 Subject: [PATCH 38/41] Bump ruff from 0.11.9 to 0.11.10 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.11.9 to 0.11.10. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.11.9...0.11.10) --- updated-dependencies: - dependency-name: ruff dependency-version: 0.11.10 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index c09da5e8..ea1b4d0d 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,5 +8,5 @@ types-ujson==5.10.0.20250326 types-PyYAML==6.0.12.20250402 types-requests==2.32.0.20250328 # 2.31.0.7 introduced a regression types-python-dateutil==2.9.0.20241206 -ruff==0.11.9 +ruff==0.11.10 wheel==0.45.1 \ No newline at end of file From f3ffe2cc57b6bb5f25ea40a8088df8c09b7a20c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 May 2025 21:36:38 +0000 Subject: [PATCH 39/41] Bump types-pyyaml from 6.0.12.20250402 to 6.0.12.20250516 Bumps [types-pyyaml](https://github.com/typeshed-internal/stub_uploader) from 6.0.12.20250402 to 6.0.12.20250516. - [Commits](https://github.com/typeshed-internal/stub_uploader/commits) --- updated-dependencies: - dependency-name: types-pyyaml dependency-version: 6.0.12.20250516 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index ea1b4d0d..610fbe90 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,7 +5,7 @@ pytest-asyncio==0.26.0 types-certifi==2021.10.8.3 types-chardet==5.0.4.6 types-ujson==5.10.0.20250326 -types-PyYAML==6.0.12.20250402 +types-PyYAML==6.0.12.20250516 types-requests==2.32.0.20250328 # 2.31.0.7 introduced a regression types-python-dateutil==2.9.0.20241206 ruff==0.11.10 From 1dc420ed6b71742afad480b24985f82873b6d804 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 May 2025 23:26:43 +0000 Subject: [PATCH 40/41] Bump types-requests from 2.32.0.20250328 to 2.32.0.20250515 Bumps [types-requests](https://github.com/typeshed-internal/stub_uploader) from 2.32.0.20250328 to 2.32.0.20250515. - [Commits](https://github.com/typeshed-internal/stub_uploader/commits) --- updated-dependencies: - dependency-name: types-requests dependency-version: 2.32.0.20250515 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 610fbe90..15b02ddd 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,7 +6,7 @@ types-certifi==2021.10.8.3 types-chardet==5.0.4.6 types-ujson==5.10.0.20250326 types-PyYAML==6.0.12.20250516 -types-requests==2.32.0.20250328 # 2.31.0.7 introduced a regression +types-requests==2.32.0.20250515 # 2.31.0.7 introduced a regression types-python-dateutil==2.9.0.20241206 ruff==0.11.10 wheel==0.45.1 \ No newline at end of file From 791abbc54d7e50486f5efb69366a707173c20358 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 May 2025 23:39:12 +0000 Subject: [PATCH 41/41] Bump types-python-dateutil from 2.9.0.20241206 to 2.9.0.20250516 Bumps [types-python-dateutil](https://github.com/typeshed-internal/stub_uploader) from 2.9.0.20241206 to 2.9.0.20250516. - [Commits](https://github.com/typeshed-internal/stub_uploader/commits) --- updated-dependencies: - dependency-name: types-python-dateutil dependency-version: 2.9.0.20250516 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 15b02ddd..a25537fa 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -7,6 +7,6 @@ types-chardet==5.0.4.6 types-ujson==5.10.0.20250326 types-PyYAML==6.0.12.20250516 types-requests==2.32.0.20250515 # 2.31.0.7 introduced a regression -types-python-dateutil==2.9.0.20241206 +types-python-dateutil==2.9.0.20250516 ruff==0.11.10 wheel==0.45.1 \ No newline at end of file