diff --git a/README.md b/README.md index 8df24ef2..7b135c7a 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,6 @@ Modules that require an API key: Documentation to setup API keys can be found at - https://github.com/laramies/theHarvester/wiki/Installation#api-keys * bevigil - Free upto 50 queries. Pricing can be found here: https://bevigil.com/pricing/osint -* binaryedge - $10/month * bing * bufferoverun - uses the free binaAPI * censys - API keys are required and can be retrieved from your [Censys account](https://search.censys.io/account/api). diff --git a/requirements/dev.txt b/requirements/dev.txt index 8d83c670..efa0c7b2 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,6 +2,10 @@ pytest-asyncio==0.26.0 pytest-cov==4.1.0 pytest-mock==3.12.0 pytest-timeout==2.2.0 +mypy==1.15.0 +mypy-extensions==1.1.0 +pytest==8.3.5 +pytest-asyncio==0.26.0 types-certifi==2021.10.8.3 types-chardet==5.0.4.6 types-ujson==5.10.0.20250326 @@ -13,9 +17,3 @@ types-beautifulsoup4==4.12.0.20240106 types-netaddr==0.8.0.20240106 ruff==0.11.10 wheel==0.45.1 -black==24.2.0 -isort==5.13.2 -flake8==7.0.0 -coverage==7.4.3 -mypy==1.15.0 -mypy-extensions==1.1.0 diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index b2604dc6..23f8f820 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -161,7 +161,7 @@ async def start(rest_args: argparse.Namespace | None = None): parser.add_argument( '-b', '--source', - help="""anubis, baidu, bevigil, binaryedge, bing, bingapi, brave, bufferoverun, + help="""anubis, baidu, bevigil, bing, bingapi, brave, bufferoverun, censys, certspotter, criminalip, crtsh, dehashed, duckduckgo, fullhunt, github-code, hackertarget, hunter, hunterhow, intelx, netlas, onyphe, otx, pentesttools, projectdiscovery, rapiddns, rocketreach, securityTrails, sitedossier, subdomaincenter, subdomainfinderc99, threatminer, tomba, @@ -418,6 +418,7 @@ async def start(rest_args: argparse.Namespace | None = None): ) except Exception as e: print(e) + elif engineitem == 'binaryedge': try: @@ -853,6 +854,7 @@ async def start(rest_args: argparse.Namespace | None = None): print(e) else: print(f'An exception has occurred in venacus search: {e}') + elif engineitem == 'haveibeenpwned': try: @@ -921,6 +923,7 @@ async def start(rest_args: argparse.Namespace | None = None): print(e) else: print(f'An exception has occurred in BuiltWith search: {e}') + else: if rest_args is not None: try: diff --git a/theHarvester/data/api-keys.yaml b/theHarvester/data/api-keys.yaml index aa9367c8..4f4daf41 100644 --- a/theHarvester/data/api-keys.yaml +++ b/theHarvester/data/api-keys.yaml @@ -1,4 +1,5 @@ apikeys: + bevigil: key: diff --git a/theHarvester/discovery/binaryedgesearch.py b/theHarvester/discovery/binaryedgesearch.py deleted file mode 100644 index 895e0cd4..00000000 --- a/theHarvester/discovery/binaryedgesearch.py +++ /dev/null @@ -1,42 +0,0 @@ -import asyncio - -from theHarvester.discovery.constants import MissingKey, get_delay -from theHarvester.lib.core import AsyncFetcher, Core - - -class SearchBinaryEdge: - def __init__(self, word, limit) -> None: - self.word = word - self.totalhosts: set = set() - self.proxy = False - self.key = Core.binaryedge_key() - self.limit = 501 if limit >= 501 else limit - self.limit = 2 if self.limit == 1 else self.limit - if self.key is None: - raise MissingKey('binaryedge') - - async def do_search(self) -> None: - base_url = f'https://api.binaryedge.io/v2/query/domains/subdomain/{self.word}' - headers = {'X-KEY': self.key, 'User-Agent': Core.get_user_agent()} - for page in range(1, self.limit): - params = {'page': page} - response = await AsyncFetcher.fetch_all([base_url], json=True, proxy=self.proxy, params=params, headers=headers) - responses = response[0] - dct = responses - if ('status' in dct.keys() and 'message' in dct.keys()) and ( - dct['status'] == 400 or 'Bad Parameter' in dct['message'] or 'Error' in dct['message'] - ): - # 400 status code means no more results - break - if 'events' in dct.keys(): - if len(dct['events']) == 0: - break - self.totalhosts.update({host for host in dct['events']}) - await asyncio.sleep(get_delay()) - - async def get_hostnames(self) -> set: - return self.totalhosts - - async def process(self, proxy: bool = False) -> None: - self.proxy = proxy - await self.do_search() diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 2ed64a50..a5ba3ed8 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -54,10 +54,6 @@ class Core: def bevigil_key() -> str: return Core.api_keys()['bevigil']['key'] - @staticmethod - def binaryedge_key() -> str: - return Core.api_keys()['binaryedge']['key'] - @staticmethod def bing_key() -> str: return Core.api_keys()['bing']['key'] @@ -192,7 +188,6 @@ class Core: return [ 'baidu', 'bevigil', - 'binaryedge', 'bing', 'bingapi', 'bufferoverun', @@ -203,6 +198,7 @@ class Core: 'crtsh', 'dnsdumpster', 'duckduckgo', + 'dehashed', 'fullhunt', 'github-code', 'hackertarget', @@ -234,6 +230,7 @@ class Core: 'twitter', 'urlscan', 'virustotal', + 'whoisxml', 'yahoo', 'zoomeye', 'zoomeyeapi', @@ -242,6 +239,8 @@ class Core: 'securityscorecard', 'builtwith', 'api_endpoints' # Add the new API endpoint scanner + 'venacus', + ] @staticmethod @@ -249,24 +248,6 @@ class Core: # User-Agents from https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ # Lasted updated 5/16/25 # TODO use bs4 to auto parse user agents - user_agents = [ - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0', - 'Mozilla/5.0 (Windows NT 10.0; rv:114.0) Gecko/20100101 Firefox/114.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.43', - 'Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 OPR/99.0.0.0', - 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0', @@ -333,11 +314,6 @@ class Core: 'Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4590.2 Mobile Safari/537.36 Chrome-Lighthouse', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:137.0) Gecko/20100101 Firefox/137.0', - 'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko)', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15', 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.3', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Safari/605.1.15',