WEBDIENSTE/theHarvester
1
0
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2026-09-28 12:34:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into master

Browse Source
This commit is contained in:
yiğit ibrahim
2025-05-17 03:18:28 +03:00
committed by GitHub
parent 793aea1363 791abbc54d
commit 6ea2f3d7ac
6 changed files with 13 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
-1
View File
@@ -108,7 +108,6 @@ Modules that require an API key:
Documentation to setup API keys can be found at - https://github.com/laramies/theHarvester/wiki/Installation#api-keys
* bevigil - Free upto 50 queries. Pricing can be found here: https://bevigil.com/pricing/osint
* binaryedge - $10/month
* bing
* bufferoverun - uses the free binaAPI
* censys - API keys are required and can be retrieved from your [Censys account](https://search.censys.io/account/api).
requirements/dev.txt
+4 -6
View File
@@ -2,6 +2,10 @@ pytest-asyncio==0.26.0
pytest-cov==4.1.0
pytest-mock==3.12.0
pytest-timeout==2.2.0
mypy==1.15.0
mypy-extensions==1.1.0
pytest==8.3.5
pytest-asyncio==0.26.0
types-certifi==2021.10.8.3
types-chardet==5.0.4.6
types-ujson==5.10.0.20250326
@@ -13,9 +17,3 @@ types-beautifulsoup4==4.12.0.20240106
types-netaddr==0.8.0.20240106
ruff==0.11.10
wheel==0.45.1
black==24.2.0
isort==5.13.2
flake8==7.0.0
coverage==7.4.3
mypy==1.15.0
mypy-extensions==1.1.0
theHarvester/__main__.py
+4 -1
View File
@@ -161,7 +161,7 @@ async def start(rest_args: argparse.Namespace | None = None):
parser.add_argument(
'-b',
'--source',
help="""anubis, baidu, bevigil, binaryedge, bing, bingapi, brave, bufferoverun,
help="""anubis, baidu, bevigil, bing, bingapi, brave, bufferoverun,
censys, certspotter, criminalip, crtsh, dehashed, duckduckgo, fullhunt, github-code,
hackertarget, hunter, hunterhow, intelx, netlas, onyphe, otx, pentesttools, projectdiscovery,
rapiddns, rocketreach, securityTrails, sitedossier, subdomaincenter, subdomainfinderc99, threatminer, tomba,
@@ -418,6 +418,7 @@ async def start(rest_args: argparse.Namespace | None = None):
)
except Exception as e:
print(e)
elif engineitem == 'binaryedge':
try:
@@ -853,6 +854,7 @@ async def start(rest_args: argparse.Namespace | None = None):
print(e)
else:
print(f'An exception has occurred in venacus search: {e}')
elif engineitem == 'haveibeenpwned':
try:
@@ -921,6 +923,7 @@ async def start(rest_args: argparse.Namespace | None = None):
print(e)
else:
print(f'An exception has occurred in BuiltWith search: {e}')
else:
if rest_args is not None:
try:
theHarvester/data/api-keys.yaml
+1
View File
@@ -1,4 +1,5 @@
apikeys:
bevigil:
key:
theHarvester/discovery/binaryedgesearch.py
-42
View File
@@ -1,42 +0,0 @@
import asyncio
from theHarvester.discovery.constants import MissingKey, get_delay
from theHarvester.lib.core import AsyncFetcher, Core
class SearchBinaryEdge:
def __init__(self, word, limit) -> None:
self.word = word
self.totalhosts: set = set()
self.proxy = False
self.key = Core.binaryedge_key()
self.limit = 501 if limit >= 501 else limit
self.limit = 2 if self.limit == 1 else self.limit
if self.key is None:
raise MissingKey('binaryedge')
async def do_search(self) -> None:
base_url = f'https://api.binaryedge.io/v2/query/domains/subdomain/{self.word}'
headers = {'X-KEY': self.key, 'User-Agent': Core.get_user_agent()}
for page in range(1, self.limit):
params = {'page': page}
response = await AsyncFetcher.fetch_all([base_url], json=True, proxy=self.proxy, params=params, headers=headers)
responses = response[0]
dct = responses
if ('status' in dct.keys() and 'message' in dct.keys()) and (
dct['status'] == 400 or 'Bad Parameter' in dct['message'] or 'Error' in dct['message']
):
# 400 status code means no more results
break
if 'events' in dct.keys():
if len(dct['events']) == 0:
break
self.totalhosts.update({host for host in dct['events']})
await asyncio.sleep(get_delay())
async def get_hostnames(self) -> set:
return self.totalhosts
async def process(self, proxy: bool = False) -> None:
self.proxy = proxy
await self.do_search()
theHarvester/lib/core.py
+4 -28
View File
@@ -54,10 +54,6 @@ class Core:
def bevigil_key() -> str:
return Core.api_keys()['bevigil']['key']
@staticmethod
def binaryedge_key() -> str:
return Core.api_keys()['binaryedge']['key']
@staticmethod
def bing_key() -> str:
return Core.api_keys()['bing']['key']
@@ -192,7 +188,6 @@ class Core:
return [
'baidu',
'bevigil',
'binaryedge',
'bing',
'bingapi',
'bufferoverun',
@@ -203,6 +198,7 @@ class Core:
'crtsh',
'dnsdumpster',
'duckduckgo',
'dehashed',
'fullhunt',
'github-code',
'hackertarget',
@@ -234,6 +230,7 @@ class Core:
'twitter',
'urlscan',
'virustotal',
'whoisxml',
'yahoo',
'zoomeye',
'zoomeyeapi',
@@ -242,6 +239,8 @@ class Core:
'securityscorecard',
'builtwith',
'api_endpoints' # Add the new API endpoint scanner
'venacus',
]
@staticmethod
@@ -249,24 +248,6 @@ class Core:
# User-Agents from https://techblog.willshouse.com/2012/01/03/most-common-user-agents/
# Lasted updated 5/16/25
# TODO use bs4 to auto parse user agents
user_agents = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0',
'Mozilla/5.0 (Windows NT 10.0; rv:114.0) Gecko/20100101 Firefox/114.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.43',
'Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 OPR/99.0.0.0',
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0',
@@ -333,11 +314,6 @@ class Core:
'Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4590.2 Mobile Safari/537.36 Chrome-Lighthouse',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:137.0) Gecko/20100101 Firefox/137.0',
'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko)',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15',
'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.3',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Safari/605.1.15',