WEBDIENSTE/theHarvester
1
0
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2026-09-07 02:07:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor: route operator output through logging

Browse Source
This commit is contained in:
NotoriousRebel
2026-07-20 23:37:09 -04:00
parent 0166df00a7
commit 90bbd9ddd6
56 changed files with 629 additions and 501 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyproject.toml
+2 -1
View File
@@ -139,7 +139,8 @@ select = ["E",
"PT",
"TC",
"FURB",
"ASYNC"
"ASYNC",
"T20"
]
ignore = [
"B018",
tests/test_logging.py
+65 -11
View File
@@ -1,30 +1,84 @@
from __future__ import annotations
import ast
import logging
import os
import subprocess
import sys
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from pathlib import Path
import textwrap
from pathlib import Path
def test_import_does_not_configure_application_logging() -> None:
result = subprocess.run(
[
sys.executable,
'-c',
'import logging; import theHarvester.__main__; print(len(logging.getLogger().handlers))',
],
def run_python(script: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[sys.executable, '-c', textwrap.dedent(script)],
check=True,
capture_output=True,
text=True,
)
def test_import_does_not_configure_application_logging() -> None:
result = run_python(
"""
import logging
import sys
import theHarvester.__main__
sys.stdout.write(str(len(logging.getLogger().handlers)) + '\\n')
"""
)
assert result.stdout.splitlines()[-1] == '0'
def test_operator_output_uses_stdout_without_verbose_logging() -> None:
result = run_python(
"""
from theHarvester.lib.output import configure_logging, output_logger
configure_logging(verbose=False)
output_logger.info('operator result')
"""
)
assert result.stdout == 'operator result\n'
assert result.stderr == ''
def test_diagnostics_use_stderr_only_when_verbose() -> None:
result = run_python(
"""
import logging
from theHarvester.lib.output import configure_logging
logger = logging.getLogger('theHarvester.discovery.example')
configure_logging(verbose=False)
logger.info('hidden diagnostic')
configure_logging(verbose=True)
logger.info('visible diagnostic')
"""
)
assert result.stdout == ''
assert 'hidden diagnostic' not in result.stderr
assert 'INFO theHarvester.discovery.example: visible diagnostic' in result.stderr
def test_production_code_has_no_print_calls() -> None:
package_root = Path(__file__).parents[1] / 'theHarvester'
print_calls = []
for path in package_root.rglob('*.py'):
tree = ast.parse(path.read_text())
print_calls.extend(
f'{path.relative_to(package_root)}:{node.lineno}'
for node in ast.walk(tree)
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == 'print'
)
assert print_calls == []
def test_verbose_enables_info_diagnostics(tmp_path: Path) -> None:
script = """import asyncio
import logging
theHarvester/__main__.py
+211 -209
View File
File diff suppressed because it is too large Load Diff
theHarvester/discovery/additional_apis.py
+11 -10
View File
@@ -6,6 +6,7 @@ from theHarvester.discovery.haveibeenpwned import SearchHaveIBeenPwned
from theHarvester.discovery.leaklookup import SearchLeakLookup
from theHarvester.discovery.securityscorecard import SearchSecurityScorecard
from theHarvester.discovery.shodansearch import SearchShodan
from theHarvester.lib.output import output_logger
class AdditionalAPIs:
@@ -65,7 +66,7 @@ class AdditionalAPIs:
self.hosts.update(self.haveibeenpwned.hosts)
self.emails.update(self.haveibeenpwned.emails)
except Exception as e:
print(f'Error processing HaveIBeenPwned: {e}')
output_logger.info(f'Error processing HaveIBeenPwned: {e}')
async def _process_leaklookup(self, proxy: bool = False):
"""Process Leak-Lookup API."""
@@ -75,7 +76,7 @@ class AdditionalAPIs:
self.hosts.update(self.leaklookup.hosts)
self.emails.update(self.leaklookup.emails)
except Exception as e:
print(f'Error processing Leak-Lookup: {e}')
output_logger.info(f'Error processing Leak-Lookup: {e}')
async def _process_securityscorecard(self, proxy: bool = False):
"""Process SecurityScorecard API."""
@@ -89,7 +90,7 @@ class AdditionalAPIs:
}
self.hosts.update(self.securityscorecard.hosts)
except Exception as e:
print(f'Error processing SecurityScorecard: {e}')
output_logger.info(f'Error processing SecurityScorecard: {e}')
async def _process_builtwith(self, proxy: bool = False):
"""Process BuiltWith API."""
@@ -105,7 +106,7 @@ class AdditionalAPIs:
}
self.hosts.update(self.builtwith.hosts)
except Exception as e:
print(f'Error processing BuiltWith: {e}')
output_logger.info(f'Error processing BuiltWith: {e}')
async def _process_shodan(self, proxy: bool = False):
"""Process Shodan API for IP information."""
@@ -124,9 +125,9 @@ class AdditionalAPIs:
ip = socket.gethostbyname(self.domain)
ips_to_search.add(ip)
except socket.gaierror as e:
print(f"Failed to resolve domain '{self.domain}': {e}")
output_logger.info(f"Failed to resolve domain '{self.domain}': {e}")
except Exception as e:
print(f"Unexpected error while resolving domain '{self.domain}': {e}")
output_logger.info(f"Unexpected error while resolving domain '{self.domain}': {e}")
# Add any IPs from other results
for host in self.hosts:
@@ -141,21 +142,21 @@ class AdditionalAPIs:
# Search each IP in Shodan
for ip in ips_to_search:
try:
print(f'\tSearching Shodan for {ip}')
output_logger.info(f'\tSearching Shodan for {ip}')
shodan_result = await self.shodan.search_ip(ip)
if ip in shodan_result and isinstance(shodan_result[ip], dict):
self.shodan_data[ip] = shodan_result[ip]
elif ip in shodan_result and isinstance(shodan_result[ip], str):
print(f'{ip}: {shodan_result[ip]}')
output_logger.info(f'{ip}: {shodan_result[ip]}')
await asyncio.sleep(2) # Rate limiting
except Exception as ip_error:
print(f'Error searching Shodan for {ip}: {ip_error}')
output_logger.info(f'Error searching Shodan for {ip}: {ip_error}')
continue
except Exception as e:
print(f'Error processing Shodan: {e}')
output_logger.info(f'Error processing Shodan: {e}')
@staticmethod
def _is_valid_ip(ip_str: str) -> bool:
theHarvester/discovery/bitbucket.py
+14 -13
View File
@@ -8,6 +8,7 @@ import aiohttp
from theHarvester.discovery.constants import MissingKey, get_delay
from theHarvester.lib.core import Core
from theHarvester.lib.output import output_logger
from theHarvester.parsers import myparser
@@ -58,7 +59,7 @@ class SearchBitBucket:
if match.get('fragment') is not None
]
except (AttributeError, TypeError, ValueError) as e:
print(f'Error extracting fragments: {e}')
output_logger.info(f'Error extracting fragments: {e}')
return []
@staticmethod
@@ -70,7 +71,7 @@ class SearchBitBucket:
return int(page_param)
return 0
except (AttributeError, TypeError, ValueError) as e:
print(f'Error parsing page response: {e}')
output_logger.info(f'Error parsing page response: {e}')
return None
async def handle_response(self, response: tuple[str, dict, int, Any]) -> ErrorResult | RetryResult | SuccessResult:
@@ -86,7 +87,7 @@ class SearchBitBucket:
return RetryResult(60)
return ErrorResult(status, json_data if isinstance(json_data, dict) else text)
except (TypeError, ValueError, KeyError, AttributeError) as e:
print(f'Error handling response: {e}')
output_logger.info(f'Error handling response: {e}')
return ErrorResult(500, str(e))
@staticmethod
@@ -103,7 +104,7 @@ class SearchBitBucket:
async with sess.get(url, proxy=random.choice(Core.proxy_list()) if self.proxy else None) as resp:
return await resp.text(), await resp.json(), resp.status, resp.links
except (aiohttp.ClientError, TimeoutError, ValueError, OSError) as e:
print(f'Error performing search: {e}')
output_logger.info(f'Error performing search: {e}')
return '', {}, 500, {}
async def process(self, proxy: bool = False) -> None:
@@ -117,13 +118,13 @@ class SearchBitBucket:
if isinstance(result, SuccessResult):
# Reset retry counter on any successful response
self.retry_count = 0
print(f'\tSearching {self.counter} results.')
output_logger.info(f'\tSearching {self.counter} results.')
self.total_results += ''.join(result.fragments)
self.counter += len(result.fragments)
next_or_last = result.next_page or result.last_page
# Break if pagination does not advance to avoid infinite loop
if next_or_last == self.page:
print('\tNo page advancement detected; exiting to avoid infinite loop.')
output_logger.info('\tNo page advancement detected; exiting to avoid infinite loop.')
self.page = 0
break
self.page = next_or_last
@@ -131,29 +132,29 @@ class SearchBitBucket:
elif isinstance(result, RetryResult):
self.retry_count += 1
if self.retry_count > self.max_retries:
print('\tMaximum retries reached; exiting to avoid infinite loop.')
output_logger.info('\tMaximum retries reached; exiting to avoid infinite loop.')
self.page = 0
break
sleepy_time = get_delay() + result.time
print(f'\tRetrying page in {sleepy_time} seconds...')
output_logger.info(f'\tRetrying page in {sleepy_time} seconds...')
await asyncio.sleep(sleepy_time)
else:
# On error, stop to avoid endless retries on a bad state
print(f'\tException occurred: status_code: {result.status_code} reason: {result.body}')
output_logger.info(f'\tException occurred: status_code: {result.status_code} reason: {result.body}')
self.page = 0
break
except (aiohttp.ClientError, TimeoutError, ValueError, TypeError, AttributeError) as e:
print(f'Error processing page: {e}')
output_logger.info(f'Error processing page: {e}')
await asyncio.sleep(get_delay())
except (aiohttp.ClientError, TimeoutError, ValueError, TypeError, AttributeError) as e:
print(f'An exception has occurred in bitbucket process: {e}')
output_logger.info(f'An exception has occurred in bitbucket process: {e}')
async def get_emails(self):
try:
rawres = myparser.Parser(self.total_results, self.word)
return await rawres.emails()
except (AttributeError, TypeError, re.error) as e:
print(f'Error getting emails: {e}')
output_logger.info(f'Error getting emails: {e}')
return []
async def get_hostnames(self):
@@ -161,5 +162,5 @@ class SearchBitBucket:
rawres = myparser.Parser(self.total_results, self.word)
return await rawres.hostnames()
except (AttributeError, TypeError, re.error) as e:
print(f'Error getting hostnames: {e}')
output_logger.info(f'Error getting hostnames: {e}')
return []
theHarvester/discovery/bravesearch.py
+11 -9
View File
@@ -3,6 +3,7 @@ from urllib.parse import quote
from theHarvester.discovery.constants import MissingKey, get_delay
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
from theHarvester.parsers import myparser
@@ -50,7 +51,7 @@ class SearchBrave:
# Handle API response
if resp is None:
print('No response received from Brave Search API')
output_logger.info('No response received from Brave Search API')
break
# Check for API errors (rate limit, quota exceeded, etc.)
@@ -59,15 +60,14 @@ class SearchBrave:
error_code = resp.get('error', {}).get('code', 'unknown')
if 'rate limit' in error_msg.lower() or error_code == 'rate_limit_exceeded':
print(f'Rate limit exceeded. Increasing delay to {self.rate_limit_delay * 2} seconds')
output_logger.info(f'Rate limit exceeded. Increasing delay to {self.rate_limit_delay * 2} seconds')
self.rate_limit_delay *= 2
await asyncio.sleep(self.rate_limit_delay)
continue
elif 'quota' in error_msg.lower() or error_code == 'quota_exceeded':
print(f'API quota exceeded: {error_msg}')
output_logger.info(f'API quota exceeded: {error_msg}')
break
else:
# print(f'API error ({error_code}): {error_msg}')
break
if 'web' in resp and 'results' in resp['web']:
@@ -93,7 +93,7 @@ class SearchBrave:
if len(self.results) >= self.limit:
break
else:
print('Unexpected response format from Brave Search API')
output_logger.info('Unexpected response format from Brave Search API')
break
await asyncio.sleep(get_delay())
@@ -103,17 +103,19 @@ class SearchBrave:
# Handle specific API-related exceptions
if 'rate limit' in error_msg or '429' in error_msg:
print(f'Rate limit detected in exception. Increasing delay to {self.rate_limit_delay * 2} seconds')
output_logger.info(
f'Rate limit detected in exception. Increasing delay to {self.rate_limit_delay * 2} seconds'
)
self.rate_limit_delay *= 2
await asyncio.sleep(self.rate_limit_delay)
elif 'quota' in error_msg or '403' in error_msg:
print(f'Quota exceeded or access denied: {e}')
output_logger.info(f'Quota exceeded or access denied: {e}')
break
elif 'timeout' in error_msg:
print(f'Request timeout occurred: {e}')
output_logger.info(f'Request timeout occurred: {e}')
await asyncio.sleep(get_delay() + 2)
else:
print(f'An exception has occurred in bravesearch: {e}')
output_logger.info(f'An exception has occurred in bravesearch: {e}')
await asyncio.sleep(get_delay() + 5)
continue
theHarvester/discovery/builtwith.py
+2 -1
View File
@@ -4,6 +4,7 @@ import aiohttp
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchBuiltWith:
@@ -41,7 +42,7 @@ class SearchBuiltWith:
self.tech_stack = data
self._extract_data()
except Exception as e:
print(f'Error in BuiltWith search: {e}')
output_logger.info(f'Error in BuiltWith search: {e}')
def _extract_data(self) -> None:
"""Extract and categorize technology information."""
theHarvester/discovery/censysearch.py
+2 -1
View File
@@ -10,6 +10,7 @@ from censys.search import CensysCerts
from theHarvester import __version__ as thehavester_version
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import Core
from theHarvester.lib.output import output_logger
class SearchCensys:
@@ -64,7 +65,7 @@ class SearchCensys:
self.emails.update(self._normalize_emails(email_address))
records_seen += 1
except CensysRateLimitExceededException:
print('Censys rate limit exceeded')
output_logger.info('Censys rate limit exceeded')
async def get_hostnames(self) -> set:
return self.totalhosts
theHarvester/discovery/certspottersearch.py
+5 -4
View File
@@ -1,4 +1,5 @@
from theHarvester.lib.core import AsyncFetcher
from theHarvester.lib.output import output_logger
class SearchCertspoter:
@@ -26,11 +27,11 @@ class SearchCertspoter:
else:
self.totalhosts.update({''})
except IndexError:
print('No data returned from Cert Spotter.')
output_logger.info('No data returned from Cert Spotter.')
except ConnectionError:
print('Network connection failed.')
output_logger.info('Network connection failed.')
except Exception as e:
print(f'Unexpected error occurred: {e}')
output_logger.info(f'Unexpected error occurred: {e}')
async def get_hostnames(self) -> set:
return self.totalhosts
@@ -38,4 +39,4 @@ class SearchCertspoter:
async def process(self, proxy: bool = False) -> None:
self.proxy = proxy
await self.do_search()
print('\tSearching results.')
output_logger.info('\tSearching results.')
theHarvester/discovery/chaos.py
+5 -4
View File
@@ -3,6 +3,7 @@ from types import ModuleType
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
json: ModuleType = _stdlib_json
try:
@@ -54,7 +55,7 @@ class SearchChaos:
response = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy)
if not response or not isinstance(response, list) or not response[0]:
print(f'No response from Chaos API for: {url}')
output_logger.info(f'No response from Chaos API for: {url}')
return
try:
@@ -64,7 +65,7 @@ class SearchChaos:
# Check for error messages
if 'error' in data:
error_msg = data.get('message', data.get('error', 'Unknown error'))
print(f'Chaos API error: {error_msg}')
output_logger.info(f'Chaos API error: {error_msg}')
if 'unauthorized' in error_msg.lower():
raise MissingKey('Chaos (ProjectDiscovery)')
return
@@ -98,12 +99,12 @@ class SearchChaos:
self.totalhosts.add(full_domain.lower())
except Exception as e:
print(f'Failed to parse Chaos response: {e}')
output_logger.info(f'Failed to parse Chaos response: {e}')
except MissingKey:
raise
except Exception as e:
print(f'Chaos API error: {e}')
output_logger.info(f'Chaos API error: {e}')
async def get_hostnames(self) -> set:
return self.totalhosts
theHarvester/discovery/commoncrawl.py
+5 -4
View File
@@ -3,6 +3,7 @@ import re
from types import ModuleType
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
json: ModuleType = _stdlib_json
try:
@@ -79,7 +80,7 @@ class SearchCommoncrawl:
try:
data = self._safe_parse_json_lines(response[0])
except Exception as e:
print(f'Failed to parse Common Crawl response for {index}: {e}')
output_logger.info(f'Failed to parse Common Crawl response for {index}: {e}')
continue
# Extract domains from URLs
@@ -109,14 +110,14 @@ class SearchCommoncrawl:
if domain.endswith(f'.{self.word}') or domain == self.word:
self.totalhosts.add(domain)
except Exception as e:
print(f'Failed to parse Common Crawl main domain response for {index}: {e}')
output_logger.info(f'Failed to parse Common Crawl main domain response for {index}: {e}')
except Exception as e:
print(f'Common Crawl API error for index {index}: {e}')
output_logger.info(f'Common Crawl API error for index {index}: {e}')
continue
except Exception as e:
print(f'Common Crawl API error: {e}')
output_logger.info(f'Common Crawl API error: {e}')
async def get_hostnames(self) -> set:
return self.totalhosts
theHarvester/discovery/constants.py
-1
View File
@@ -61,7 +61,6 @@ async def search(text: str) -> bool:
or 'http://www.google.com/sorry/index' in line
or 'https://www.google.com/sorry/index' in line
):
# print('\tGoogle is blocking your IP due to too many automated requests, wait or change your IP')
return True
return False
theHarvester/discovery/criminalip.py
+22 -29
View File
@@ -4,6 +4,7 @@ from urllib.parse import urlparse
from theHarvester.discovery.constants import MissingKey, get_delay
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchCriminalIP:
@@ -88,7 +89,6 @@ class SearchCriminalIP:
# https://www.criminalip.io/developer/api/get-v2-domain-report-id
url = 'https://api.criminalip.io/v1/domain/scan'
data = f'{{"query": "{self.word}"}}'
# print(f'Current key: {self.key}')
user_agent = Core.get_user_agent()
response = await AsyncFetcher.post_fetch(
url,
@@ -97,19 +97,18 @@ class SearchCriminalIP:
data=data,
proxy=self.proxy,
)
# print(f'My response: {response}')
# Expected response format:
# {'data': {'scan_id': scan_id}, 'message': 'api success', 'status': 200}
if not isinstance(response, dict):
print(f'An error has occurred searching criminalip dumping response: {response}')
output_logger.info(f'An error has occurred searching criminalip dumping response: {response}')
return
if response.get('status') != 200:
print(f'An error has occurred searching criminalip dumping response: {response}')
output_logger.info(f'An error has occurred searching criminalip dumping response: {response}')
return
scan_id = response.get('data', {}).get('scan_id')
if scan_id is None:
print(f'CriminalIP did not return a scan_id, dumping response: {response}')
output_logger.info(f'CriminalIP did not return a scan_id, dumping response: {response}')
return
scan_percentage = 0
@@ -125,26 +124,26 @@ class SearchCriminalIP:
)
status = status_response[0] if isinstance(status_response, list) and len(status_response) > 0 else {}
if not isinstance(status, dict):
print(f'CriminalIP status response is malformed dumping data: {status_response}')
output_logger.info(f'CriminalIP status response is malformed dumping data: {status_response}')
return
if status.get('status') != 200:
print(f'CriminalIP status check failed dumping data: status_response: {status}')
output_logger.info(f'CriminalIP status check failed dumping data: status_response: {status}')
return
# Expected format:
# {"data": {"scan_percentage": 100}, "message": "api success", "status": 200}
scan_percentage = status.get('data', {}).get('scan_percentage')
if scan_percentage is None:
print(f'CriminalIP status did not include scan_percentage dumping data: {status}')
output_logger.info(f'CriminalIP status did not include scan_percentage dumping data: {status}')
return
if scan_percentage == 100:
break
if scan_percentage == -2:
print(f'CriminalIP failed to scan: {self.word} does not exist, verify manually')
print(f'Dumping data: scan_response: {response} status_response: {status}')
output_logger.info(f'CriminalIP failed to scan: {self.word} does not exist, verify manually')
output_logger.info(f'Dumping data: scan_response: {response} status_response: {status}')
return
if scan_percentage == -1:
print(f'CriminalIP scan failed dumping data: scan_response: {response} status_response: {status}')
output_logger.info(f'CriminalIP scan failed dumping data: scan_response: {response} status_response: {status}')
return
# Wait for scan to finish
if counter >= 5:
@@ -153,10 +152,12 @@ class SearchCriminalIP:
await asyncio.sleep(10 * get_delay())
counter += 1
if counter == 10:
print(
output_logger.info(
'Ten iterations have occurred in CriminalIP waiting for scan to finish, returning to prevent infinite loop.'
)
print(f'Verify results manually on CriminalIP dumping data: scan_response: {response} status_response: {status}')
output_logger.info(
f'Verify results manually on CriminalIP dumping data: scan_response: {response} status_response: {status}'
)
return
report_url = f'https://api.criminalip.io/v2/domain/report/{scan_id}'
@@ -168,32 +169,29 @@ class SearchCriminalIP:
)
scan = scan_response[0] if isinstance(scan_response, list) and len(scan_response) > 0 else {}
if not isinstance(scan, dict):
print(f'CriminalIP report response is malformed dumping data: {scan_response}')
output_logger.info(f'CriminalIP report response is malformed dumping data: {scan_response}')
return
if scan.get('status') != 200:
print(f'CriminalIP report request failed dumping data: {scan}')
output_logger.info(f'CriminalIP report request failed dumping data: {scan}')
return
# json_formatted_str = json.dumps(scan, indent=2)
# print(json_formatted_str)
try:
await self.parser(scan)
except Exception as e:
print(f'An exception occurred while parsing criminalip result: {e}')
print('Dumping json: ')
print(scan)
output_logger.info(f'An exception occurred while parsing criminalip result: {e}')
output_logger.info('Dumping json: ')
output_logger.info(scan)
async def parser(self, jlines):
# TODO when new scope field is added to parse lines for potential new scope!
# TODO map as_name to asn for asn data
# TODO determine if worth storing interesting urls
if not isinstance(jlines, dict) or 'data' not in jlines.keys() or not isinstance(jlines['data'], dict):
print(f'Error with criminalip data, dumping: {jlines}')
output_logger.info(f'Error with criminalip data, dumping: {jlines}')
return
data = jlines['data']
for cert in data.get('certificates', []):
# print(f'Current cert: {cert}')
if isinstance(cert, dict):
self._add_host(cert.get('subject'))
@@ -206,11 +204,10 @@ class SearchCriminalIP:
self._add_host(main_domain)
subdomains = [sub.get('domain') for sub in connected_domain.get('subdomains', []) if isinstance(sub, dict)]
for sub in subdomains:
# print(f'Current sub: {sub}')
self._add_host(sub)
except Exception as e:
print(f'An exception has occurred: {e}')
print(f'Main line: {connected_domain}')
output_logger.info(f'An exception has occurred: {e}')
output_logger.info(f'Main line: {connected_domain}')
for ip_info in data.get('connected_ip_info', []):
if not isinstance(ip_info, dict):
@@ -313,10 +310,6 @@ class SearchCriminalIP:
self.totalhosts = {host.removeprefix('www.') for host in self.totalhosts if '*.' + self.word != host}
# print(f'hostnames: {self.totalhosts}')
# print(f'asns: {self.asns}')
# print(f'ips: {self.totalips}')
async def get_asns(self) -> set:
return self.asns
theHarvester/discovery/crtsh.py
+4 -3
View File
@@ -1,4 +1,5 @@
from theHarvester.lib.core import AsyncFetcher
from theHarvester.lib.output import output_logger
class SearchCrtsh:
@@ -16,11 +17,11 @@ class SearchCrtsh:
data = set([(dct['name_value'][2:] if dct['name_value'][:2] == '*.' else dct['name_value']) for dct in response])
data = {domain for domain in data if (domain[0] != '*' and str(domain[0:4]).isnumeric() is False)}
except IndexError:
print('No response from crt.sh or malformed list.')
output_logger.info('No response from crt.sh or malformed list.')
except KeyError as ke:
print(f'Missing expected key in response: {ke}')
output_logger.info(f'Missing expected key in response: {ke}')
except Exception as e:
print(f'Unexpected error: {e}')
output_logger.info(f'Unexpected error: {e}')
clean: list = []
for x in data:
pre = x.split()
theHarvester/discovery/dnssearch.py
+3 -2
View File
@@ -15,6 +15,7 @@ from aiodns import DNSResolver
from theHarvester.lib import hostchecker
from theHarvester.lib.core import DATA_DIR
from theHarvester.lib.output import output_logger
#####################################################################
# DNS FORCE
@@ -37,7 +38,7 @@ class DnsForce:
self.list = [f'{word.strip()}.{self.domain}' for word in self.list]
async def run(self):
print(f'Starting DNS brute forcing with {len(self.list)} words')
output_logger.info(f'Starting DNS brute forcing with {len(self.list)} words')
checker = hostchecker.Checker(self.list, nameservers=self.dnsserver)
resolved_pair, hosts, ips = await checker.check()
return resolved_pair, hosts, ips
@@ -193,7 +194,7 @@ def log_result(host: str) -> None:
"""
if host:
print(host)
output_logger.info(host)
def generate_postprocessing_callback(target: str, **allhosts: list[str]) -> Callable:
theHarvester/discovery/duckduckgosearch.py
+2 -1
View File
@@ -1,6 +1,7 @@
import ujson
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
from theHarvester.parsers import myparser
@@ -70,7 +71,7 @@ class SearchDuckDuckGo:
tmp.add(url)
return tmp
except Exception as e:
print(f'Exception occurred: {e}')
output_logger.info(f'Exception occurred: {e}')
return set()
async def get_emails(self):
theHarvester/discovery/fofa.py
+5 -4
View File
@@ -4,6 +4,7 @@ from types import ModuleType
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
json: ModuleType = _stdlib_json
try:
@@ -72,7 +73,7 @@ class SearchFofa:
response = await AsyncFetcher.fetch_all([full_url], headers=headers, proxy=self.proxy)
if not response or not isinstance(response, list) or not response[0]:
print(f'No response from Fofa API for: {self.word}')
output_logger.info(f'No response from Fofa API for: {self.word}')
return
try:
@@ -82,7 +83,7 @@ class SearchFofa:
# Check for errors
if data.get('error', False):
error_msg = data.get('errmsg', 'Unknown error')
print(f'Fofa API error: {error_msg}')
output_logger.info(f'Fofa API error: {error_msg}')
if '账号无效' in error_msg or 'invalid' in error_msg.lower():
raise MissingKey('Fofa API (Invalid credentials)')
return
@@ -107,12 +108,12 @@ class SearchFofa:
self.totalips.add(ip)
except Exception as e:
print(f'Failed to parse Fofa response: {e}')
output_logger.info(f'Failed to parse Fofa response: {e}')
except MissingKey:
raise
except Exception as e:
print(f'Fofa API error: {e}')
output_logger.info(f'Fofa API error: {e}')
async def get_hostnames(self) -> set:
return self.totalhosts
theHarvester/discovery/fullhuntsearch.py
+2 -1
View File
@@ -3,6 +3,7 @@ from urllib.parse import quote
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchFullHunt:
@@ -391,7 +392,7 @@ class SearchFullHunt:
await self.extract_data_from_search_results(search_results)
except Exception as e:
print(f'Error during FullHunt search: {e}')
output_logger.info(f'Error during FullHunt search: {e}')
async def get_hostnames(self) -> list[str]:
"""Return list of discovered subdomains"""
theHarvester/discovery/githubcode.py
+15 -14
View File
@@ -7,6 +7,7 @@ import aiohttp
from theHarvester.discovery.constants import MissingKey, get_delay
from theHarvester.lib.core import Core
from theHarvester.lib.output import output_logger
from theHarvester.parsers import myparser
@@ -49,7 +50,7 @@ class SearchGithubCode:
self.retry_count = 0
self.max_retries = 3
except Exception as e:
print(f'Error initializing SearchGithubCode: {e}')
output_logger.info(f'Error initializing SearchGithubCode: {e}')
raise
@staticmethod
@@ -62,7 +63,7 @@ class SearchGithubCode:
if match.get('fragment') is not None
]
except Exception as e:
print(f'Error extracting fragments: {e}')
output_logger.info(f'Error extracting fragments: {e}')
return []
@staticmethod
@@ -74,7 +75,7 @@ class SearchGithubCode:
return int(page_param)
return 0
except Exception as e:
print(f'Error parsing page response: {e}')
output_logger.info(f'Error parsing page response: {e}')
return None
async def handle_response(self, response: tuple[str, dict, int, Any]) -> ErrorResult | RetryResult | SuccessResult:
@@ -90,7 +91,7 @@ class SearchGithubCode:
return RetryResult(60)
return ErrorResult(status, json_data if isinstance(json_data, dict) else text)
except Exception as e:
print(f'Error handling response: {e}')
output_logger.info(f'Error handling response: {e}')
return ErrorResult(500, str(e))
@staticmethod
@@ -107,7 +108,7 @@ class SearchGithubCode:
async with sess.get(url, proxy=random.choice(Core.proxy_list()) if self.proxy else None) as resp:
return await resp.text(), await resp.json(), resp.status, resp.links
except Exception as e:
print(f'Error performing search: {e}')
output_logger.info(f'Error performing search: {e}')
return '', {}, 500, {}
async def process(self, proxy: bool = False) -> None:
@@ -121,13 +122,13 @@ class SearchGithubCode:
if isinstance(result, SuccessResult):
# Reset retry counter on any successful response
self.retry_count = 0
print(f'\tSearching {self.counter} results.')
output_logger.info(f'\tSearching {self.counter} results.')
self.total_results += ''.join(result.fragments)
self.counter += len(result.fragments)
next_or_last = result.next_page or result.last_page
# Break if pagination does not advance to avoid infinite loop
if next_or_last == self.page:
print('\tNo page advancement detected; exiting to avoid infinite loop.')
output_logger.info('\tNo page advancement detected; exiting to avoid infinite loop.')
self.page = 0
break
self.page = next_or_last
@@ -135,29 +136,29 @@ class SearchGithubCode:
elif isinstance(result, RetryResult):
self.retry_count += 1
if self.retry_count > self.max_retries:
print('\tMaximum retries reached; exiting to avoid infinite loop.')
output_logger.info('\tMaximum retries reached; exiting to avoid infinite loop.')
self.page = 0
break
sleepy_time = get_delay() + result.time
print(f'\tRetrying page in {sleepy_time} seconds...')
output_logger.info(f'\tRetrying page in {sleepy_time} seconds...')
await asyncio.sleep(sleepy_time)
else:
# On error, stop to avoid endless retries on a bad state
print(f'\tException occurred: status_code: {result.status_code} reason: {result.body}')
output_logger.info(f'\tException occurred: status_code: {result.status_code} reason: {result.body}')
self.page = 0
break
except Exception as e:
print(f'Error processing page: {e}')
output_logger.info(f'Error processing page: {e}')
await asyncio.sleep(get_delay())
except Exception as e:
print(f'An exception has occurred in githubcode process: {e}')
output_logger.info(f'An exception has occurred in githubcode process: {e}')
async def get_emails(self):
try:
rawres = myparser.Parser(self.total_results, self.word)
return await rawres.emails()
except Exception as e:
print(f'Error getting emails: {e}')
output_logger.info(f'Error getting emails: {e}')
return []
async def get_hostnames(self):
@@ -165,5 +166,5 @@ class SearchGithubCode:
rawres = myparser.Parser(self.total_results, self.word)
return await rawres.hostnames()
except Exception as e:
print(f'Error getting hostnames: {e}')
output_logger.info(f'Error getting hostnames: {e}')
return []
theHarvester/discovery/gitlabsearch.py
+5 -4
View File
@@ -3,6 +3,7 @@ import re
from types import ModuleType
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
json: ModuleType = _stdlib_json
try:
@@ -128,10 +129,10 @@ class SearchGitlab:
pass # README might not exist or be accessible
except Exception as e:
print(f'Failed to parse GitLab projects response: {e}')
output_logger.info(f'Failed to parse GitLab projects response: {e}')
except Exception as e:
print(f'GitLab API projects search error: {e}')
output_logger.info(f'GitLab API projects search error: {e}')
async def search_users(self) -> None:
"""Search GitLab users for domain references"""
@@ -179,10 +180,10 @@ class SearchGitlab:
self.totalurls.add(web_url)
except Exception as e:
print(f'Failed to parse GitLab users response: {e}')
output_logger.info(f'Failed to parse GitLab users response: {e}')
except Exception as e:
print(f'GitLab API users search error: {e}')
output_logger.info(f'GitLab API users search error: {e}')
async def do_search(self) -> None:
await self.search_projects()
theHarvester/discovery/haveibeenpwned.py
+2 -1
View File
@@ -2,6 +2,7 @@ import aiohttp
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchHaveIBeenPwned:
@@ -37,7 +38,7 @@ class SearchHaveIBeenPwned:
self.breaches = await response.json()
self._extract_data()
except Exception as e:
print(f'Error in HaveIBeenPwned search: {e}')
output_logger.info(f'Error in HaveIBeenPwned search: {e}')
def _extract_data(self) -> None:
"""Extract and categorize breach information."""
theHarvester/discovery/huntersearch.py
+8 -3
View File
@@ -2,6 +2,7 @@ import asyncio
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchHunter:
@@ -46,9 +47,13 @@ class SearchHunter:
total_number_reqs = response[0]['data']['total'] // 100
# Parse out meta field within initial JSON response to determine the total number of results
if total_requests_avail < total_number_reqs:
print('WARNING: account does not have enough requests to gather all emails')
print(f'Total requests available: {total_requests_avail}, total requests needed to be made: {total_number_reqs}')
print('RETURNING current results, if you would still like to run this module comment out the if request')
output_logger.info('WARNING: account does not have enough requests to gather all emails')
output_logger.info(
f'Total requests available: {total_requests_avail}, total requests needed to be made: {total_number_reqs}'
)
output_logger.info(
'RETURNING current results, if you would still like to run this module comment out the if request'
)
return
self.limit = 100
# max number of emails you can get per request is 100
theHarvester/discovery/intelxsearch.py
+3 -2
View File
@@ -6,6 +6,7 @@ import aiohttp
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import Core
from theHarvester.lib.output import output_logger
from theHarvester.parsers import intelxparser
@@ -46,7 +47,7 @@ class SearchIntelx:
async with session.post(f'{self.database}/phonebook/search', headers=headers, json=data) as total_resp:
search_data = await total_resp.json()
if not search_data['success']:
print(f'Error: {search_data["message"]}')
output_logger.info(f'Error: {search_data["message"]}')
return
phonebook_id = search_data['id']
@@ -59,7 +60,7 @@ class SearchIntelx:
self.results = await resp.json()
except Exception as e:
print(f'An exception has occurred in Intelx: {e}')
output_logger.info(f'An exception has occurred in Intelx: {e}')
async def process(self, proxy: bool = False):
self.proxy = proxy
theHarvester/discovery/leakix.py
+5 -4
View File
@@ -2,6 +2,7 @@ import json as _stdlib_json
from types import ModuleType
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
json: ModuleType = _stdlib_json
try:
@@ -69,7 +70,7 @@ class SearchLeakix:
or 'unauthorized' in response[0].lower()
or 'error' in response[0].lower()
):
print(f'LeakIX API requires authentication: {response[0][:100]}')
output_logger.info(f'LeakIX API requires authentication: {response[0][:100]}')
continue
try:
@@ -95,14 +96,14 @@ class SearchLeakix:
self.totalhosts.add(value.lower())
except Exception as e:
print(f'Failed to parse LeakIX response: {e}')
output_logger.info(f'Failed to parse LeakIX response: {e}')
except Exception as e:
print(f'LeakIX API error for {query_url}: {e}')
output_logger.info(f'LeakIX API error for {query_url}: {e}')
continue
except Exception as e:
print(f'LeakIX API error: {e}')
output_logger.info(f'LeakIX API error: {e}')
async def get_hostnames(self) -> set:
return self.totalhosts
theHarvester/discovery/leaklookup.py
+3 -2
View File
@@ -2,6 +2,7 @@ import aiohttp
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchLeakLookup:
@@ -37,10 +38,10 @@ class SearchLeakLookup:
self.leaks = await response.json()
self._extract_data()
elif response.status == 401:
print('[!] Missing API key for Leak-Lookup.')
output_logger.info('[!] Missing API key for Leak-Lookup.')
raise MissingKey('Leak-Lookup')
except Exception as e:
print(f'Error in Leak-Lookup search: {e}')
output_logger.info(f'Error in Leak-Lookup search: {e}')
def _extract_data(self) -> None:
"""Extract and categorize leak information."""
theHarvester/discovery/mojeek.py
+6 -5
View File
@@ -1,4 +1,5 @@
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
from theHarvester.parsers import myparser
@@ -17,9 +18,9 @@ class SearchMojeek:
self.api_key = ''
if self.api_key:
print('[*] Mojeek: API key detected.')
output_logger.info('[*] Mojeek: API key detected.')
else:
print('[*] Mojeek: No API key found, using default scraping mode.')
output_logger.info('[*] Mojeek: No API key found, using default scraping mode.')
async def do_search(self) -> None:
headers = {'User-Agent': Core.get_user_agent()}
@@ -45,14 +46,14 @@ class SearchMojeek:
self.total_results += f' {url} {result.get("title", "")} {result.get("desc", "")} '
elif data and 'status' in data and 'denied' in data['status'].lower():
print(f'[!] Mojeek API: Access denied ({data["status"]}).')
output_logger.info(f'[!] Mojeek API: Access denied ({data["status"]}).')
break
if api_success:
print('[*] Mojeek: API search completed successfully.')
output_logger.info('[*] Mojeek: API search completed successfully.')
return
else:
print('[*] Mojeek: API returned no results, falling back to scraping...')
output_logger.info('[*] Mojeek: API returned no results, falling back to scraping...')
urls = [f'https://{self.server}/search?q={self.word}&s={num}' for num in range(0, self.limit, 10)]
theHarvester/discovery/onyphe.py
+2 -1
View File
@@ -2,6 +2,7 @@ from urllib.parse import urlparse
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
# from theHarvester.parsers import myparser
@@ -80,7 +81,7 @@ class SearchOnyphe:
except Exception:
continue
else:
print(f'Onhyphe API query did not succeed dumping current response: {self.response}')
output_logger.info(f'Onhyphe API query did not succeed dumping current response: {self.response}')
async def get_asns(self) -> set:
return self.asns
theHarvester/discovery/pentesttools.py
+2 -1
View File
@@ -4,6 +4,7 @@ import ujson
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchPentestTools:
@@ -37,7 +38,7 @@ class SearchPentestTools:
self.total_results = await self.parse_json(res_json)
break
else:
print(f'Operation get_scan_status failed because: {res_json["error"]}. {res_json["details"]}')
output_logger.info(f'Operation get_scan_status failed because: {res_json["error"]}. {res_json["details"]}')
break
@staticmethod
theHarvester/discovery/rapiddns.py
+2 -1
View File
@@ -2,6 +2,7 @@ from bs4 import BeautifulSoup
from bs4.element import Tag
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchRapidDns:
@@ -42,7 +43,7 @@ class SearchRapidDns:
self.total_results.append(f'{subdomain}:{str(cells[1].get_text()).strip()}')
self.total_results = list({domain for domain in self.total_results})
except Exception as e:
print(f'An exception has occurred: {e!s}')
output_logger.info(f'An exception has occurred: {e!s}')
async def process(self, proxy: bool = False) -> None:
self.proxy = proxy
theHarvester/discovery/robtex.py
+7 -6
View File
@@ -4,6 +4,7 @@ from types import ModuleType
import aiohttp
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
json: ModuleType = _stdlib_json
try:
@@ -11,9 +12,9 @@ try:
json = _ujson
except ImportError as e:
print(f"'ujson' not available. Falling back to standard 'json' module. Reason: {e}")
output_logger.info(f"'ujson' not available. Falling back to standard 'json' module. Reason: {e}")
except (AttributeError, OSError, RuntimeError, SystemError, ValueError) as e:
print(f"Unexpected error while importing 'ujson'. Falling back to standard 'json'. Reason: {e}")
output_logger.info(f"Unexpected error while importing 'ujson'. Falling back to standard 'json'. Reason: {e}")
class SearchRobtex:
@@ -50,13 +51,13 @@ class SearchRobtex:
response = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy)
if not response or not isinstance(response, list) or not response[0]:
print(f'No response from Robtex API for: {url}')
output_logger.info(f'No response from Robtex API for: {url}')
return
try:
data = self._safe_parse_json_lines(response[0])
except (TypeError, ValueError) as e:
print(f'Failed to parse JSON lines from Robtex response: {e}')
output_logger.info(f'Failed to parse JSON lines from Robtex response: {e}')
return
# Extract subdomains from DNS records
@@ -99,10 +100,10 @@ class SearchRobtex:
if rrdata and (rrdata.endswith(self.word) or f'.{self.word}' in rrdata):
self.totalhosts.add(rrdata.rstrip('.'))
except (TypeError, ValueError) as e:
print(f'Failed to parse reverse DNS data from Robtex: {e}')
output_logger.info(f'Failed to parse reverse DNS data from Robtex: {e}')
except (aiohttp.ClientError, TimeoutError, OSError, TypeError, ValueError) as e:
print(f'Robtex API error: {e}')
output_logger.info(f'Robtex API error: {e}')
async def get_hostnames(self) -> set:
return self.totalhosts
theHarvester/discovery/rocketreach.py
+3 -2
View File
@@ -2,6 +2,7 @@ import asyncio
from theHarvester.discovery.constants import MissingKey, get_delay
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchRocketReach:
@@ -49,7 +50,7 @@ class SearchRocketReach:
if detail and 'Request was throttled.' in str(detail):
# Rate limit has been triggered need to sleep extra
print(
output_logger.info(
f'RocketReach requests have been throttled; '
f'{str(detail).split(" ", 3)[-1].replace("available", "availability")}'
)
@@ -81,7 +82,7 @@ class SearchRocketReach:
await asyncio.sleep(get_delay() + 5)
except Exception as e:
print(f'An exception has occurred rocketreach: {e}')
output_logger.info(f'An exception has occurred rocketreach: {e}')
async def get_links(self):
return self.links
theHarvester/discovery/search_dehashed.py
+8 -7
View File
@@ -5,6 +5,7 @@ import aiohttp
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import Core
from theHarvester.lib.output import output_logger
class SearchDehashed:
@@ -24,7 +25,7 @@ class SearchDehashed:
self.proxy: bool = False
async def do_search(self) -> None:
print(f'\t[+] Performing Dehashed search for: {self.word}')
output_logger.info(f'\t[+] Performing Dehashed search for: {self.word}')
page = 1
size = 100
while True:
@@ -61,23 +62,23 @@ class SearchDehashed:
break
self.data.extend(entries)
print(f'\t[+] Page {page} - Retrieved {len(entries)} entries.')
output_logger.info(f'\t[+] Page {page} - Retrieved {len(entries)} entries.')
if len(entries) < size:
break
page += 1
await asyncio.sleep(0.5)
except Exception as e:
print(f'\t[!] Dehashed error: {e}')
output_logger.info(f'\t[!] Dehashed error: {e}')
break
async def print_csv_results(self) -> None:
if not self.data:
print('\t[!] No data found.')
output_logger.info('\t[!] No data found.')
return
print('\n[Dehashed Results]')
print('Email,Username,Password,Phone,IP,Source')
output_logger.info('\n[Dehashed Results]')
output_logger.info('Email,Username,Password,Phone,IP,Source')
for entry in self.data:
email = entry.get('email', '')
@@ -88,7 +89,7 @@ class SearchDehashed:
source = entry.get('database_name', '')
csv_line = f'"{email}","{username}","{password}","{phone}","{ip}","{source}"'
print(csv_line)
output_logger.info(csv_line)
async def process(self, proxy: bool = False) -> None:
self.proxy = proxy
theHarvester/discovery/search_dnsdumpster.py
+2 -1
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchDNSDumpster:
@@ -39,7 +40,7 @@ class SearchDNSDumpster:
self.ips.add(ip_info['ip'])
except Exception as e:
print(f'Error occurred in DNSDumpster search: {e}')
output_logger.info(f'Error occurred in DNSDumpster search: {e}')
async def process(self, proxy: bool = False) -> None:
await self.do_search()
theHarvester/discovery/searchhunterhow.py
+2 -3
View File
@@ -5,6 +5,7 @@ from dateutil.relativedelta import relativedelta
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchHunterHow:
@@ -40,11 +41,9 @@ class SearchHunterHow:
proxy=self.proxy,
)
dct = response[0]
# print(f'json response: ')
# print(dct)
if 'code' in dct.keys():
if dct['code'] == 40001:
print(f'Code 40001 indicates for searchhunterhow: {dct["message"]}')
output_logger.info(f'Code 40001 indicates for searchhunterhow: {dct["message"]}')
return
# total = dct['data']['total']
# TODO determine if total is ever 100 how to get more subdomains?
theHarvester/discovery/securityscorecard.py
+2 -1
View File
@@ -2,6 +2,7 @@ import aiohttp
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchSecurityScorecard:
@@ -36,7 +37,7 @@ class SearchSecurityScorecard:
data = await response.json()
self._extract_data(data)
except Exception as e:
print(f'Error in SecurityScorecard search: {e}')
output_logger.info(f'Error in SecurityScorecard search: {e}')
def _extract_data(self, data: dict) -> None:
"""Extract and categorize security scorecard information."""
theHarvester/discovery/securitytrailssearch.py
+5 -4
View File
@@ -2,6 +2,7 @@ import asyncio
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
from theHarvester.parsers import securitytrailsparser
@@ -27,7 +28,7 @@ class SearchSecuritytrail:
auth_responses = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy)
auth_responses = auth_responses[0]
if 'False' in auth_responses or 'Invalid authentication' in auth_responses:
print('\tKey could not be authenticated exiting program.')
output_logger.info('\tKey could not be authenticated exiting program.')
await asyncio.sleep(5)
async def do_search(self) -> None:
@@ -42,7 +43,7 @@ class SearchSecuritytrail:
if domain_response and isinstance(domain_response[0], dict | list):
self.domain_data = domain_response[0] if isinstance(domain_response[0], dict) else {}
else:
print('SecurityTrails: No JSON response received for domain query')
output_logger.info('SecurityTrails: No JSON response received for domain query')
# keep legacy string totalresults for any downstream reliance
if domain_response and domain_response[0]:
self.results = str(domain_response[0])
@@ -57,12 +58,12 @@ class SearchSecuritytrail:
if subdomain_response and isinstance(subdomain_response[0], dict | list):
self.subdomains_data = subdomain_response[0] if isinstance(subdomain_response[0], dict) else {}
else:
print('SecurityTrails: No JSON response received for subdomain query')
output_logger.info('SecurityTrails: No JSON response received for subdomain query')
if subdomain_response and subdomain_response[0]:
self.results = str(subdomain_response[0])
self.totalresults += self.results
except Exception as e:
print(f'SecurityTrails API error: {e}')
output_logger.info(f'SecurityTrails API error: {e}')
return
async def process(self, proxy: bool = False) -> None:
theHarvester/discovery/sherlockeye.py
+4 -3
View File
@@ -6,6 +6,7 @@ import aiohttp
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import Core
from theHarvester.lib.output import output_logger
class SearchSherlockeye:
@@ -95,7 +96,7 @@ class SearchSherlockeye:
def _extract_response(self, response: dict[str, Any]) -> None:
if response.get('success') is False:
message = response.get('message', 'Unknown Sherlockeye API error')
print(f'Sherlockeye API error: {message}')
output_logger.info(f'Sherlockeye API error: {message}')
return
data = response.get('data')
@@ -128,14 +129,14 @@ class SearchSherlockeye:
) as response:
if response.status != 200:
error_body = await response.text()
print(f'Sherlockeye API error ({response.status}): {error_body[:200]}')
output_logger.info(f'Sherlockeye API error ({response.status}): {error_body[:200]}')
return
response_data = await response.json()
if isinstance(response_data, dict):
self._extract_response(response_data)
except Exception as error:
print(f'Sherlockeye API error: {error}')
output_logger.info(f'Sherlockeye API error: {error}')
async def get_hostnames(self) -> set[str]:
return self.totalhosts
theHarvester/discovery/shodan_internetdb.py
+3 -2
View File
@@ -2,6 +2,7 @@ import asyncio
import socket
from theHarvester.lib.core import AsyncFetcher
from theHarvester.lib.output import output_logger
class SearchShodanInternetDB:
@@ -31,7 +32,7 @@ class SearchShodanInternetDB:
try:
addr_infos = await asyncio.to_thread(socket.getaddrinfo, self.word, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
except socket.gaierror:
print(f'Shodan InternetDB: Could not resolve domain {self.word}')
output_logger.info(f'Shodan InternetDB: Could not resolve domain {self.word}')
return
# Deduplicate IPs from the resolution results
@@ -42,7 +43,7 @@ class SearchShodanInternetDB:
resolved_ips.add(ip)
if not resolved_ips:
print(f'Shodan InternetDB: No IPs resolved for {self.word}')
output_logger.info(f'Shodan InternetDB: No IPs resolved for {self.word}')
return
# Query InternetDB for each resolved IP
theHarvester/discovery/shodansearch.py
+3 -3
View File
@@ -4,6 +4,7 @@ from shodan import Shodan, exception
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import Core
from theHarvester.lib.output import output_logger
class SearchShodan:
@@ -21,7 +22,7 @@ class SearchShodan:
results = self.api.host(ipaddress)
if not results or 'data' not in results or not results['data']:
print(f'Shodan: No data found for IP {ip}')
output_logger.info(f'Shodan: No data found for IP {ip}')
return OrderedDict()
asn = ''
domains: list = list()
@@ -106,10 +107,9 @@ class SearchShodan:
return self.tracker
except exception.APIError:
print(f'{ip}: Not in Shodan')
output_logger.info(f'{ip}: Not in Shodan')
self.tracker[ip] = 'Not in Shodan'
except Exception as e:
# print(f'Error occurred in the Shodan IP search module: {e}')
self.tracker[ip] = f'Error occurred in the Shodan IP search module: {e}'
return self.tracker
theHarvester/discovery/subdomaincenter.py
+2 -1
View File
@@ -1,4 +1,5 @@
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SubdomainCenter:
@@ -16,7 +17,7 @@ class SubdomainCenter:
self.results = resp[0]
self.results = {sub[4:] if sub[:4] == 'www.' and sub[4:] else sub for sub in self.results}
except Exception as e:
print(f'An exception has occurred in SubdomainCenter on : {e}')
output_logger.info(f'An exception has occurred in SubdomainCenter on : {e}')
async def get_hostnames(self):
return self.results
theHarvester/discovery/subdomainfinderc99.py
-9
View File
@@ -30,16 +30,7 @@ class SearchSubdomainfinderc99:
await asyncio.sleep(get_delay())
second_resp = await AsyncFetcher.post_fetch(self.server, headers=headers, proxy=self.proxy, data=ujson.dumps(data))
# print(second_resp)
self.totalresults += second_resp
# y = await self.get_hostnames()
# print(list(sorted(y)))
# print(f'Found: {len(y)} subdomains')
# regex = r"value='(https://subdomainfinder\.c99\.nl/scans/\d{4}-\d{2}-\d{2}/" + self.word + r")'"
# match = re.search(regex, second_resp)
# if match:
# print(match.group(1))
async def get_hostnames(self):
rawres = myparser.Parser(self.totalresults, self.word)
theHarvester/discovery/takeover.py
+9 -8
View File
@@ -5,6 +5,7 @@ from random import shuffle
import ujson
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class TakeOver:
@@ -32,7 +33,7 @@ class TakeOver:
if unparsed_fingerprint['status'] == 'Vulnerable' or unparsed_fingerprint['status'] == 'Edge case':
self.fingerprints[unparsed_fingerprint['fingerprint']] = unparsed_fingerprint['service']
except Exception as e:
print(f'An exception has occurred populating takeover fingerprints: {e}, defaulting to static list')
output_logger.info(f'An exception has occurred populating takeover fingerprints: {e}, defaulting to static list')
self.fingerprints = {
"'Trying to access your account?'": 'Campaign Monitor',
'404 Not Found': 'Fly.io',
@@ -66,11 +67,11 @@ class TakeOver:
matches = re.findall(regex, resp)
matches = list(set(matches))
for match in matches:
print(f'\t Takeover detected: {url}')
output_logger.info(f'\t Takeover detected: {url}')
if match in self.fingerprints.keys():
# Validation check as to not error out
service = self.fingerprints[match]
print(f'\t Type of takeover is: {service} with match: {match}')
output_logger.info(f'\t Type of takeover is: {service} with match: {match}')
self.results[url].append({match: service})
async def do_take(self) -> None:
@@ -88,15 +89,15 @@ class TakeOver:
else:
return
except IndexError:
print('Response was empty — possible network error or invalid URL.')
output_logger.info('Response was empty — possible network error or invalid URL.')
except ujson.JSONDecodeError:
print('Failed to parse JSON — cert fingerprints might be unavailable.')
output_logger.info('Failed to parse JSON — cert fingerprints might be unavailable.')
except KeyError as ke:
print(f'Missing expected field in fingerprint: {ke}')
output_logger.info(f'Missing expected field in fingerprint: {ke}')
except TypeError as te:
print(f'Invalid response structure: {te}')
output_logger.info(f'Invalid response structure: {te}')
except Exception as e:
print(f'Unexpected error: {e}')
output_logger.info(f'Unexpected error: {e}')
async def process(self, proxy: bool = False) -> None:
self.proxy = proxy
theHarvester/discovery/thc.py
+7 -4
View File
@@ -3,6 +3,7 @@ import asyncio
import aiohttp
from theHarvester.lib.core import Core
from theHarvester.lib.output import output_logger
class SearchThc:
@@ -27,12 +28,14 @@ class SearchThc:
if response.status == 429:
rate_remaining = response.headers.get('x-ratelimit-remaining', '0')
wait_time = self.base_delay * (attempt + 1)
print(f'THC rate limit hit (remaining: {rate_remaining}). Waiting {wait_time}s before retry...')
output_logger.info(
f'THC rate limit hit (remaining: {rate_remaining}). Waiting {wait_time}s before retry...'
)
await asyncio.sleep(wait_time)
continue
if response.status != 200:
print(f'THC returned status {response.status}')
output_logger.info(f'THC returned status {response.status}')
return
text = await response.text()
@@ -47,10 +50,10 @@ class SearchThc:
error_msg = str(e).lower()
if '429' in error_msg or 'rate' in error_msg:
wait_time = self.base_delay * (attempt + 1)
print(f'THC rate limit detected. Waiting {wait_time}s before retry...')
output_logger.info(f'THC rate limit detected. Waiting {wait_time}s before retry...')
await asyncio.sleep(wait_time)
continue
print(f'An exception has occurred in THC: {e}')
output_logger.info(f'An exception has occurred in THC: {e}')
return
async def get_hostnames(self) -> set:
theHarvester/discovery/threatcrowd.py
+5 -4
View File
@@ -2,6 +2,7 @@ import json as _stdlib_json
from types import ModuleType
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
json: ModuleType = _stdlib_json
try:
@@ -45,7 +46,7 @@ class SearchThreatcrowd:
response = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy)
if not response or not isinstance(response, list) or not response[0]:
print(f'No response from ThreatCrowd API for: {self.word}')
output_logger.info(f'No response from ThreatCrowd API for: {self.word}')
return
try:
@@ -55,7 +56,7 @@ class SearchThreatcrowd:
# Check response code - '1' means success in ThreatCrowd API
response_code = data.get('response_code', '')
if response_code and response_code != '1':
print(f'ThreatCrowd API returned error code: {response_code}')
output_logger.info(f'ThreatCrowd API returned error code: {response_code}')
return
# Extract subdomains - direct list in response
@@ -82,10 +83,10 @@ class SearchThreatcrowd:
self.totalips.add(resolution.strip())
except Exception as e:
print(f'Failed to parse ThreatCrowd response: {e}')
output_logger.info(f'Failed to parse ThreatCrowd response: {e}')
except Exception as e:
print(f'ThreatCrowd API error: {e}')
output_logger.info(f'ThreatCrowd API error: {e}')
async def get_hostnames(self) -> set:
return self.totalhosts
theHarvester/discovery/tombasearch.py
+6 -3
View File
@@ -2,6 +2,7 @@ import asyncio
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchTomba:
@@ -53,9 +54,11 @@ class SearchTomba:
total_number_reqs = response[0]['data']['total'] // 100
# Parse out meta field within initial JSON response to determine the total number of results
if total_requests_avail < total_number_reqs:
print('WARNING: The account does not have enough requests to gather all the emails.')
print(f'Total requests available: {total_requests_avail}, total requests needed to be made: {total_number_reqs}')
print(
output_logger.info('WARNING: The account does not have enough requests to gather all the emails.')
output_logger.info(
f'Total requests available: {total_requests_avail}, total requests needed to be made: {total_number_reqs}'
)
output_logger.info(
'RETURNING current results, If you still wish to run this module despite the current results, please comment out the "if request" line.'
)
return
theHarvester/discovery/venacussearch.py
+4 -3
View File
@@ -4,6 +4,7 @@ import aiohttp
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import Core
from theHarvester.lib.output import output_logger
from theHarvester.parsers import venacusparser
@@ -48,7 +49,7 @@ class SearchVenacus:
current_results = search_data.get('data', [])
if not current_results:
print('No more results found.')
output_logger.info('No more results found.')
break
total_results.extend(current_results)
@@ -61,10 +62,10 @@ class SearchVenacus:
self.results = total_results
if not self.results:
print('No results found.')
output_logger.info('No results found.')
except Exception as e:
print(f'An exception has occurred in Venacus: {e}')
output_logger.info(f'An exception has occurred in Venacus: {e}')
async def process(self, proxy: bool = False):
self.proxy = proxy
theHarvester/discovery/waybackarchive.py
+3 -2
View File
@@ -1,6 +1,7 @@
import re
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchWaybackarchive:
@@ -61,11 +62,11 @@ class SearchWaybackarchive:
self.totalhosts.add(domain)
except Exception as e:
print(f'Wayback Archive API error for URL {url}: {e}')
output_logger.info(f'Wayback Archive API error for URL {url}: {e}')
continue
except Exception as e:
print(f'Wayback Archive API error: {e}')
output_logger.info(f'Wayback Archive API error: {e}')
async def get_hostnames(self) -> set:
return self.totalhosts
theHarvester/discovery/whoisxml.py
+2 -1
View File
@@ -1,5 +1,6 @@
from theHarvester.discovery.constants import MissingKey
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
class SearchWhoisXML:
@@ -25,7 +26,7 @@ class SearchWhoisXML:
# Parse the response according to the example JSON structure:
# {"search":"example.com.com","result":{"count":10000,"records":[{"domain":"test.example.com","firstSeen":1678169400,"lastSeen":1678169400}]}}
self.total_results = []
print(response[0])
output_logger.info(response[0])
if response and response[0]:
# Extract domains from the records array
if 'result' in response[0] and 'records' in response[0]['result']:
theHarvester/discovery/windvane.py
+17 -16
View File
@@ -2,6 +2,7 @@ import json as _stdlib_json
from types import ModuleType
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
json: ModuleType = _stdlib_json
try:
@@ -76,11 +77,11 @@ class SearchWindvane:
await self._search_emails(headers)
else:
# Without API key, try alternative/limited approaches
print('[*] Windvane API key not found. Using limited unauthenticated access.')
output_logger.info('[*] Windvane API key not found. Using limited unauthenticated access.')
await self._search_subdomains_limited(headers)
except Exception as e:
print(f'Windvane API error: {e}')
output_logger.info(f'Windvane API error: {e}')
async def _search_subdomains(self, headers: dict) -> None:
"""Search for subdomains using /ListSubDomain endpoint"""
@@ -112,15 +113,15 @@ class SearchWindvane:
else:
# API error - stop pagination
if response_data.get('code') != 0:
print(f'Windvane subdomain API error: {response_data.get("msg", "Unknown error")}')
output_logger.info(f'Windvane subdomain API error: {response_data.get("msg", "Unknown error")}')
break
except Exception as e:
print(f'Windvane subdomain request failed: {e}')
output_logger.info(f'Windvane subdomain request failed: {e}')
break
except Exception as e:
print(f'Windvane subdomain search error: {e}')
output_logger.info(f'Windvane subdomain search error: {e}')
async def _search_dns_history(self, headers: dict) -> None:
"""Search DNS history using /ListDNS endpoint for additional subdomains and IPs"""
@@ -160,11 +161,11 @@ class SearchWindvane:
break
except Exception as e:
print(f'Windvane DNS history request failed: {e}')
output_logger.info(f'Windvane DNS history request failed: {e}')
break
except Exception as e:
print(f'Windvane DNS history search error: {e}')
output_logger.info(f'Windvane DNS history search error: {e}')
async def _search_emails(self, headers: dict) -> None:
"""Search for emails using /ListEmail endpoint"""
@@ -194,10 +195,10 @@ class SearchWindvane:
self.totalhosts.add(domain.lower())
except Exception as e:
print(f'Windvane email search request failed: {e}')
output_logger.info(f'Windvane email search request failed: {e}')
except Exception as e:
print(f'Windvane email search error: {e}')
output_logger.info(f'Windvane email search error: {e}')
async def _search_subdomains_limited(self, headers: dict) -> None:
"""Limited subdomain search without API key - tries simpler approaches"""
@@ -229,22 +230,22 @@ class SearchWindvane:
if domain and domain.endswith(self.word):
self.totalhosts.add(domain.lower())
print(f'[*] Found {len(subdomains)} subdomains with limited access')
output_logger.info(f'[*] Found {len(subdomains)} subdomains with limited access')
else:
# If API call fails, try fallback approaches
await self._fallback_search()
except Exception as e:
print(f'Windvane limited API failed: {e}')
output_logger.info(f'Windvane limited API failed: {e}')
await self._fallback_search()
except Exception as e:
print(f'Windvane limited search error: {e}')
output_logger.info(f'Windvane limited search error: {e}')
async def _fallback_search(self) -> None:
"""Fallback search using common subdomain patterns when API is unavailable"""
try:
print('[*] API unavailable, using fallback subdomain pattern search...')
output_logger.info('[*] API unavailable, using fallback subdomain pattern search...')
# Common subdomain prefixes to try
common_subdomains = [
@@ -296,12 +297,12 @@ class SearchWindvane:
continue
if found_count > 0:
print(f'[*] Found {found_count} subdomains using DNS fallback')
output_logger.info(f'[*] Found {found_count} subdomains using DNS fallback')
else:
print('[*] No additional subdomains found via fallback methods')
output_logger.info('[*] No additional subdomains found via fallback methods')
except Exception as e:
print(f'Fallback search error: {e}')
output_logger.info(f'Fallback search error: {e}')
def set_api_key(self, api_key: str) -> None:
"""Set the API key for authenticated requests
theHarvester/discovery/zoomeyesearch.py
+5 -4
View File
@@ -6,6 +6,7 @@ from typing import Any
from theHarvester.discovery.constants import MissingKey, get_delay
from theHarvester.lib.core import AsyncFetcher, Core
from theHarvester.lib.output import output_logger
from theHarvester.parsers import myparser
@@ -76,7 +77,7 @@ class SearchZoomEye:
# some responses put HTTP-like code here
return resp.get('status') in (0, 200)
except Exception as e:
print(f'An error occurred while trying to parse {resp} : {e}')
output_logger.info(f'An error occurred while trying to parse {resp} : {e}')
return False
# If no explicit status, assume success and let parsing validate
@@ -95,9 +96,9 @@ class SearchZoomEye:
try:
return int(payload['available'])
except ValueError:
print('Payload availablity is not a integer')
output_logger.info('Payload availablity is not a integer')
except Exception as e:
print(f'An error occurred in page_total_from_payload : {e}')
output_logger.info(f'An error occurred in page_total_from_payload : {e}')
total_results = payload.get('total') or payload.get('count') or payload.get('total_count')
if isinstance(total_results, int) and total_results >= 0:
size = payload.get('size') or page_size
@@ -350,7 +351,7 @@ class SearchZoomEye:
except Exception as e:
# Continue processing other matches instead of failing completely
print(f'ZoomEye parsing error: {e}')
output_logger.info(f'ZoomEye parsing error: {e}')
return hostnames, emails, ips, asns, iurls
theHarvester/lib/api/api.py
+4 -3
View File
@@ -17,6 +17,7 @@ from starlette.staticfiles import StaticFiles
from theHarvester import __main__
from theHarvester.lib.api.additional_endpoints import router as additional_router
from theHarvester.lib.output import output_logger
API_RATE_LIMIT = os.getenv('API_RATE_LIMIT', '5/minute')
@@ -185,7 +186,7 @@ async def getsources(request: Request) -> Response:
except Exception as e:
# Log the error and return a detailed error response
error_traceback = traceback.format_exc()
print(f'Error in getsources endpoint: {e!s}\n{error_traceback}')
output_logger.info(f'Error in getsources endpoint: {e!s}\n{error_traceback}')
return JSONResponse(
{
@@ -264,7 +265,7 @@ async def dnsbrute(
except Exception as e:
# Log the error and return a detailed error response
error_traceback = traceback.format_exc()
print(f'Error in dnsbrute endpoint: {e!s}\n{error_traceback}')
output_logger.info(f'Error in dnsbrute endpoint: {e!s}\n{error_traceback}')
return JSONResponse(
{
@@ -383,7 +384,7 @@ async def query(
except Exception as e:
# Log the error and return a detailed error response
error_traceback = traceback.format_exc()
print(f'Error in query endpoint: {e!s}\n{error_traceback}')
output_logger.info(f'Error in query endpoint: {e!s}\n{error_traceback}')
return JSONResponse(
{
theHarvester/lib/api/api_example.py
+19 -19
View File
@@ -5,7 +5,7 @@ import asyncio
import aiohttp
import netaddr
from theHarvester.lib.output import print_section, sorted_unique
from theHarvester.lib.output import output_logger, print_section, sorted_unique
async def fetch_json(session, url):
@@ -14,7 +14,7 @@ async def fetch_json(session, url):
response.raise_for_status() # Raise an exception for 4XX/5XX responses
return await response.json()
except Exception as e:
print(f'Error fetching data from {url}: {e}')
output_logger.info(f'Error fetching data from {url}: {e}')
return {}
@@ -24,7 +24,7 @@ async def fetch(session, url):
response.raise_for_status() # Raise an exception for 4XX/5XX responses
return await response.text()
except Exception as e:
print(f'Error fetching data from {url}: {e}')
output_logger.info(f'Error fetching data from {url}: {e}')
return ''
@@ -57,7 +57,7 @@ async def main() -> None:
interesting_urls = sorted_unique(interesting_urls)
if len(twitter_people_list_tracker) == 0:
print('\n[*] No Twitter users found.')
output_logger.info('\n[*] No Twitter users found.')
elif len(twitter_people_list_tracker) >= 1:
print_section(
'\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker)),
@@ -67,7 +67,7 @@ async def main() -> None:
twitter_people_list_tracker = sorted_unique(twitter_people_list_tracker)
if len(linkedin_people_list_tracker) == 0:
print('\n[*] No LinkedIn users found.')
output_logger.info('\n[*] No LinkedIn users found.')
elif len(linkedin_people_list_tracker) >= 1:
print_section(
'\n[*] LinkedIn Users found: ' + str(len(linkedin_people_list_tracker)),
@@ -77,7 +77,7 @@ async def main() -> None:
linkedin_people_list_tracker = sorted_unique(linkedin_people_list_tracker)
if len(linkedin_links_tracker) == 0:
print('\n[*] No LinkedIn links found.')
output_logger.info('\n[*] No LinkedIn links found.')
else:
print_section(
f'\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}', linkedin_links_tracker, '---------------------'
@@ -86,33 +86,33 @@ async def main() -> None:
length_urls = len(trello_urls)
if length_urls == 0:
print('\n[*] No Trello URLs found.')
output_logger.info('\n[*] No Trello URLs found.')
else:
print_section('\n[*] Trello URLs found: ' + str(length_urls), trello_urls, '--------------------')
if len(ips) == 0:
print('\n[*] No IPs found.')
output_logger.info('\n[*] No IPs found.')
else:
print('\n[*] IPs found: ' + str(len(ips)))
print('-------------------')
output_logger.info('\n[*] IPs found: ' + str(len(ips)))
output_logger.info('-------------------')
# use netaddr as the list may contain ipv4 and ipv6 addresses
ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(ips)])
print('\n'.join(map(str, ip_list)))
output_logger.info('\n'.join(map(str, ip_list)))
if len(emails) == 0:
print('\n[*] No emails found.')
output_logger.info('\n[*] No emails found.')
else:
print('\n[*] Emails found: ' + str(len(emails)))
print('----------------------')
output_logger.info('\n[*] Emails found: ' + str(len(emails)))
output_logger.info('----------------------')
all_emails = sorted_unique(emails)
print('\n'.join(all_emails))
output_logger.info('\n'.join(all_emails))
if len(hosts) == 0:
print('\n[*] No hosts found.\n\n')
output_logger.info('\n[*] No hosts found.\n\n')
else:
print('\n[*] Hosts found: ' + str(len(hosts)))
print('---------------------')
print('\n'.join(hosts))
output_logger.info('\n[*] Hosts found: ' + str(len(hosts)))
output_logger.info('---------------------')
output_logger.info('\n'.join(hosts))
if __name__ == '__main__':
theHarvester/lib/core.py
+19 -18
View File
@@ -16,6 +16,7 @@ import yaml
from aiohttp_socks import ProxyConnector
from theHarvester import __version__
from theHarvester.lib.output import output_logger
if TYPE_CHECKING:
from collections.abc import Sized
@@ -77,7 +78,7 @@ class Core:
file = path.expanduser() / filename
config = file.read_text()
if not Core.quiet:
print(f'Read {filename} from {file}')
output_logger.info(f'Read {filename} from {file}')
return config
# Fallback to creating default in the user's home dir
@@ -85,7 +86,7 @@ class Core:
dest = CONFIG_DIRS[0].expanduser() / filename
dest.parent.mkdir(exist_ok=True)
dest.write_text(default)
print(f'Created default {filename} at {dest}')
output_logger.info(f'Created default {filename} at {dest}')
return default
@staticmethod
@@ -259,19 +260,19 @@ class Core:
@staticmethod
def banner() -> None:
print('*******************************************************************')
print('* _ _ _ *')
print(r'* | |_| |__ ___ /\ /\__ _ _ ____ _____ ___| |_ ___ _ __ *')
print(r"* | __| _ \ / _ \ / /_/ / _` | '__\ \ / / _ \/ __| __/ _ \ '__| *")
print(r'* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *')
print(r'* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *')
print('* *')
print('* theHarvester {version}{filler}*'.format(version=__version__, filler=' ' * (51 - len(__version__))))
print('* Coded by Christian Martorella *')
print('* Edge-Security Research *')
print('* cmartorella@edge-security.com *')
print('* *')
print('*******************************************************************')
output_logger.info('*******************************************************************')
output_logger.info('* _ _ _ *')
output_logger.info(r'* | |_| |__ ___ /\ /\__ _ _ ____ _____ ___| |_ ___ _ __ *')
output_logger.info(r"* | __| _ \ / _ \ / /_/ / _` | '__\ \ / / _ \/ __| __/ _ \ '__| *")
output_logger.info(r'* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *')
output_logger.info(r'* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *')
output_logger.info('* *')
output_logger.info('* theHarvester {version}{filler}*'.format(version=__version__, filler=' ' * (51 - len(__version__))))
output_logger.info('* Coded by Christian Martorella *')
output_logger.info('* Edge-Security Research *')
output_logger.info('* cmartorella@edge-security.com *')
output_logger.info('* *')
output_logger.info('*******************************************************************')
@staticmethod
def get_supportedengines() -> list[str]:
@@ -667,7 +668,7 @@ class AsyncFetcher:
await asyncio.sleep(5)
return url, await response.text()
except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError) as e:
print(f'Takeover check error: {e}')
output_logger.info(f'Takeover check error: {e}')
return url, ''
@classmethod
@@ -743,5 +744,5 @@ class AsyncFetcher:
def show_default_error_message(engine_name: str, word: str, error) -> None:
print(f"Failed to process {engine_name} search for word: '{word}'")
print(f'Error Message: {error}')
output_logger.info(f"Failed to process {engine_name} search for word: '{word}'")
output_logger.info(f'Error Message: {error}')
theHarvester/lib/output.py
+42 -10
View File
@@ -1,11 +1,43 @@
from __future__ import annotations
import logging
import sys
from collections.abc import Hashable, Iterable, Sequence
from typing import TypeVar
T = TypeVar('T', bound=Hashable)
class _OperatorOutputHandler(logging.Handler):
"""Write operator-facing messages to the current stdout stream."""
def emit(self, record: logging.LogRecord) -> None:
try:
sys.stdout.write(f'{self.format(record)}\n')
except Exception:
self.handleError(record)
output_logger = logging.getLogger('theHarvester.output')
output_logger.setLevel(logging.INFO)
output_logger.propagate = False
if not any(isinstance(handler, _OperatorOutputHandler) for handler in output_logger.handlers):
output_logger.addHandler(_OperatorOutputHandler())
def configure_logging(*, verbose: bool) -> None:
"""Configure CLI diagnostics without taking ownership from an embedding host."""
root_logger = logging.getLogger()
if not root_logger.handlers:
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter('%(levelname)s %(name)s: %(message)s'))
root_logger.addHandler(handler)
root_logger.setLevel(logging.WARNING)
if verbose:
logging.getLogger('theHarvester').setLevel(logging.INFO)
def sorted_unique[T: Hashable](items: Iterable[T]) -> list[T]:
unique_items = list(dict.fromkeys(items))
unique_items.sort(key=lambda item: str(item))
@@ -13,25 +45,25 @@ def sorted_unique[T: Hashable](items: Iterable[T]) -> list[T]:
def print_section(header: str, items: Iterable[str], separator: str) -> None:
print(header)
print(separator)
output_logger.info(header)
output_logger.info(separator)
for item in sorted_unique(items):
print(item)
output_logger.info(item)
def print_linkedin_sections(
engines: Sequence[str], people: Sequence[str], links: Sequence[str], separator: str = '---------------------'
) -> None:
if len(people) == 0 and 'linkedin' in engines:
print('\n[*] No LinkedIn users found.\n\n')
output_logger.info('\n[*] No LinkedIn users found.\n\n')
elif len(people) >= 1:
print('\n[*] LinkedIn Users found: ' + str(len(people)))
print(separator)
output_logger.info('\n[*] LinkedIn Users found: ' + str(len(people)))
output_logger.info(separator)
for usr in sorted_unique(people):
print(usr)
output_logger.info(usr)
if 'linkedin' in engines or 'rocketreach' in engines:
print(f'\n[*] LinkedIn Links found: {len(links)}')
print(separator)
output_logger.info(f'\n[*] LinkedIn Links found: {len(links)}')
output_logger.info(separator)
for link in sorted_unique(links):
print(link)
output_logger.info(link)
theHarvester/lib/stash.py
+13 -11
View File
@@ -5,6 +5,8 @@ from sqlite3.dbapi2 import Row
import aiosqlite
from theHarvester.lib.output import output_logger
db_path = os.path.expanduser('~/.local/share/theHarvester')
if not os.path.isdir(db_path):
@@ -56,7 +58,7 @@ class StashManager:
)
await db.commit()
except Exception as e:
print(f'Unexpected error while storing result: {e}')
output_logger.info(f'Unexpected error while storing result: {e}')
async def store_all(self, domain, all, res_type, source) -> None:
# people are not stored in the database
@@ -77,7 +79,7 @@ class StashManager:
)
await db.commit()
except Exception as e:
print(f'Unexpected error while storing result: {e}')
output_logger.info(f'Unexpected error while storing result: {e}')
async def generatedashboardcode(self, domain):
try:
@@ -165,7 +167,7 @@ class StashManager:
self.latestscandomain['scandetailsshodan'] = scandetailsshodan
return self.latestscandomain
except Exception as e:
print(f'Unexpected error while generating the dashboard code: {e}')
output_logger.info(f'Unexpected error while generating the dashboard code: {e}')
async def getlatestscanresults(self, domain, previousday: bool = False) -> Iterable[Row | str] | None:
try:
@@ -206,7 +208,7 @@ class StashManager:
self.previousscanresults = list(results)
return self.previousscanresults
except Exception as e:
print(f'Error in getting the previous scan results from the database: {e}')
output_logger.info(f'Error in getting the previous scan results from the database: {e}')
else:
try:
cursor = await conn.execute(
@@ -231,9 +233,9 @@ class StashManager:
self.latestscanresults = list(results)
return self.latestscanresults
except Exception as e:
print(f'Error in getting the latest scan results from the database: {e}')
output_logger.info(f'Error in getting the latest scan results from the database: {e}')
except Exception as e:
print(f'Error connecting to theHarvester database: {e}')
output_logger.info(f'Error connecting to theHarvester database: {e}')
return self.latestscanresults
async def getscanboarddata(self):
@@ -259,7 +261,7 @@ class StashManager:
self.scanboarddata['domains'] = self._col0_int(data)
return self.scanboarddata
except Exception as e:
print(f'Unexpected error while getting the scanboard data: {e}')
output_logger.info(f'Unexpected error while getting the scanboard data: {e}')
async def getscanhistorydomain(self, domain):
try:
@@ -306,7 +308,7 @@ class StashManager:
self.domainscanhistory.append(results)
return self.domainscanhistory
except Exception as e:
print(f'Unexpected error while getting the scanhistory of a domain: {e}')
output_logger.info(f'Unexpected error while getting the scanhistory of a domain: {e}')
async def getpluginscanstatistics(self) -> Iterable[Row] | None:
try:
@@ -321,7 +323,7 @@ class StashManager:
results = await cursor.fetchall()
self.scanstats = list(results)
except Exception as e:
print(f'Unexpected error while getting a plugins scanstatistics: {e}')
output_logger.info(f'Unexpected error while getting a plugins scanstatistics: {e}')
return self.scanstats
async def latestscanchartdata(self, domain):
@@ -409,6 +411,6 @@ class StashManager:
self.latestscandomain['scandetailsshodan'] = scandetailsshodan
return self.latestscandomain
except aiosqlite.Error as db_err:
print(f"Database error occurred for domain '{domain}': {db_err}")
output_logger.info(f"Database error occurred for domain '{domain}': {db_err}")
except Exception as e:
print(f"Unexpected error in latestscanchartdata for domain '{domain}': {e}")
output_logger.info(f"Unexpected error in latestscanchartdata for domain '{domain}': {e}")
theHarvester/screenshot/screenshot.py
+9 -7
View File
@@ -13,6 +13,8 @@ import certifi
from aiohttp_socks import ProxyConnector
from playwright.async_api import async_playwright
from theHarvester.lib.output import output_logger
class ScreenShotter:
def __init__(self, output) -> None:
@@ -31,7 +33,7 @@ class ScreenShotter:
return False
return True
except Exception as e:
print(f"An exception has occurred while attempting to verify output path's existence: {e}")
output_logger.info(f"An exception has occurred while attempting to verify output path's existence: {e}")
return False
@staticmethod
@@ -41,9 +43,9 @@ class ScreenShotter:
async with async_playwright() as p:
browser = await p.chromium.launch()
await browser.close()
print('Playwright and Chromium are successfully installed.')
output_logger.info('Playwright and Chromium are successfully installed.')
except Exception as e:
print(f'An exception has occurred while attempting to verify installation: {e}')
output_logger.info(f'An exception has occurred while attempting to verify installation: {e}')
@staticmethod
def chunk_list(items: Collection, chunk_size: int) -> list:
@@ -86,13 +88,13 @@ class ScreenShotter:
text = await resp.text('UTF-8')
return f'http://{url}' if not url.startswith('http') else url, text
except Exception as e:
print(f'An exception has occurred while attempting to visit {url} : {e}')
output_logger.info(f'An exception has occurred while attempting to visit {url} : {e}')
return '', ''
async def take_screenshot(self, url: str) -> None:
url = f'http://{url}' if not url.startswith('http') else url
url = url.replace('www.', '')
print(f'Attempting to take a screenshot of: {url}')
output_logger.info(f'Attempting to take a screenshot of: {url}')
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
# New browser context
@@ -106,10 +108,10 @@ class ScreenShotter:
await page.goto(url, timeout=35000)
await page.screenshot(path=path)
except Exception as e:
print(f'An exception has occurred attempting to screenshot: {url} : {e}')
output_logger.info(f'An exception has occurred attempting to screenshot: {url} : {e}')
path = ''
finally:
await page.close()
await context.close()
await browser.close()
print(date, url, path)
output_logger.info('%s %s %s', date, url, path)