diff --git a/pyproject.toml b/pyproject.toml index 3a0a2066..b45321cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,7 +139,8 @@ select = ["E", "PT", "TC", "FURB", - "ASYNC" + "ASYNC", + "T20" ] ignore = [ "B018", diff --git a/tests/test_logging.py b/tests/test_logging.py index fba6d9e8..bfabc57f 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,30 +1,84 @@ from __future__ import annotations +import ast import logging import os import subprocess import sys -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from pathlib import Path +import textwrap +from pathlib import Path -def test_import_does_not_configure_application_logging() -> None: - result = subprocess.run( - [ - sys.executable, - '-c', - 'import logging; import theHarvester.__main__; print(len(logging.getLogger().handlers))', - ], +def run_python(script: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, '-c', textwrap.dedent(script)], check=True, capture_output=True, text=True, ) + +def test_import_does_not_configure_application_logging() -> None: + result = run_python( + """ + import logging + import sys + import theHarvester.__main__ + sys.stdout.write(str(len(logging.getLogger().handlers)) + '\\n') + """ + ) + assert result.stdout.splitlines()[-1] == '0' +def test_operator_output_uses_stdout_without_verbose_logging() -> None: + result = run_python( + """ + from theHarvester.lib.output import configure_logging, output_logger + + configure_logging(verbose=False) + output_logger.info('operator result') + """ + ) + + assert result.stdout == 'operator result\n' + assert result.stderr == '' + + +def test_diagnostics_use_stderr_only_when_verbose() -> None: + result = run_python( + """ + import logging + from theHarvester.lib.output import configure_logging + + logger = logging.getLogger('theHarvester.discovery.example') + configure_logging(verbose=False) + logger.info('hidden diagnostic') + configure_logging(verbose=True) + logger.info('visible diagnostic') + """ + ) + + assert result.stdout == '' + assert 'hidden diagnostic' not in result.stderr + assert 'INFO theHarvester.discovery.example: visible diagnostic' in result.stderr + + +def test_production_code_has_no_print_calls() -> None: + package_root = Path(__file__).parents[1] / 'theHarvester' + print_calls = [] + + for path in package_root.rglob('*.py'): + tree = ast.parse(path.read_text()) + print_calls.extend( + f'{path.relative_to(package_root)}:{node.lineno}' + for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == 'print' + ) + + assert print_calls == [] + + def test_verbose_enables_info_diagnostics(tmp_path: Path) -> None: script = """import asyncio import logging diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 0e8703bf..8692294d 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -78,7 +78,7 @@ from theHarvester.discovery import ( from theHarvester.discovery.constants import MissingKey from theHarvester.lib import hostchecker, stash from theHarvester.lib.core import DATA_DIR, Core, show_default_error_message -from theHarvester.lib.output import print_linkedin_sections, print_section, sorted_unique +from theHarvester.lib.output import configure_logging, output_logger, print_linkedin_sections, print_section, sorted_unique from theHarvester.screenshot.screenshot import ScreenShotter if TYPE_CHECKING: @@ -229,12 +229,8 @@ async def start(rest_args: argparse.Namespace | None = None): args = parser.parse_args() filename = args.filename dnsbrute = (args.dns_brute, False) - logging.basicConfig( - level=logging.WARNING, - format='%(levelname)s %(name)s: %(message)s', - ) + configure_logging(verbose=args.verbose) if args.verbose: - logging.getLogger('theHarvester').setLevel(logging.INFO) logger.info('Verbose logging enabled') Core.quiet = getattr(args, 'quiet', False) try: @@ -242,7 +238,7 @@ async def start(rest_args: argparse.Namespace | None = None): await db.do_init() except (AttributeError, OSError, RuntimeError, ValueError) as init_error: if not args.quiet: - print(f'Error initializing StashManager: {init_error}') + output_logger.info(f'Error initializing StashManager: {init_error}') raise ValueError('Failed to initialize StashManager') if len(filename) > 0: @@ -283,8 +279,8 @@ async def start(rest_args: argparse.Namespace | None = None): _ = netaddr.IPAddress(line) final_dns_resolver_list.append(line) except (netaddr.core.AddrFormatError, ValueError, TypeError) as e: - print(f'An exception has occurred while reading from: {dnsresolve}, {e}') - print(f'Current line: {line}') + output_logger.info(f'An exception has occurred while reading from: {dnsresolve}, {e}') + output_logger.info(f'Current line: {line}') else: cleaned = dnsresolve.replace(' ', '') resolver_candidates = cleaned.split(',') if ',' in cleaned else [cleaned] @@ -296,12 +292,12 @@ async def start(rest_args: argparse.Namespace | None = None): _ = netaddr.IPAddress(item) final_dns_resolver_list.append(item) except (netaddr.core.AddrFormatError, ValueError, TypeError) as e: - print(f'Passed DNS resolver is invalid, skipping: {item} ({e})') + output_logger.info(f'Passed DNS resolver is invalid, skipping: {item} ({e})') # if for some reason, there are duplicates final_dns_resolver_list = list(set(final_dns_resolver_list)) if len(final_dns_resolver_list) == 0: - print('No valid DNS resolvers were parsed from --dns-resolve; continuing without custom resolvers.') + output_logger.info('No valid DNS resolvers were parsed from --dns-resolve; continuing without custom resolvers.') engines: list = [] # If the user specifies @@ -357,15 +353,20 @@ async def start(rest_args: argparse.Namespace | None = None): :param store_interestingurls: whether to store interesting urls :param store_asns: whether to store asns """ - ( - await search_engine.process(use_proxy) - if process_param is None - else await search_engine.process(process_param, use_proxy) - ) + logger.info('Source %s started', source) + try: + ( + await search_engine.process(use_proxy) + if process_param is None + else await search_engine.process(process_param, use_proxy) + ) + except Exception: + logger.exception('Source %s failed', source) + raise db_stash = stash.StashManager() if source: - print(f'[*] Searching {source[0].upper() + source[1:]}. ') + output_logger.info(f'[*] Searching {source[0].upper() + source[1:]}. ') if store_host: host_names = list({host for host in await search_engine.get_hostnames() if f'.{word}' in host}) @@ -433,6 +434,7 @@ async def start(rest_args: argparse.Namespace | None = None): total_asns.extend(fasns) if len(fasns) > 0: await db.store_all(word, fasns, 'asns', source) + logger.info('Source %s completed', source) stor_lst = [] if args.source is not None: @@ -442,7 +444,7 @@ async def start(rest_args: argparse.Namespace | None = None): engines = Core.get_supportedengines() # Iterate through search engines in order if set(engines).issubset(Core.get_supportedengines()): - print(f'\n[*] Target: {word} \n') + output_logger.info(f'\n[*] Target: {word} \n') for engineitem in engines: if engineitem == 'baidu': @@ -486,7 +488,7 @@ async def start(rest_args: argparse.Namespace | None = None): ) except Exception as ex: if isinstance(ex, MissingKey): - print(MissingKey('Bitbucket')) + output_logger.info(MissingKey('Bitbucket')) else: show_default_error_message(engineitem, word, ex) @@ -524,8 +526,8 @@ async def start(rest_args: argparse.Namespace | None = None): stor_lst.append(store(builtwith_search, engineitem, store_host=True, store_interestingurls=True)) except Exception as e: if isinstance(e, MissingKey): - print(f"Failed to perform BuiltWith search for word: '{word}'") - print(f'A Missing Key Error occurred in builtwith: {e}') + output_logger.info(f"Failed to perform BuiltWith search for word: '{word}'") + output_logger.info(f'A Missing Key Error occurred in builtwith: {e}') else: show_default_error_message(engineitem, word, e) @@ -542,19 +544,19 @@ async def start(rest_args: argparse.Namespace | None = None): ) except MissingKey as mk: if not args.quiet: - print(f'Censys API key is missing or invalid: {mk}') + output_logger.info(f'Censys API key is missing or invalid: {mk}') except ConnectionError as ce: if not args.quiet: - print(f'Network error while querying Censys: {ce}') + output_logger.info(f'Network error while querying Censys: {ce}') except TimeoutError as te: if not args.quiet: - print(f'Timeout occurred while contacting Censys: {te}') + output_logger.info(f'Timeout occurred while contacting Censys: {te}') except ValueError as ve: if not args.quiet: - print(f'Censys returned unexpected data: {ve}') + output_logger.info(f'Censys returned unexpected data: {ve}') except Exception as e: if not args.quiet: - print(f'Unexpected error occurred in Censys module: {e}') + output_logger.info(f'Unexpected error occurred in Censys module: {e}') elif engineitem == 'certspotter': try: @@ -562,19 +564,19 @@ async def start(rest_args: argparse.Namespace | None = None): stor_lst.append(store(certspotter_search, engineitem, None, store_host=True)) except ConnectionError as ce: if not args.quiet: - print(f'Network connection error while accessing Certspotter: {ce}') + output_logger.info(f'Network connection error while accessing Certspotter: {ce}') except TimeoutError as te: if not args.quiet: - print(f'Request to Certspotter timed out: {te}') + output_logger.info(f'Request to Certspotter timed out: {te}') except ValueError as ve: if not args.quiet: - print(f'Certspotter returned invalid data: {ve}') + output_logger.info(f'Certspotter returned invalid data: {ve}') except MissingKey as mk: if not args.quiet: - print(f'Unexpected response structure from Certspotter (missing key): {mk}') + output_logger.info(f'Unexpected response structure from Certspotter (missing key): {mk}') except Exception as e: if not args.quiet: - print(f'Unexpected error occurred in Certspotter module: {e}') + output_logger.info(f'Unexpected error occurred in Certspotter module: {e}') elif engineitem == 'chaos': try: @@ -589,7 +591,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in Chaos: {e}') + output_logger.info(f'A Missing Key error occurred in Chaos: {e}') else: show_default_error_message(engineitem, word, e) @@ -621,7 +623,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing key error occurred in criminalip: {e}') + output_logger.info(f'A Missing key error occurred in criminalip: {e}') else: show_default_error_message(engineitem, word, e) @@ -630,7 +632,7 @@ async def start(rest_args: argparse.Namespace | None = None): crtsh_search = crtsh.SearchCrtsh(word) stor_lst.append(store(crtsh_search, 'CRTsh', store_host=True)) except Exception as e: - print(f'[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}') + output_logger.info(f'[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}') elif engineitem == 'dehashed': try: @@ -646,7 +648,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in dehashed: {e}') + output_logger.info(f'A Missing Key error occurred in dehashed: {e}') else: show_default_error_message(engineitem, word, e) @@ -663,7 +665,7 @@ async def start(rest_args: argparse.Namespace | None = None): ) except MissingKey as e: if not args.quiet: - print(e) + output_logger.info(e) except Exception as e: show_default_error_message(engineitem, word, e) @@ -685,7 +687,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in dymo: {e}') + output_logger.info(f'A Missing Key error occurred in dymo: {e}') else: show_default_error_message(engineitem, word, e) @@ -703,7 +705,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in Fofa: {e}') + output_logger.info(f'A Missing Key error occurred in Fofa: {e}') else: show_default_error_message(engineitem, word, e) @@ -714,7 +716,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in fullhunt: {e}') + output_logger.info(f'A Missing Key error occurred in fullhunt: {e}') elif engineitem == 'github-code': try: @@ -729,7 +731,7 @@ async def start(rest_args: argparse.Namespace | None = None): ) except MissingKey as ex: if not args.quiet: - print(f'A Missing Key error occurred in github-code: {ex}') + output_logger.info(f'A Missing Key error occurred in github-code: {ex}') elif engineitem == 'gitlab': try: @@ -765,9 +767,9 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(MissingKey('HaveIBeenPwned')) + output_logger.info(MissingKey('HaveIBeenPwned')) else: - print(f'An exception has occurred in HaveIBeenPwned search: {e}') + output_logger.info(f'An exception has occurred in HaveIBeenPwned search: {e}') elif engineitem == 'hudsonrock': try: @@ -782,7 +784,7 @@ async def start(rest_args: argparse.Namespace | None = None): ) ) except Exception as e: - print(f'An exception has occurred in Hudson Rock search: {e}') + output_logger.info(f'An exception has occurred in Hudson Rock search: {e}') elif engineitem == 'hunter': try: @@ -798,7 +800,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in Hunter: {e}') + output_logger.info(f'A Missing Key error occurred in Hunter: {e}') elif engineitem == 'hunterhow': try: @@ -807,9 +809,9 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in Hunter How: {e}') + output_logger.info(f'A Missing Key error occurred in Hunter How: {e}') else: - print(f'An exception has occurred in hunterhow search: {e}') + output_logger.info(f'An exception has occurred in hunterhow search: {e}') elif engineitem == 'intelx': try: @@ -825,9 +827,9 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in intelx: {e}') + output_logger.info(f'A Missing Key error occurred in intelx: {e}') else: - print(f'An exception has occurred in Intelx search: {e}') + output_logger.info(f'An exception has occurred in Intelx search: {e}') elif engineitem == 'leakix': try: @@ -855,9 +857,9 @@ async def start(rest_args: argparse.Namespace | None = None): ) except Exception as e: if isinstance(e, MissingKey): - print(f'A Missing Key error occurred in LeakLookup: {e}') + output_logger.info(f'A Missing Key error occurred in LeakLookup: {e}') else: - print(f'An exception has occurred in LeakLookup search: {e}') + output_logger.info(f'An exception has occurred in LeakLookup search: {e}') elif engineitem == 'mojeek': try: @@ -872,9 +874,9 @@ async def start(rest_args: argparse.Namespace | None = None): ) except Exception as e: if isinstance(e, MissingKey): - print(f'A Missing Key error occurred in Mojeek: {e}') + output_logger.info(f'A Missing Key error occurred in Mojeek: {e}') else: - print(f'An exception has occurred in Mojeek search: {e}') + output_logger.info(f'An exception has occurred in Mojeek search: {e}') elif engineitem == 'netlas': try: @@ -890,7 +892,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in Netlas: {e}') + output_logger.info(f'A Missing Key error occurred in Netlas: {e}') elif engineitem == 'onyphe': try: @@ -906,19 +908,19 @@ async def start(rest_args: argparse.Namespace | None = None): ) except ConnectionError as ce: if not args.quiet: - print(f'Network connection error while accessing Onyphe: {ce}') + output_logger.info(f'Network connection error while accessing Onyphe: {ce}') except TimeoutError as te: if not args.quiet: - print(f'Request to Onyphe timed out: {te}') + output_logger.info(f'Request to Onyphe timed out: {te}') except ValueError as ve: if not args.quiet: - print(f'Onyphe returned invalid or unexpected data: {ve}') + output_logger.info(f'Onyphe returned invalid or unexpected data: {ve}') except KeyError as ke: if not args.quiet: - print(f'Unexpected response structure from Onyphe (missing key): {ke}') + output_logger.info(f'Unexpected response structure from Onyphe (missing key): {ke}') except Exception as e: if not args.quiet: - print(f'Unexpected error occurred in Onyphe module: {e}') + output_logger.info(f'Unexpected error occurred in Onyphe module: {e}') elif engineitem == 'otx': try: @@ -933,19 +935,19 @@ async def start(rest_args: argparse.Namespace | None = None): ) except ConnectionError as ce: if not args.quiet: - print(f'Network connection error while accessing OTX: {ce}') + output_logger.info(f'Network connection error while accessing OTX: {ce}') except TimeoutError as te: if not args.quiet: - print(f'Request to OTX timed out: {te}') + output_logger.info(f'Request to OTX timed out: {te}') except ValueError as ve: if not args.quiet: - print(f'OTX returned invalid or unexpected data: {ve}') + output_logger.info(f'OTX returned invalid or unexpected data: {ve}') except KeyError as ke: if not args.quiet: - print(f'Unexpected response structure from OTX (missing key): {ke}') + output_logger.info(f'Unexpected response structure from OTX (missing key): {ke}') except Exception as e: if not args.quiet: - print(f'Unexpected error occurred in OTX module: {e}') + output_logger.info(f'Unexpected error occurred in OTX module: {e}') elif engineitem == 'pentesttools': try: @@ -954,9 +956,9 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in PentestTools search: {e}') + output_logger.info(f'A Missing Key error occurred in PentestTools search: {e}') else: - print(f'An exception has occurred in PentestTools search: {e}') + output_logger.info(f'An exception has occurred in PentestTools search: {e}') elif engineitem == 'projectdiscovery': try: @@ -965,9 +967,9 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in ProjectDiscovery: {e}') + output_logger.info(f'A Missing Key error occurred in ProjectDiscovery: {e}') else: - print('An exception has occurred in ProjectDiscovery') + output_logger.info('An exception has occurred in ProjectDiscovery') elif engineitem == 'rapiddns': try: @@ -975,19 +977,19 @@ async def start(rest_args: argparse.Namespace | None = None): stor_lst.append(store(rapiddns_search, engineitem, store_host=True)) except ConnectionError as ce: if not args.quiet: - print(f'Network connection error while accessing RapidDNS: {ce}') + output_logger.info(f'Network connection error while accessing RapidDNS: {ce}') except TimeoutError as te: if not args.quiet: - print(f'Request to RapidDNS timed out: {te}') + output_logger.info(f'Request to RapidDNS timed out: {te}') except ValueError as ve: if not args.quiet: - print(f'RapidDNS returned invalid or unexpected data: {ve}') + output_logger.info(f'RapidDNS returned invalid or unexpected data: {ve}') except KeyError as ke: if not args.quiet: - print(f'Unexpected response structure from RapidDNS (missing key): {ke}') + output_logger.info(f'Unexpected response structure from RapidDNS (missing key): {ke}') except Exception as e: if not args.quiet: - print(f'Unexpected error occurred in RapidDNS module: {e}') + output_logger.info(f'Unexpected error occurred in RapidDNS module: {e}') elif engineitem == 'robtex': try: @@ -1010,9 +1012,9 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in RocketReach: {e}') + output_logger.info(f'A Missing Key error occurred in RocketReach: {e}') else: - print(f'An exception has occurred in RocketReach: {e}') + output_logger.info(f'An exception has occurred in RocketReach: {e}') elif engineitem == 'securityscorecard': try: @@ -1029,9 +1031,9 @@ async def start(rest_args: argparse.Namespace | None = None): ) except Exception as e: if isinstance(e, MissingKey): - print(MissingKey('SecurityScorecard')) + output_logger.info(MissingKey('SecurityScorecard')) else: - print(f'An exception has occurred in SecurityScorecard search: {e}') + output_logger.info(f'An exception has occurred in SecurityScorecard search: {e}') elif engineitem == 'securityTrails': try: @@ -1047,7 +1049,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred Security Trails: {e}') + output_logger.info(f'A Missing Key error occurred Security Trails: {e}') elif engineitem == 'sherlockeye': try: @@ -1064,7 +1066,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in sherlockeye: {e}') + output_logger.info(f'A Missing Key error occurred in sherlockeye: {e}') else: show_default_error_message(engineitem, word, e) @@ -1085,7 +1087,7 @@ async def start(rest_args: argparse.Namespace | None = None): try: # Resolve domain to IP and search in Shodan ip = socket.gethostbyname(self.word) - print(f'\tSearching Shodan for {ip}') + output_logger.info(f'\tSearching Shodan for {ip}') result = await self.shodan.search_ip(ip) if ip in result and isinstance(result[ip], dict): # Add the IP as a host for consistency with other modules @@ -1094,11 +1096,11 @@ async def start(rest_args: argparse.Namespace | None = None): for host in result[ip].get('hostnames', []): self.hosts.add(host) - print(f'Found Shodan data for {ip}') + output_logger.info(f'Found Shodan data for {ip}') elif ip in result and isinstance(result[ip], str): - print(f'{ip}: {result[ip]}') + output_logger.info(f'{ip}: {result[ip]}') except Exception as e: - print(f'Error in Shodan search: {e}') + output_logger.info(f'Error in Shodan search: {e}') async def get_hostnames(self): return list(self.hosts) @@ -1108,9 +1110,9 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in Shodan search: {e}') + output_logger.info(f'A Missing Key error occurred in Shodan search: {e}') else: - print(f'An exception has occurred in Shodan search: {e}') + output_logger.info(f'An exception has occurred in Shodan search: {e}') elif engineitem == 'shodanInternetDB': try: @@ -1125,13 +1127,13 @@ async def start(rest_args: argparse.Namespace | None = None): ) except ConnectionError as ce: if not args.quiet: - print(f'Network connection error while accessing Shodan InternetDB: {ce}') + output_logger.info(f'Network connection error while accessing Shodan InternetDB: {ce}') except TimeoutError as te: if not args.quiet: - print(f'Request to Shodan InternetDB timed out: {te}') + output_logger.info(f'Request to Shodan InternetDB timed out: {te}') except Exception as e: if not args.quiet: - print(f'Unexpected error occurred in Shodan InternetDB module: {e}') + output_logger.info(f'Unexpected error occurred in Shodan InternetDB module: {e}') elif engineitem == 'subdomaincenter': try: @@ -1139,19 +1141,19 @@ async def start(rest_args: argparse.Namespace | None = None): stor_lst.append(store(subdomaincenter_search, engineitem, store_host=True)) except ConnectionError as ce: if not args.quiet: - print(f'Network connection error while accessing SubdomainCenter: {ce}') + output_logger.info(f'Network connection error while accessing SubdomainCenter: {ce}') except TimeoutError as te: if not args.quiet: - print(f'Request to SubdomainCenter timed out: {te}') + output_logger.info(f'Request to SubdomainCenter timed out: {te}') except ValueError as ve: if not args.quiet: - print(f'SubdomainCenter returned invalid or unexpected data: {ve}') + output_logger.info(f'SubdomainCenter returned invalid or unexpected data: {ve}') except KeyError as ke: if not args.quiet: - print(f'Unexpected response structure from SubdomainCenter (missing key): {ke}') + output_logger.info(f'Unexpected response structure from SubdomainCenter (missing key): {ke}') except Exception as e: if not args.quiet: - print(f'Unexpected error occurred in SubdomainCenter module: {e}') + output_logger.info(f'Unexpected error occurred in SubdomainCenter module: {e}') elif engineitem == 'subdomainfinderc99': try: @@ -1160,9 +1162,9 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in Subdomainfinderc99 search: {e}') + output_logger.info(f'A Missing Key error occurred in Subdomainfinderc99 search: {e}') else: - print(f'An exception has occurred in Subdomainfinderc99 search: {e}') + output_logger.info(f'An exception has occurred in Subdomainfinderc99 search: {e}') elif engineitem == 'thc': try: @@ -1199,7 +1201,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in Tomba: {e}') + output_logger.info(f'A Missing Key error occurred in Tomba: {e}') elif engineitem == 'urlscan': try: @@ -1216,19 +1218,19 @@ async def start(rest_args: argparse.Namespace | None = None): ) except ConnectionError as ce: if not args.quiet: - print(f'Network connection error while accessing Urlscan: {ce}') + output_logger.info(f'Network connection error while accessing Urlscan: {ce}') except TimeoutError as te: if not args.quiet: - print(f'Request to Urlscan timed out: {te}') + output_logger.info(f'Request to Urlscan timed out: {te}') except ValueError as ve: if not args.quiet: - print(f'Urlscan returned invalid or unexpected data: {ve}') + output_logger.info(f'Urlscan returned invalid or unexpected data: {ve}') except KeyError as ke: if not args.quiet: - print(f'Unexpected response structure from Urlscan (missing key): {ke}') + output_logger.info(f'Unexpected response structure from Urlscan (missing key): {ke}') except Exception as e: if not args.quiet: - print(f'Unexpected error occurred in Urlscan module: {e}') + output_logger.info(f'Unexpected error occurred in Urlscan module: {e}') elif engineitem == 'venacus': try: @@ -1246,9 +1248,9 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in venacus search: {e}') + output_logger.info(f'A Missing Key error occurred in venacus search: {e}') else: - print(f'An exception has occurred in venacus search: {e}') + output_logger.info(f'An exception has occurred in venacus search: {e}') elif engineitem == 'virustotal': try: @@ -1257,7 +1259,7 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in virustotal search: {e}') + output_logger.info(f'A Missing Key error occurred in virustotal search: {e}') elif engineitem == 'waybackarchive': try: @@ -1279,9 +1281,9 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in whoisxml search: {e}') + output_logger.info(f'A Missing Key error occurred in whoisxml search: {e}') else: - print(f'An exception has occurred in WhoisXML search: {e}') + output_logger.info(f'An exception has occurred in WhoisXML search: {e}') elif engineitem == 'windvane': try: @@ -1311,19 +1313,19 @@ async def start(rest_args: argparse.Namespace | None = None): ) except ConnectionError as ce: if not args.quiet: - print(f'Network connection error while accessing Yahoo: {ce}') + output_logger.info(f'Network connection error while accessing Yahoo: {ce}') except TimeoutError as te: if not args.quiet: - print(f'Request to Yahoo timed out: {te}') + output_logger.info(f'Request to Yahoo timed out: {te}') except ValueError as ve: if not args.quiet: - print(f'Yahoo returned invalid or unexpected data: {ve}') + output_logger.info(f'Yahoo returned invalid or unexpected data: {ve}') except KeyError as ke: if not args.quiet: - print(f'Unexpected response structure from Yahoo (missing key): {ke}') + output_logger.info(f'Unexpected response structure from Yahoo (missing key): {ke}') except Exception as e: if not args.quiet: - print(f'Unexpected error occurred in Yahoo module: {e}') + output_logger.info(f'Unexpected error occurred in Yahoo module: {e}') elif engineitem == 'zoomeye': try: @@ -1342,20 +1344,20 @@ async def start(rest_args: argparse.Namespace | None = None): except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(f'A Missing Key error occurred in zoomeye: {e}') + output_logger.info(f'A Missing Key error occurred in zoomeye: {e}') elif rest_args is not None: try: rest_args.dns_brute except AttributeError: - print('\n[!] Invalid source.\n') + output_logger.info('\n[!] Invalid source.\n') sys.exit(1) else: # Print which engines aren't supported unsupported_engines = set(engines) - set(Core.get_supportedengines()) if unsupported_engines: - print(f'The following engines are not supported: {unsupported_engines}') - print('\n[!] Invalid source.\n') + output_logger.info(f'The following engines are not supported: {unsupported_engines}') + output_logger.info('\n[!] Invalid source.\n') sys.exit(1) async def worker(queue): @@ -1367,7 +1369,7 @@ async def start(rest_args: argparse.Namespace | None = None): queue.task_done() # Notify the queue that the "work item" has been processed. except Exception as work_item_error: - print( + output_logger.info( f'\n An error occurred while processing a "work item": {type(work_item_error).__name__}: {work_item_error}\n' ) queue.task_done() @@ -1416,12 +1418,12 @@ async def start(rest_args: argparse.Namespace | None = None): try: all_emails except NameError: - print('\n\n[!] No emails found because all_emails is not defined.\n\n ') + output_logger.info('\n\n[!] No emails found because all_emails is not defined.\n\n ') sys.exit(1) try: all_hosts except NameError: - print('\n\n[!] No hosts found because all_hosts is not defined.\n\n ') + output_logger.info('\n\n[!] No hosts found because all_hosts is not defined.\n\n ') sys.exit(1) # Results @@ -1434,7 +1436,7 @@ async def start(rest_args: argparse.Namespace | None = None): interesting_urls = sorted_unique(interesting_urls) if len(twitter_people_list_tracker) == 0 and 'twitter' in engines: - print('\n[*] No Twitter users found.\n\n') + output_logger.info('\n[*] No Twitter users found.\n\n') elif len(twitter_people_list_tracker) >= 1: print_section( '\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker)), @@ -1450,17 +1452,17 @@ async def start(rest_args: argparse.Namespace | None = None): length_urls = len(all_urls) if length_urls == 0: if len(engines) >= 1 and 'trello' in engines: - print('\n[*] No Trello URLs found.') + output_logger.info('\n[*] No Trello URLs found.') else: total = length_urls print_section('\n[*] Trello URLs found: ' + str(total), all_urls, '--------------------') all_urls = sorted_unique(all_urls) if len(all_ip) == 0: - print('\n[*] No IPs found.') + output_logger.info('\n[*] No IPs found.') else: - print('\n[*] IPs found: ' + str(len(all_ip))) - print('-------------------') + output_logger.info('\n[*] IPs found: ' + str(len(all_ip))) + output_logger.info('-------------------') # use netaddr as the list may contain ipv4 and ipv6 addresses ip_list = [] for ip in set(all_ip): @@ -1472,31 +1474,31 @@ async def start(rest_args: argparse.Namespace | None = None): else: ip_list.append(str(netaddr.IPAddress(ip))) except (netaddr.core.AddrFormatError, ValueError, TypeError) as e: - print(f'An exception has occurred while adding: {ip} to ip_list: {e}') + output_logger.info(f'An exception has occurred while adding: {ip} to ip_list: {e}') continue ip_list = list(sorted(ip_list)) - print('\n'.join(map(str, ip_list))) + output_logger.info('\n'.join(map(str, ip_list))) # Populate host_ip from ip_list for DNS lookup, virtual hosts search, and Shodan search host_ip = ip_list if len(all_emails) == 0: - print('\n[*] No emails found.') + output_logger.info('\n[*] No emails found.') else: - print('\n[*] Emails found: ' + str(len(all_emails))) - print('----------------------') + output_logger.info('\n[*] Emails found: ' + str(len(all_emails))) + output_logger.info('----------------------') all_emails = sorted(list(set(all_emails))) - print('\n'.join(all_emails)) + output_logger.info('\n'.join(all_emails)) if len(all_people) == 0: - print('\n[*] No people found.') + output_logger.info('\n[*] No people found.') else: - print('\n[*] People found: ' + str(len(all_people))) - print('----------------------') + output_logger.info('\n[*] People found: ' + str(len(all_people))) + output_logger.info('----------------------') for person in all_people: - print(person) + output_logger.info(person) if len(all_hosts) == 0: - print('\n[*] No hosts found.\n\n') + output_logger.info('\n[*] No hosts found.\n\n') else: db = stash.StashManager() if dnsresolve is None or len(final_dns_resolver_list) > 0: @@ -1516,28 +1518,28 @@ async def start(rest_args: argparse.Namespace | None = None): temp.add(host) full = list(sorted(temp)) full.sort(key=lambda el: el.split(':')[0]) - print('\n[*] Hosts found: ' + str(len(full))) - print('---------------------') + output_logger.info('\n[*] Hosts found: ' + str(len(full))) + output_logger.info('---------------------') for host in full: - print(host) + output_logger.info(host) try: if ':' in host: _, addr = host.split(':', 1) await db.store(word, addr, 'ip', 'DNS-resolver') except (OSError, RuntimeError, ValueError, TypeError) as e: - print(f'An exception has occurred while attempting to insert: {host} IP into DB: {e}') + output_logger.info(f'An exception has occurred while attempting to insert: {host} IP into DB: {e}') continue else: all_hosts = [host.replace('www.', '') for host in all_hosts if host.replace('www.', '') in all_hosts] all_hosts = list(sorted(set(all_hosts))) - print('\n[*] Hosts found: ' + str(len(all_hosts))) - print('---------------------') + output_logger.info('\n[*] Hosts found: ' + str(len(all_hosts))) + output_logger.info('---------------------') for host in all_hosts: - print(host) + output_logger.info(host) # DNS brute force if dnsbrute and dnsbrute[0] is True: - print('\n[*] Starting DNS brute force.') + output_logger.info('\n[*] Starting DNS brute force.') dns_force = dnssearch.DnsForce(word, final_dns_resolver_list, verbose=True) resolved_pair, hosts, ips = await dns_force.run() # Check if Rest API is being used if so return found hosts @@ -1566,32 +1568,31 @@ async def start(rest_args: argparse.Namespace | None = None): temp.add(host) if host not in all_hosts: all_hosts.append(host) - print('\n[*] Hosts found after DNS brute force:') + output_logger.info('\n[*] Hosts found after DNS brute force:') for sub in temp: - print(sub) + output_logger.info(sub) await db.store_all(word, list(sorted(temp)), 'host', 'dns_bruteforce') takeover_results = dict() # TakeOver Checking if takeover_status: - print('\n[*] Performing subdomain takeover check') - print('\n[*] Subdomain Takeover checking IS ACTIVE RECON') + output_logger.info('\n[*] Performing subdomain takeover check') + output_logger.info('\n[*] Subdomain Takeover checking IS ACTIVE RECON') search_take = takeover.TakeOver(all_hosts) await search_take.populate_fingerprints() await search_take.process(proxy=use_proxy) takeover_results = await search_take.get_takeover_results() # DNS reverse lookup dnsrev: list = [] - # print(f'DNSlookup: {dnslookup}') if dnslookup is True: - print('\n[*] Starting active queries for DNSLookup.') + output_logger.info('\n[*] Starting active queries for DNSLookup.') # reverse each iprange in a separate task __reverse_dns_tasks: dict = {} for entry in host_ip: __ip_range = dnssearch.serialize_ip_range(ip=entry, netmask='24') if __ip_range and __ip_range not in set(__reverse_dns_tasks.keys()): - print('\n[*] Performing reverse lookup on ' + __ip_range) + output_logger.info('\n[*] Performing reverse lookup on ' + __ip_range) __reverse_dns_tasks[__ip_range] = asyncio.create_task( dnssearch.reverse_all_ips_in_range( iprange=__ip_range, @@ -1605,10 +1606,10 @@ async def start(rest_args: argparse.Namespace | None = None): # run all the reversing tasks concurrently await asyncio.gather(*__reverse_dns_tasks.values()) - print('\n[*] Hosts found after reverse lookup (in target domain):') - print('--------------------------------------------------------') + output_logger.info('\n[*] Hosts found after reverse lookup (in target domain):') + output_logger.info('--------------------------------------------------------') for xh in dnsrev: - print(xh) + output_logger.info(xh) # Screenshots screenshot_tups = [] @@ -1618,56 +1619,56 @@ async def start(rest_args: argparse.Namespace | None = None): # Verify the path exists, if not create it or if user does not create it skips screenshot if path_exists: await screen_shotter.verify_installation() - print(f'\nScreenshots can be found in: {screen_shotter.output}{screen_shotter.slash}') + output_logger.info(f'\nScreenshots can be found in: {screen_shotter.output}{screen_shotter.slash}') start_time = time.perf_counter() - print('Filtering domains for ones we can reach') + output_logger.info('Filtering domains for ones we can reach') if dnsresolve is None or len(final_dns_resolver_list) > 0: unique_resolved_domains = {url.split(':')[0] for url in full if ':' in url and 'www.' not in url} else: # Technically not resolved in this case, which is not ideal # You should always use dns resolve when doing screenshotting - print('NOTE for future use cases you should only use screenshotting in tandem with DNS resolving') + output_logger.info('NOTE for future use cases you should only use screenshotting in tandem with DNS resolving') unique_resolved_domains = set(all_hosts) if len(unique_resolved_domains) > 0: # First filter out ones that didn't resolve - print('Attempting to visit unique resolved domains, this is ACTIVE RECON') + output_logger.info('Attempting to visit unique resolved domains, this is ACTIVE RECON') async with Pool(10) as pool: results = await pool.map(screen_shotter.visit, list(unique_resolved_domains)) # Filter out domains that we couldn't connect to unique_resolved_domains_list = list(sorted({tup[0] for tup in results if len(tup[1]) > 0})) async with Pool(3) as pool: - print(f'Length of unique resolved domains: {len(unique_resolved_domains_list)} chunking now!\n') + output_logger.info(f'Length of unique resolved domains: {len(unique_resolved_domains_list)} chunking now!\n') # If you have the resources, you could make the function faster by increasing the chunk number chunk_number = 14 for chunk in screen_shotter.chunk_list(unique_resolved_domains_list, chunk_number): try: screenshot_tups.extend(await pool.map(screen_shotter.take_screenshot, chunk)) except Exception as ee: - print(f'An exception has occurred while mapping: {ee}') + output_logger.info(f'An exception has occurred while mapping: {ee}') end = time.perf_counter() # There is probably an easier way to do this total = int(end - start_time) mon, sec = divmod(total, 60) hr, mon = divmod(mon, 60) total_time = f'{mon:02d}:{sec:02d}' - print(f'Finished taking screenshots in {total_time} seconds') - print('[+] Note there may be leftover chrome processes you may have to kill manually\n') + output_logger.info(f'Finished taking screenshots in {total_time} seconds') + output_logger.info('[+] Note there may be leftover chrome processes you may have to kill manually\n') # Shodan shodanres = [] if shodan is True: - print('[*] Searching Shodan. ') + output_logger.info('[*] Searching Shodan. ') try: for ip in host_ip: try: - print('\tSearching for ' + ip) + output_logger.info('\tSearching for ' + ip) shodan_search = shodansearch.SearchShodan() shodandict = await shodan_search.search_ip(ip) await asyncio.sleep(5) # Check if the result is a string (error message) if isinstance(shodandict[ip], str): - print(f'{ip}: {shodandict[ip]}') + output_logger.info(f'{ip}: {shodandict[ip]}') continue # Process the results if it's a dictionary @@ -1680,18 +1681,18 @@ async def start(rest_args: argparse.Namespace | None = None): value = ', '.join(map(str, value)) rowdata.append(value) shodanres.append(rowdata) - print(ujson.dumps(shodandict[ip], indent=4, sort_keys=True)) - print('\n') + output_logger.info(ujson.dumps(shodandict[ip], indent=4, sort_keys=True)) + output_logger.info('\n') except Exception as ip_error: - print(f'[SHODAN-error] Error searching {ip}: {ip_error}') + output_logger.info(f'[SHODAN-error] Error searching {ip}: {ip_error}') continue except Exception as e: - print(f'[!] An error occurred with Shodan: {e} ') + output_logger.info(f'[!] An error occurred with Shodan: {e} ') else: pass if filename != '': - print('\n[*] Reporting started.') + output_logger.info('\n[*] Reporting started.') try: if len(rest_filename) == 0: filename = filename.rsplit('.', 1)[0] + '.xml' @@ -1722,9 +1723,9 @@ async def start(rest_args: argparse.Namespace | None = None): await file.write(f'{sanitize_for_xml(host)}') # TODO add Shodan output into XML report await file.write('') - print('[*] XML File saved.') + output_logger.info('[*] XML File saved.') except (OSError, ValueError, TypeError, UnicodeEncodeError) as error: - print(f'[!] An error occurred while saving the XML file: {error}') + output_logger.info(f'[!] An error occurred while saving the XML file: {error}') try: # JSON REPORT SECTION @@ -1780,10 +1781,10 @@ async def start(rest_args: argparse.Namespace | None = None): async with await anyio.open_file(filename, 'w+') as fp: dumped_json = ujson.dumps(json_dict, sort_keys=True) await fp.write(dumped_json) - print('[*] JSON File saved.') + output_logger.info('[*] JSON File saved.') except (OSError, ValueError, TypeError, UnicodeEncodeError) as er: - print(f'[!] An error occurred while saving the JSON file: {er} ') - print('\n\n') + output_logger.info(f'[!] An error occurred while saving the JSON file: {er} ') + output_logger.info('\n\n') # Enhanced code block for API Endpoint scanning feature if args.api_scan or 'api_endpoints' in engines: @@ -1792,8 +1793,8 @@ async def start(rest_args: argparse.Namespace | None = None): wordlist = args.wordlist or str(DATA_DIR / 'wordlists' / 'api_endpoints.txt') if not await anyio.Path(wordlist).exists(): - print(f'\n[!] Wordlist not found: {wordlist}') - print('Creating a basic API wordlist for scanning...') + output_logger.info(f'\n[!] Wordlist not found: {wordlist}') + output_logger.info('Creating a basic API wordlist for scanning...') # Create a default simple API endpoint list basic_endpoints = [ '/api', @@ -1820,43 +1821,43 @@ async def start(rest_args: argparse.Namespace | None = None): async with await anyio.open_file(temp_wordlist, 'w') as f: await f.write('\n'.join(basic_endpoints)) wordlist = temp_wordlist - print(f'Basic API wordlist created with {len(basic_endpoints)} endpoints.') + output_logger.info(f'Basic API wordlist created with {len(basic_endpoints)} endpoints.') - print(f'\n[*] Starting API endpoint scanning with wordlist: {wordlist}') + output_logger.info(f'\n[*] Starting API endpoint scanning with wordlist: {wordlist}') api_scanner = api_endpoints.SearchApiEndpoints(word=args.domain, wordlist=wordlist) await api_scanner.do_search() # Print results endpoints_found = api_scanner.get_found_endpoints() - print(f'\n[*] API Endpoints found: {len(endpoints_found)}') + output_logger.info(f'\n[*] API Endpoints found: {len(endpoints_found)}') for endpoint in endpoints_found: - print(f' - {endpoint}') + output_logger.info(f' - {endpoint}') interesting_endpoints = api_scanner.get_interesting_endpoints() - print(f'\n[*] Interesting endpoints (200, 201, 202): {len(interesting_endpoints)}') + output_logger.info(f'\n[*] Interesting endpoints (200, 201, 202): {len(interesting_endpoints)}') for endpoint in interesting_endpoints: - print(f' - {endpoint}') + output_logger.info(f' - {endpoint}') auth_required = api_scanner.get_auth_required() - print(f'\n[*] Endpoints requiring authentication: {len(auth_required)}') + output_logger.info(f'\n[*] Endpoints requiring authentication: {len(auth_required)}') for endpoint in auth_required: - print(f' - {endpoint}') + output_logger.info(f' - {endpoint}') api_versions = api_scanner.get_api_versions() - print(f'\n[*] Detected API versions: {len(api_versions)}') + output_logger.info(f'\n[*] Detected API versions: {len(api_versions)}') for version in api_versions: - print(f' - {version}') + output_logger.info(f' - {version}') rate_limits = api_scanner.get_rate_limits() - print(f'\n[*] Rate limited endpoints: {len(rate_limits)}') + output_logger.info(f'\n[*] Rate limited endpoints: {len(rate_limits)}') for endpoint, info in rate_limits.items(): - print(f' - {endpoint} ({info.method})') + output_logger.info(f' - {endpoint} ({info.method})') methods = api_scanner.get_methods() - print(f'\n[*] HTTP methods used: {", ".join(methods)}') + output_logger.info(f'\n[*] HTTP methods used: {", ".join(methods)}') status_codes = api_scanner.get_status_codes() - print(f'\n[*] HTTP status codes encountered: {", ".join(map(str, status_codes))}') + output_logger.info(f'\n[*] HTTP status codes encountered: {", ".join(map(str, status_codes))}') # Add results to storage db = stash.StashManager() @@ -1868,7 +1869,7 @@ async def start(rest_args: argparse.Namespace | None = None): db_storage = stash.StashManager() await db_storage.store_all(word, endpoints_found, 'api_endpoint', 'api_scan') except AttributeError: - print('\n[*] No custom database functions found') + output_logger.info('\n[*] No custom database functions found') # Add to interesting URLs if any endpoints were found if interesting_endpoints: @@ -1878,70 +1879,70 @@ async def start(rest_args: argparse.Namespace | None = None): # Also add complete domain paths to the interesting_urls list all_urls.extend(new_urls) - print('\n[+] API scanning completed successfully.') + output_logger.info('\n[+] API scanning completed successfully.') except MissingKey: - print('\n[!] API endpoint scanning requires a wordlist. Use -w to specify a wordlist file.') - print(' Creating a basic wordlist and trying again...') + output_logger.info('\n[!] API endpoint scanning requires a wordlist. Use -w to specify a wordlist file.') + output_logger.info(' Creating a basic wordlist and trying again...') # The wordlist creation code above could be used here except Exception as e: - print(f'\n[!] An exception has occurred in API Endpoints scanning: {e}') - print(' Continuing with the rest of the scan...') + output_logger.info(f'\n[!] An exception has occurred in API Endpoints scanning: {e}') + output_logger.info(' Continuing with the rest of the scan...') traceback.print_exc() # More detailed error information for developers if 'securityscorecard' in engines: try: - print('\n[*] Performing SecurityScorecard scan...') + output_logger.info('\n[*] Performing SecurityScorecard scan...') securityscorecard_scanner = securityscorecard.SearchSecurityScorecard(word) await securityscorecard_scanner.process(use_proxy) # Use the existing API to get results hosts = await securityscorecard_scanner.get_hostnames() if hosts: - print(f'\n[*] SecurityScorecard results: {len(hosts)} hosts found') + output_logger.info(f'\n[*] SecurityScorecard results: {len(hosts)} hosts found') for host in hosts: - print(f' - {host}') + output_logger.info(f' - {host}') all_hosts.extend(hosts) ips = await securityscorecard_scanner.get_ips() if ips: - print(f'\n[*] SecurityScorecard IPs found: {len(ips)}') + output_logger.info(f'\n[*] SecurityScorecard IPs found: {len(ips)}') for ip in ips: - print(f' - {ip}') + output_logger.info(f' - {ip}') all_ip.extend(ips) except Exception as e: - print(f'An exception has occurred in SecurityScorecard scanning: {e}') + output_logger.info(f'An exception has occurred in SecurityScorecard scanning: {e}') if 'builtwith' in engines: try: - print('\n[*] Performing BuiltWith scan...') + output_logger.info('\n[*] Performing BuiltWith scan...') builtwith_scanner = builtwith.SearchBuiltWith(word) await builtwith_scanner.process(use_proxy) hosts = await builtwith_scanner.get_hostnames() if hosts: - print(f'\n[*] BuiltWith results: {len(hosts)} hosts found') + output_logger.info(f'\n[*] BuiltWith results: {len(hosts)} hosts found') for host in hosts: - print(f' - {host}') + output_logger.info(f' - {host}') # Add results to the main host list all_hosts.extend(hosts) urls = list(await builtwith_scanner.get_interesting_urls()) if urls: - print(f'\n[*] BuiltWith interesting URLs found: {len(urls)}') + output_logger.info(f'\n[*] BuiltWith interesting URLs found: {len(urls)}') for url in urls: - print(f' - {url}') + output_logger.info(f' - {url}') interesting_urls.extend(urls) except Exception as e: if isinstance(e, MissingKey): if not args.quiet: - print(MissingKey('BuiltWith')) + output_logger.info(MissingKey('BuiltWith')) else: - print(f'An exception has occurred in BuiltWith scanning: {e}') + output_logger.info(f'An exception has occurred in BuiltWith scanning: {e}') if rest_args is not None: all_hosts = sorted({host.replace('www.', '') for host in all_hosts}) @@ -1961,10 +1962,11 @@ async def start(rest_args: argparse.Namespace | None = None): async def entry_point() -> None: try: + configure_logging(verbose=False) Core.banner() await start() except KeyboardInterrupt: - print('\n\n[!] ctrl+c detected from user, quitting.\n\n ') + output_logger.info('\n\n[!] ctrl+c detected from user, quitting.\n\n ') except Exception as error_entry_point: - print(error_entry_point) + output_logger.info(error_entry_point) sys.exit(1) diff --git a/theHarvester/discovery/additional_apis.py b/theHarvester/discovery/additional_apis.py index af301640..e61e6ede 100644 --- a/theHarvester/discovery/additional_apis.py +++ b/theHarvester/discovery/additional_apis.py @@ -6,6 +6,7 @@ from theHarvester.discovery.haveibeenpwned import SearchHaveIBeenPwned from theHarvester.discovery.leaklookup import SearchLeakLookup from theHarvester.discovery.securityscorecard import SearchSecurityScorecard from theHarvester.discovery.shodansearch import SearchShodan +from theHarvester.lib.output import output_logger class AdditionalAPIs: @@ -65,7 +66,7 @@ class AdditionalAPIs: self.hosts.update(self.haveibeenpwned.hosts) self.emails.update(self.haveibeenpwned.emails) except Exception as e: - print(f'Error processing HaveIBeenPwned: {e}') + output_logger.info(f'Error processing HaveIBeenPwned: {e}') async def _process_leaklookup(self, proxy: bool = False): """Process Leak-Lookup API.""" @@ -75,7 +76,7 @@ class AdditionalAPIs: self.hosts.update(self.leaklookup.hosts) self.emails.update(self.leaklookup.emails) except Exception as e: - print(f'Error processing Leak-Lookup: {e}') + output_logger.info(f'Error processing Leak-Lookup: {e}') async def _process_securityscorecard(self, proxy: bool = False): """Process SecurityScorecard API.""" @@ -89,7 +90,7 @@ class AdditionalAPIs: } self.hosts.update(self.securityscorecard.hosts) except Exception as e: - print(f'Error processing SecurityScorecard: {e}') + output_logger.info(f'Error processing SecurityScorecard: {e}') async def _process_builtwith(self, proxy: bool = False): """Process BuiltWith API.""" @@ -105,7 +106,7 @@ class AdditionalAPIs: } self.hosts.update(self.builtwith.hosts) except Exception as e: - print(f'Error processing BuiltWith: {e}') + output_logger.info(f'Error processing BuiltWith: {e}') async def _process_shodan(self, proxy: bool = False): """Process Shodan API for IP information.""" @@ -124,9 +125,9 @@ class AdditionalAPIs: ip = socket.gethostbyname(self.domain) ips_to_search.add(ip) except socket.gaierror as e: - print(f"Failed to resolve domain '{self.domain}': {e}") + output_logger.info(f"Failed to resolve domain '{self.domain}': {e}") except Exception as e: - print(f"Unexpected error while resolving domain '{self.domain}': {e}") + output_logger.info(f"Unexpected error while resolving domain '{self.domain}': {e}") # Add any IPs from other results for host in self.hosts: @@ -141,21 +142,21 @@ class AdditionalAPIs: # Search each IP in Shodan for ip in ips_to_search: try: - print(f'\tSearching Shodan for {ip}') + output_logger.info(f'\tSearching Shodan for {ip}') shodan_result = await self.shodan.search_ip(ip) if ip in shodan_result and isinstance(shodan_result[ip], dict): self.shodan_data[ip] = shodan_result[ip] elif ip in shodan_result and isinstance(shodan_result[ip], str): - print(f'{ip}: {shodan_result[ip]}') + output_logger.info(f'{ip}: {shodan_result[ip]}') await asyncio.sleep(2) # Rate limiting except Exception as ip_error: - print(f'Error searching Shodan for {ip}: {ip_error}') + output_logger.info(f'Error searching Shodan for {ip}: {ip_error}') continue except Exception as e: - print(f'Error processing Shodan: {e}') + output_logger.info(f'Error processing Shodan: {e}') @staticmethod def _is_valid_ip(ip_str: str) -> bool: diff --git a/theHarvester/discovery/bitbucket.py b/theHarvester/discovery/bitbucket.py index ae9afb74..59dbaa6b 100644 --- a/theHarvester/discovery/bitbucket.py +++ b/theHarvester/discovery/bitbucket.py @@ -8,6 +8,7 @@ import aiohttp from theHarvester.discovery.constants import MissingKey, get_delay from theHarvester.lib.core import Core +from theHarvester.lib.output import output_logger from theHarvester.parsers import myparser @@ -58,7 +59,7 @@ class SearchBitBucket: if match.get('fragment') is not None ] except (AttributeError, TypeError, ValueError) as e: - print(f'Error extracting fragments: {e}') + output_logger.info(f'Error extracting fragments: {e}') return [] @staticmethod @@ -70,7 +71,7 @@ class SearchBitBucket: return int(page_param) return 0 except (AttributeError, TypeError, ValueError) as e: - print(f'Error parsing page response: {e}') + output_logger.info(f'Error parsing page response: {e}') return None async def handle_response(self, response: tuple[str, dict, int, Any]) -> ErrorResult | RetryResult | SuccessResult: @@ -86,7 +87,7 @@ class SearchBitBucket: return RetryResult(60) return ErrorResult(status, json_data if isinstance(json_data, dict) else text) except (TypeError, ValueError, KeyError, AttributeError) as e: - print(f'Error handling response: {e}') + output_logger.info(f'Error handling response: {e}') return ErrorResult(500, str(e)) @staticmethod @@ -103,7 +104,7 @@ class SearchBitBucket: async with sess.get(url, proxy=random.choice(Core.proxy_list()) if self.proxy else None) as resp: return await resp.text(), await resp.json(), resp.status, resp.links except (aiohttp.ClientError, TimeoutError, ValueError, OSError) as e: - print(f'Error performing search: {e}') + output_logger.info(f'Error performing search: {e}') return '', {}, 500, {} async def process(self, proxy: bool = False) -> None: @@ -117,13 +118,13 @@ class SearchBitBucket: if isinstance(result, SuccessResult): # Reset retry counter on any successful response self.retry_count = 0 - print(f'\tSearching {self.counter} results.') + output_logger.info(f'\tSearching {self.counter} results.') self.total_results += ''.join(result.fragments) self.counter += len(result.fragments) next_or_last = result.next_page or result.last_page # Break if pagination does not advance to avoid infinite loop if next_or_last == self.page: - print('\tNo page advancement detected; exiting to avoid infinite loop.') + output_logger.info('\tNo page advancement detected; exiting to avoid infinite loop.') self.page = 0 break self.page = next_or_last @@ -131,29 +132,29 @@ class SearchBitBucket: elif isinstance(result, RetryResult): self.retry_count += 1 if self.retry_count > self.max_retries: - print('\tMaximum retries reached; exiting to avoid infinite loop.') + output_logger.info('\tMaximum retries reached; exiting to avoid infinite loop.') self.page = 0 break sleepy_time = get_delay() + result.time - print(f'\tRetrying page in {sleepy_time} seconds...') + output_logger.info(f'\tRetrying page in {sleepy_time} seconds...') await asyncio.sleep(sleepy_time) else: # On error, stop to avoid endless retries on a bad state - print(f'\tException occurred: status_code: {result.status_code} reason: {result.body}') + output_logger.info(f'\tException occurred: status_code: {result.status_code} reason: {result.body}') self.page = 0 break except (aiohttp.ClientError, TimeoutError, ValueError, TypeError, AttributeError) as e: - print(f'Error processing page: {e}') + output_logger.info(f'Error processing page: {e}') await asyncio.sleep(get_delay()) except (aiohttp.ClientError, TimeoutError, ValueError, TypeError, AttributeError) as e: - print(f'An exception has occurred in bitbucket process: {e}') + output_logger.info(f'An exception has occurred in bitbucket process: {e}') async def get_emails(self): try: rawres = myparser.Parser(self.total_results, self.word) return await rawres.emails() except (AttributeError, TypeError, re.error) as e: - print(f'Error getting emails: {e}') + output_logger.info(f'Error getting emails: {e}') return [] async def get_hostnames(self): @@ -161,5 +162,5 @@ class SearchBitBucket: rawres = myparser.Parser(self.total_results, self.word) return await rawres.hostnames() except (AttributeError, TypeError, re.error) as e: - print(f'Error getting hostnames: {e}') + output_logger.info(f'Error getting hostnames: {e}') return [] diff --git a/theHarvester/discovery/bravesearch.py b/theHarvester/discovery/bravesearch.py index b72ba1ca..666c2cc8 100644 --- a/theHarvester/discovery/bravesearch.py +++ b/theHarvester/discovery/bravesearch.py @@ -3,6 +3,7 @@ from urllib.parse import quote from theHarvester.discovery.constants import MissingKey, get_delay from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger from theHarvester.parsers import myparser @@ -50,7 +51,7 @@ class SearchBrave: # Handle API response if resp is None: - print('No response received from Brave Search API') + output_logger.info('No response received from Brave Search API') break # Check for API errors (rate limit, quota exceeded, etc.) @@ -59,15 +60,14 @@ class SearchBrave: error_code = resp.get('error', {}).get('code', 'unknown') if 'rate limit' in error_msg.lower() or error_code == 'rate_limit_exceeded': - print(f'Rate limit exceeded. Increasing delay to {self.rate_limit_delay * 2} seconds') + output_logger.info(f'Rate limit exceeded. Increasing delay to {self.rate_limit_delay * 2} seconds') self.rate_limit_delay *= 2 await asyncio.sleep(self.rate_limit_delay) continue elif 'quota' in error_msg.lower() or error_code == 'quota_exceeded': - print(f'API quota exceeded: {error_msg}') + output_logger.info(f'API quota exceeded: {error_msg}') break else: - # print(f'API error ({error_code}): {error_msg}') break if 'web' in resp and 'results' in resp['web']: @@ -93,7 +93,7 @@ class SearchBrave: if len(self.results) >= self.limit: break else: - print('Unexpected response format from Brave Search API') + output_logger.info('Unexpected response format from Brave Search API') break await asyncio.sleep(get_delay()) @@ -103,17 +103,19 @@ class SearchBrave: # Handle specific API-related exceptions if 'rate limit' in error_msg or '429' in error_msg: - print(f'Rate limit detected in exception. Increasing delay to {self.rate_limit_delay * 2} seconds') + output_logger.info( + f'Rate limit detected in exception. Increasing delay to {self.rate_limit_delay * 2} seconds' + ) self.rate_limit_delay *= 2 await asyncio.sleep(self.rate_limit_delay) elif 'quota' in error_msg or '403' in error_msg: - print(f'Quota exceeded or access denied: {e}') + output_logger.info(f'Quota exceeded or access denied: {e}') break elif 'timeout' in error_msg: - print(f'Request timeout occurred: {e}') + output_logger.info(f'Request timeout occurred: {e}') await asyncio.sleep(get_delay() + 2) else: - print(f'An exception has occurred in bravesearch: {e}') + output_logger.info(f'An exception has occurred in bravesearch: {e}') await asyncio.sleep(get_delay() + 5) continue diff --git a/theHarvester/discovery/builtwith.py b/theHarvester/discovery/builtwith.py index 669e3b6e..309c4b08 100644 --- a/theHarvester/discovery/builtwith.py +++ b/theHarvester/discovery/builtwith.py @@ -4,6 +4,7 @@ import aiohttp from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchBuiltWith: @@ -41,7 +42,7 @@ class SearchBuiltWith: self.tech_stack = data self._extract_data() except Exception as e: - print(f'Error in BuiltWith search: {e}') + output_logger.info(f'Error in BuiltWith search: {e}') def _extract_data(self) -> None: """Extract and categorize technology information.""" diff --git a/theHarvester/discovery/censysearch.py b/theHarvester/discovery/censysearch.py index c5de1c50..913a328f 100644 --- a/theHarvester/discovery/censysearch.py +++ b/theHarvester/discovery/censysearch.py @@ -10,6 +10,7 @@ from censys.search import CensysCerts from theHarvester import __version__ as thehavester_version from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import Core +from theHarvester.lib.output import output_logger class SearchCensys: @@ -64,7 +65,7 @@ class SearchCensys: self.emails.update(self._normalize_emails(email_address)) records_seen += 1 except CensysRateLimitExceededException: - print('Censys rate limit exceeded') + output_logger.info('Censys rate limit exceeded') async def get_hostnames(self) -> set: return self.totalhosts diff --git a/theHarvester/discovery/certspottersearch.py b/theHarvester/discovery/certspottersearch.py index f9fd1048..cccb803a 100644 --- a/theHarvester/discovery/certspottersearch.py +++ b/theHarvester/discovery/certspottersearch.py @@ -1,4 +1,5 @@ from theHarvester.lib.core import AsyncFetcher +from theHarvester.lib.output import output_logger class SearchCertspoter: @@ -26,11 +27,11 @@ class SearchCertspoter: else: self.totalhosts.update({''}) except IndexError: - print('No data returned from Cert Spotter.') + output_logger.info('No data returned from Cert Spotter.') except ConnectionError: - print('Network connection failed.') + output_logger.info('Network connection failed.') except Exception as e: - print(f'Unexpected error occurred: {e}') + output_logger.info(f'Unexpected error occurred: {e}') async def get_hostnames(self) -> set: return self.totalhosts @@ -38,4 +39,4 @@ class SearchCertspoter: async def process(self, proxy: bool = False) -> None: self.proxy = proxy await self.do_search() - print('\tSearching results.') + output_logger.info('\tSearching results.') diff --git a/theHarvester/discovery/chaos.py b/theHarvester/discovery/chaos.py index 3510e40b..5e1e83db 100644 --- a/theHarvester/discovery/chaos.py +++ b/theHarvester/discovery/chaos.py @@ -3,6 +3,7 @@ from types import ModuleType from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger json: ModuleType = _stdlib_json try: @@ -54,7 +55,7 @@ class SearchChaos: response = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy) if not response or not isinstance(response, list) or not response[0]: - print(f'No response from Chaos API for: {url}') + output_logger.info(f'No response from Chaos API for: {url}') return try: @@ -64,7 +65,7 @@ class SearchChaos: # Check for error messages if 'error' in data: error_msg = data.get('message', data.get('error', 'Unknown error')) - print(f'Chaos API error: {error_msg}') + output_logger.info(f'Chaos API error: {error_msg}') if 'unauthorized' in error_msg.lower(): raise MissingKey('Chaos (ProjectDiscovery)') return @@ -98,12 +99,12 @@ class SearchChaos: self.totalhosts.add(full_domain.lower()) except Exception as e: - print(f'Failed to parse Chaos response: {e}') + output_logger.info(f'Failed to parse Chaos response: {e}') except MissingKey: raise except Exception as e: - print(f'Chaos API error: {e}') + output_logger.info(f'Chaos API error: {e}') async def get_hostnames(self) -> set: return self.totalhosts diff --git a/theHarvester/discovery/commoncrawl.py b/theHarvester/discovery/commoncrawl.py index 4fae8f39..de498a8e 100644 --- a/theHarvester/discovery/commoncrawl.py +++ b/theHarvester/discovery/commoncrawl.py @@ -3,6 +3,7 @@ import re from types import ModuleType from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger json: ModuleType = _stdlib_json try: @@ -79,7 +80,7 @@ class SearchCommoncrawl: try: data = self._safe_parse_json_lines(response[0]) except Exception as e: - print(f'Failed to parse Common Crawl response for {index}: {e}') + output_logger.info(f'Failed to parse Common Crawl response for {index}: {e}') continue # Extract domains from URLs @@ -109,14 +110,14 @@ class SearchCommoncrawl: if domain.endswith(f'.{self.word}') or domain == self.word: self.totalhosts.add(domain) except Exception as e: - print(f'Failed to parse Common Crawl main domain response for {index}: {e}') + output_logger.info(f'Failed to parse Common Crawl main domain response for {index}: {e}') except Exception as e: - print(f'Common Crawl API error for index {index}: {e}') + output_logger.info(f'Common Crawl API error for index {index}: {e}') continue except Exception as e: - print(f'Common Crawl API error: {e}') + output_logger.info(f'Common Crawl API error: {e}') async def get_hostnames(self) -> set: return self.totalhosts diff --git a/theHarvester/discovery/constants.py b/theHarvester/discovery/constants.py index ebade1d7..ae6b95d4 100644 --- a/theHarvester/discovery/constants.py +++ b/theHarvester/discovery/constants.py @@ -61,7 +61,6 @@ async def search(text: str) -> bool: or 'http://www.google.com/sorry/index' in line or 'https://www.google.com/sorry/index' in line ): - # print('\tGoogle is blocking your IP due to too many automated requests, wait or change your IP') return True return False diff --git a/theHarvester/discovery/criminalip.py b/theHarvester/discovery/criminalip.py index 4c41d725..f4571705 100644 --- a/theHarvester/discovery/criminalip.py +++ b/theHarvester/discovery/criminalip.py @@ -4,6 +4,7 @@ from urllib.parse import urlparse from theHarvester.discovery.constants import MissingKey, get_delay from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchCriminalIP: @@ -88,7 +89,6 @@ class SearchCriminalIP: # https://www.criminalip.io/developer/api/get-v2-domain-report-id url = 'https://api.criminalip.io/v1/domain/scan' data = f'{{"query": "{self.word}"}}' - # print(f'Current key: {self.key}') user_agent = Core.get_user_agent() response = await AsyncFetcher.post_fetch( url, @@ -97,19 +97,18 @@ class SearchCriminalIP: data=data, proxy=self.proxy, ) - # print(f'My response: {response}') # Expected response format: # {'data': {'scan_id': scan_id}, 'message': 'api success', 'status': 200} if not isinstance(response, dict): - print(f'An error has occurred searching criminalip dumping response: {response}') + output_logger.info(f'An error has occurred searching criminalip dumping response: {response}') return if response.get('status') != 200: - print(f'An error has occurred searching criminalip dumping response: {response}') + output_logger.info(f'An error has occurred searching criminalip dumping response: {response}') return scan_id = response.get('data', {}).get('scan_id') if scan_id is None: - print(f'CriminalIP did not return a scan_id, dumping response: {response}') + output_logger.info(f'CriminalIP did not return a scan_id, dumping response: {response}') return scan_percentage = 0 @@ -125,26 +124,26 @@ class SearchCriminalIP: ) status = status_response[0] if isinstance(status_response, list) and len(status_response) > 0 else {} if not isinstance(status, dict): - print(f'CriminalIP status response is malformed dumping data: {status_response}') + output_logger.info(f'CriminalIP status response is malformed dumping data: {status_response}') return if status.get('status') != 200: - print(f'CriminalIP status check failed dumping data: status_response: {status}') + output_logger.info(f'CriminalIP status check failed dumping data: status_response: {status}') return # Expected format: # {"data": {"scan_percentage": 100}, "message": "api success", "status": 200} scan_percentage = status.get('data', {}).get('scan_percentage') if scan_percentage is None: - print(f'CriminalIP status did not include scan_percentage dumping data: {status}') + output_logger.info(f'CriminalIP status did not include scan_percentage dumping data: {status}') return if scan_percentage == 100: break if scan_percentage == -2: - print(f'CriminalIP failed to scan: {self.word} does not exist, verify manually') - print(f'Dumping data: scan_response: {response} status_response: {status}') + output_logger.info(f'CriminalIP failed to scan: {self.word} does not exist, verify manually') + output_logger.info(f'Dumping data: scan_response: {response} status_response: {status}') return if scan_percentage == -1: - print(f'CriminalIP scan failed dumping data: scan_response: {response} status_response: {status}') + output_logger.info(f'CriminalIP scan failed dumping data: scan_response: {response} status_response: {status}') return # Wait for scan to finish if counter >= 5: @@ -153,10 +152,12 @@ class SearchCriminalIP: await asyncio.sleep(10 * get_delay()) counter += 1 if counter == 10: - print( + output_logger.info( 'Ten iterations have occurred in CriminalIP waiting for scan to finish, returning to prevent infinite loop.' ) - print(f'Verify results manually on CriminalIP dumping data: scan_response: {response} status_response: {status}') + output_logger.info( + f'Verify results manually on CriminalIP dumping data: scan_response: {response} status_response: {status}' + ) return report_url = f'https://api.criminalip.io/v2/domain/report/{scan_id}' @@ -168,32 +169,29 @@ class SearchCriminalIP: ) scan = scan_response[0] if isinstance(scan_response, list) and len(scan_response) > 0 else {} if not isinstance(scan, dict): - print(f'CriminalIP report response is malformed dumping data: {scan_response}') + output_logger.info(f'CriminalIP report response is malformed dumping data: {scan_response}') return if scan.get('status') != 200: - print(f'CriminalIP report request failed dumping data: {scan}') + output_logger.info(f'CriminalIP report request failed dumping data: {scan}') return - # json_formatted_str = json.dumps(scan, indent=2) - # print(json_formatted_str) try: await self.parser(scan) except Exception as e: - print(f'An exception occurred while parsing criminalip result: {e}') - print('Dumping json: ') - print(scan) + output_logger.info(f'An exception occurred while parsing criminalip result: {e}') + output_logger.info('Dumping json: ') + output_logger.info(scan) async def parser(self, jlines): # TODO when new scope field is added to parse lines for potential new scope! # TODO map as_name to asn for asn data # TODO determine if worth storing interesting urls if not isinstance(jlines, dict) or 'data' not in jlines.keys() or not isinstance(jlines['data'], dict): - print(f'Error with criminalip data, dumping: {jlines}') + output_logger.info(f'Error with criminalip data, dumping: {jlines}') return data = jlines['data'] for cert in data.get('certificates', []): - # print(f'Current cert: {cert}') if isinstance(cert, dict): self._add_host(cert.get('subject')) @@ -206,11 +204,10 @@ class SearchCriminalIP: self._add_host(main_domain) subdomains = [sub.get('domain') for sub in connected_domain.get('subdomains', []) if isinstance(sub, dict)] for sub in subdomains: - # print(f'Current sub: {sub}') self._add_host(sub) except Exception as e: - print(f'An exception has occurred: {e}') - print(f'Main line: {connected_domain}') + output_logger.info(f'An exception has occurred: {e}') + output_logger.info(f'Main line: {connected_domain}') for ip_info in data.get('connected_ip_info', []): if not isinstance(ip_info, dict): @@ -313,10 +310,6 @@ class SearchCriminalIP: self.totalhosts = {host.removeprefix('www.') for host in self.totalhosts if '*.' + self.word != host} - # print(f'hostnames: {self.totalhosts}') - # print(f'asns: {self.asns}') - # print(f'ips: {self.totalips}') - async def get_asns(self) -> set: return self.asns diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py index b8b0f0e6..1ad72e79 100644 --- a/theHarvester/discovery/crtsh.py +++ b/theHarvester/discovery/crtsh.py @@ -1,4 +1,5 @@ from theHarvester.lib.core import AsyncFetcher +from theHarvester.lib.output import output_logger class SearchCrtsh: @@ -16,11 +17,11 @@ class SearchCrtsh: data = set([(dct['name_value'][2:] if dct['name_value'][:2] == '*.' else dct['name_value']) for dct in response]) data = {domain for domain in data if (domain[0] != '*' and str(domain[0:4]).isnumeric() is False)} except IndexError: - print('No response from crt.sh or malformed list.') + output_logger.info('No response from crt.sh or malformed list.') except KeyError as ke: - print(f'Missing expected key in response: {ke}') + output_logger.info(f'Missing expected key in response: {ke}') except Exception as e: - print(f'Unexpected error: {e}') + output_logger.info(f'Unexpected error: {e}') clean: list = [] for x in data: pre = x.split() diff --git a/theHarvester/discovery/dnssearch.py b/theHarvester/discovery/dnssearch.py index 7789fd48..9a703577 100644 --- a/theHarvester/discovery/dnssearch.py +++ b/theHarvester/discovery/dnssearch.py @@ -15,6 +15,7 @@ from aiodns import DNSResolver from theHarvester.lib import hostchecker from theHarvester.lib.core import DATA_DIR +from theHarvester.lib.output import output_logger ##################################################################### # DNS FORCE @@ -37,7 +38,7 @@ class DnsForce: self.list = [f'{word.strip()}.{self.domain}' for word in self.list] async def run(self): - print(f'Starting DNS brute forcing with {len(self.list)} words') + output_logger.info(f'Starting DNS brute forcing with {len(self.list)} words') checker = hostchecker.Checker(self.list, nameservers=self.dnsserver) resolved_pair, hosts, ips = await checker.check() return resolved_pair, hosts, ips @@ -193,7 +194,7 @@ def log_result(host: str) -> None: """ if host: - print(host) + output_logger.info(host) def generate_postprocessing_callback(target: str, **allhosts: list[str]) -> Callable: diff --git a/theHarvester/discovery/duckduckgosearch.py b/theHarvester/discovery/duckduckgosearch.py index bbe205b1..a4bc58d9 100644 --- a/theHarvester/discovery/duckduckgosearch.py +++ b/theHarvester/discovery/duckduckgosearch.py @@ -1,6 +1,7 @@ import ujson from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger from theHarvester.parsers import myparser @@ -70,7 +71,7 @@ class SearchDuckDuckGo: tmp.add(url) return tmp except Exception as e: - print(f'Exception occurred: {e}') + output_logger.info(f'Exception occurred: {e}') return set() async def get_emails(self): diff --git a/theHarvester/discovery/fofa.py b/theHarvester/discovery/fofa.py index 9fde7d9f..c8f27912 100644 --- a/theHarvester/discovery/fofa.py +++ b/theHarvester/discovery/fofa.py @@ -4,6 +4,7 @@ from types import ModuleType from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger json: ModuleType = _stdlib_json try: @@ -72,7 +73,7 @@ class SearchFofa: response = await AsyncFetcher.fetch_all([full_url], headers=headers, proxy=self.proxy) if not response or not isinstance(response, list) or not response[0]: - print(f'No response from Fofa API for: {self.word}') + output_logger.info(f'No response from Fofa API for: {self.word}') return try: @@ -82,7 +83,7 @@ class SearchFofa: # Check for errors if data.get('error', False): error_msg = data.get('errmsg', 'Unknown error') - print(f'Fofa API error: {error_msg}') + output_logger.info(f'Fofa API error: {error_msg}') if '账号无效' in error_msg or 'invalid' in error_msg.lower(): raise MissingKey('Fofa API (Invalid credentials)') return @@ -107,12 +108,12 @@ class SearchFofa: self.totalips.add(ip) except Exception as e: - print(f'Failed to parse Fofa response: {e}') + output_logger.info(f'Failed to parse Fofa response: {e}') except MissingKey: raise except Exception as e: - print(f'Fofa API error: {e}') + output_logger.info(f'Fofa API error: {e}') async def get_hostnames(self) -> set: return self.totalhosts diff --git a/theHarvester/discovery/fullhuntsearch.py b/theHarvester/discovery/fullhuntsearch.py index 6b2b855e..f4c6dbc1 100644 --- a/theHarvester/discovery/fullhuntsearch.py +++ b/theHarvester/discovery/fullhuntsearch.py @@ -3,6 +3,7 @@ from urllib.parse import quote from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchFullHunt: @@ -391,7 +392,7 @@ class SearchFullHunt: await self.extract_data_from_search_results(search_results) except Exception as e: - print(f'Error during FullHunt search: {e}') + output_logger.info(f'Error during FullHunt search: {e}') async def get_hostnames(self) -> list[str]: """Return list of discovered subdomains""" diff --git a/theHarvester/discovery/githubcode.py b/theHarvester/discovery/githubcode.py index adbf487d..eb1f7903 100644 --- a/theHarvester/discovery/githubcode.py +++ b/theHarvester/discovery/githubcode.py @@ -7,6 +7,7 @@ import aiohttp from theHarvester.discovery.constants import MissingKey, get_delay from theHarvester.lib.core import Core +from theHarvester.lib.output import output_logger from theHarvester.parsers import myparser @@ -49,7 +50,7 @@ class SearchGithubCode: self.retry_count = 0 self.max_retries = 3 except Exception as e: - print(f'Error initializing SearchGithubCode: {e}') + output_logger.info(f'Error initializing SearchGithubCode: {e}') raise @staticmethod @@ -62,7 +63,7 @@ class SearchGithubCode: if match.get('fragment') is not None ] except Exception as e: - print(f'Error extracting fragments: {e}') + output_logger.info(f'Error extracting fragments: {e}') return [] @staticmethod @@ -74,7 +75,7 @@ class SearchGithubCode: return int(page_param) return 0 except Exception as e: - print(f'Error parsing page response: {e}') + output_logger.info(f'Error parsing page response: {e}') return None async def handle_response(self, response: tuple[str, dict, int, Any]) -> ErrorResult | RetryResult | SuccessResult: @@ -90,7 +91,7 @@ class SearchGithubCode: return RetryResult(60) return ErrorResult(status, json_data if isinstance(json_data, dict) else text) except Exception as e: - print(f'Error handling response: {e}') + output_logger.info(f'Error handling response: {e}') return ErrorResult(500, str(e)) @staticmethod @@ -107,7 +108,7 @@ class SearchGithubCode: async with sess.get(url, proxy=random.choice(Core.proxy_list()) if self.proxy else None) as resp: return await resp.text(), await resp.json(), resp.status, resp.links except Exception as e: - print(f'Error performing search: {e}') + output_logger.info(f'Error performing search: {e}') return '', {}, 500, {} async def process(self, proxy: bool = False) -> None: @@ -121,13 +122,13 @@ class SearchGithubCode: if isinstance(result, SuccessResult): # Reset retry counter on any successful response self.retry_count = 0 - print(f'\tSearching {self.counter} results.') + output_logger.info(f'\tSearching {self.counter} results.') self.total_results += ''.join(result.fragments) self.counter += len(result.fragments) next_or_last = result.next_page or result.last_page # Break if pagination does not advance to avoid infinite loop if next_or_last == self.page: - print('\tNo page advancement detected; exiting to avoid infinite loop.') + output_logger.info('\tNo page advancement detected; exiting to avoid infinite loop.') self.page = 0 break self.page = next_or_last @@ -135,29 +136,29 @@ class SearchGithubCode: elif isinstance(result, RetryResult): self.retry_count += 1 if self.retry_count > self.max_retries: - print('\tMaximum retries reached; exiting to avoid infinite loop.') + output_logger.info('\tMaximum retries reached; exiting to avoid infinite loop.') self.page = 0 break sleepy_time = get_delay() + result.time - print(f'\tRetrying page in {sleepy_time} seconds...') + output_logger.info(f'\tRetrying page in {sleepy_time} seconds...') await asyncio.sleep(sleepy_time) else: # On error, stop to avoid endless retries on a bad state - print(f'\tException occurred: status_code: {result.status_code} reason: {result.body}') + output_logger.info(f'\tException occurred: status_code: {result.status_code} reason: {result.body}') self.page = 0 break except Exception as e: - print(f'Error processing page: {e}') + output_logger.info(f'Error processing page: {e}') await asyncio.sleep(get_delay()) except Exception as e: - print(f'An exception has occurred in githubcode process: {e}') + output_logger.info(f'An exception has occurred in githubcode process: {e}') async def get_emails(self): try: rawres = myparser.Parser(self.total_results, self.word) return await rawres.emails() except Exception as e: - print(f'Error getting emails: {e}') + output_logger.info(f'Error getting emails: {e}') return [] async def get_hostnames(self): @@ -165,5 +166,5 @@ class SearchGithubCode: rawres = myparser.Parser(self.total_results, self.word) return await rawres.hostnames() except Exception as e: - print(f'Error getting hostnames: {e}') + output_logger.info(f'Error getting hostnames: {e}') return [] diff --git a/theHarvester/discovery/gitlabsearch.py b/theHarvester/discovery/gitlabsearch.py index 78904b96..d1fcda4c 100644 --- a/theHarvester/discovery/gitlabsearch.py +++ b/theHarvester/discovery/gitlabsearch.py @@ -3,6 +3,7 @@ import re from types import ModuleType from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger json: ModuleType = _stdlib_json try: @@ -128,10 +129,10 @@ class SearchGitlab: pass # README might not exist or be accessible except Exception as e: - print(f'Failed to parse GitLab projects response: {e}') + output_logger.info(f'Failed to parse GitLab projects response: {e}') except Exception as e: - print(f'GitLab API projects search error: {e}') + output_logger.info(f'GitLab API projects search error: {e}') async def search_users(self) -> None: """Search GitLab users for domain references""" @@ -179,10 +180,10 @@ class SearchGitlab: self.totalurls.add(web_url) except Exception as e: - print(f'Failed to parse GitLab users response: {e}') + output_logger.info(f'Failed to parse GitLab users response: {e}') except Exception as e: - print(f'GitLab API users search error: {e}') + output_logger.info(f'GitLab API users search error: {e}') async def do_search(self) -> None: await self.search_projects() diff --git a/theHarvester/discovery/haveibeenpwned.py b/theHarvester/discovery/haveibeenpwned.py index 6f775375..f8a96803 100644 --- a/theHarvester/discovery/haveibeenpwned.py +++ b/theHarvester/discovery/haveibeenpwned.py @@ -2,6 +2,7 @@ import aiohttp from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchHaveIBeenPwned: @@ -37,7 +38,7 @@ class SearchHaveIBeenPwned: self.breaches = await response.json() self._extract_data() except Exception as e: - print(f'Error in HaveIBeenPwned search: {e}') + output_logger.info(f'Error in HaveIBeenPwned search: {e}') def _extract_data(self) -> None: """Extract and categorize breach information.""" diff --git a/theHarvester/discovery/huntersearch.py b/theHarvester/discovery/huntersearch.py index 5a7f4bec..f2a04ff4 100644 --- a/theHarvester/discovery/huntersearch.py +++ b/theHarvester/discovery/huntersearch.py @@ -2,6 +2,7 @@ import asyncio from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchHunter: @@ -46,9 +47,13 @@ class SearchHunter: total_number_reqs = response[0]['data']['total'] // 100 # Parse out meta field within initial JSON response to determine the total number of results if total_requests_avail < total_number_reqs: - print('WARNING: account does not have enough requests to gather all emails') - print(f'Total requests available: {total_requests_avail}, total requests needed to be made: {total_number_reqs}') - print('RETURNING current results, if you would still like to run this module comment out the if request') + output_logger.info('WARNING: account does not have enough requests to gather all emails') + output_logger.info( + f'Total requests available: {total_requests_avail}, total requests needed to be made: {total_number_reqs}' + ) + output_logger.info( + 'RETURNING current results, if you would still like to run this module comment out the if request' + ) return self.limit = 100 # max number of emails you can get per request is 100 diff --git a/theHarvester/discovery/intelxsearch.py b/theHarvester/discovery/intelxsearch.py index 372f9170..1d9a9f48 100644 --- a/theHarvester/discovery/intelxsearch.py +++ b/theHarvester/discovery/intelxsearch.py @@ -6,6 +6,7 @@ import aiohttp from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import Core +from theHarvester.lib.output import output_logger from theHarvester.parsers import intelxparser @@ -46,7 +47,7 @@ class SearchIntelx: async with session.post(f'{self.database}/phonebook/search', headers=headers, json=data) as total_resp: search_data = await total_resp.json() if not search_data['success']: - print(f'Error: {search_data["message"]}') + output_logger.info(f'Error: {search_data["message"]}') return phonebook_id = search_data['id'] @@ -59,7 +60,7 @@ class SearchIntelx: self.results = await resp.json() except Exception as e: - print(f'An exception has occurred in Intelx: {e}') + output_logger.info(f'An exception has occurred in Intelx: {e}') async def process(self, proxy: bool = False): self.proxy = proxy diff --git a/theHarvester/discovery/leakix.py b/theHarvester/discovery/leakix.py index 5e8e4520..43015adf 100644 --- a/theHarvester/discovery/leakix.py +++ b/theHarvester/discovery/leakix.py @@ -2,6 +2,7 @@ import json as _stdlib_json from types import ModuleType from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger json: ModuleType = _stdlib_json try: @@ -69,7 +70,7 @@ class SearchLeakix: or 'unauthorized' in response[0].lower() or 'error' in response[0].lower() ): - print(f'LeakIX API requires authentication: {response[0][:100]}') + output_logger.info(f'LeakIX API requires authentication: {response[0][:100]}') continue try: @@ -95,14 +96,14 @@ class SearchLeakix: self.totalhosts.add(value.lower()) except Exception as e: - print(f'Failed to parse LeakIX response: {e}') + output_logger.info(f'Failed to parse LeakIX response: {e}') except Exception as e: - print(f'LeakIX API error for {query_url}: {e}') + output_logger.info(f'LeakIX API error for {query_url}: {e}') continue except Exception as e: - print(f'LeakIX API error: {e}') + output_logger.info(f'LeakIX API error: {e}') async def get_hostnames(self) -> set: return self.totalhosts diff --git a/theHarvester/discovery/leaklookup.py b/theHarvester/discovery/leaklookup.py index 4d2cd6e5..a1348dae 100644 --- a/theHarvester/discovery/leaklookup.py +++ b/theHarvester/discovery/leaklookup.py @@ -2,6 +2,7 @@ import aiohttp from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchLeakLookup: @@ -37,10 +38,10 @@ class SearchLeakLookup: self.leaks = await response.json() self._extract_data() elif response.status == 401: - print('[!] Missing API key for Leak-Lookup.') + output_logger.info('[!] Missing API key for Leak-Lookup.') raise MissingKey('Leak-Lookup') except Exception as e: - print(f'Error in Leak-Lookup search: {e}') + output_logger.info(f'Error in Leak-Lookup search: {e}') def _extract_data(self) -> None: """Extract and categorize leak information.""" diff --git a/theHarvester/discovery/mojeek.py b/theHarvester/discovery/mojeek.py index 9240b008..8e57f8f3 100644 --- a/theHarvester/discovery/mojeek.py +++ b/theHarvester/discovery/mojeek.py @@ -1,4 +1,5 @@ from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger from theHarvester.parsers import myparser @@ -17,9 +18,9 @@ class SearchMojeek: self.api_key = '' if self.api_key: - print('[*] Mojeek: API key detected.') + output_logger.info('[*] Mojeek: API key detected.') else: - print('[*] Mojeek: No API key found, using default scraping mode.') + output_logger.info('[*] Mojeek: No API key found, using default scraping mode.') async def do_search(self) -> None: headers = {'User-Agent': Core.get_user_agent()} @@ -45,14 +46,14 @@ class SearchMojeek: self.total_results += f' {url} {result.get("title", "")} {result.get("desc", "")} ' elif data and 'status' in data and 'denied' in data['status'].lower(): - print(f'[!] Mojeek API: Access denied ({data["status"]}).') + output_logger.info(f'[!] Mojeek API: Access denied ({data["status"]}).') break if api_success: - print('[*] Mojeek: API search completed successfully.') + output_logger.info('[*] Mojeek: API search completed successfully.') return else: - print('[*] Mojeek: API returned no results, falling back to scraping...') + output_logger.info('[*] Mojeek: API returned no results, falling back to scraping...') urls = [f'https://{self.server}/search?q={self.word}&s={num}' for num in range(0, self.limit, 10)] diff --git a/theHarvester/discovery/onyphe.py b/theHarvester/discovery/onyphe.py index 215a1359..90eaa5a7 100644 --- a/theHarvester/discovery/onyphe.py +++ b/theHarvester/discovery/onyphe.py @@ -2,6 +2,7 @@ from urllib.parse import urlparse from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger # from theHarvester.parsers import myparser @@ -80,7 +81,7 @@ class SearchOnyphe: except Exception: continue else: - print(f'Onhyphe API query did not succeed dumping current response: {self.response}') + output_logger.info(f'Onhyphe API query did not succeed dumping current response: {self.response}') async def get_asns(self) -> set: return self.asns diff --git a/theHarvester/discovery/pentesttools.py b/theHarvester/discovery/pentesttools.py index 6c1d7f7b..dfd3bb75 100644 --- a/theHarvester/discovery/pentesttools.py +++ b/theHarvester/discovery/pentesttools.py @@ -4,6 +4,7 @@ import ujson from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchPentestTools: @@ -37,7 +38,7 @@ class SearchPentestTools: self.total_results = await self.parse_json(res_json) break else: - print(f'Operation get_scan_status failed because: {res_json["error"]}. {res_json["details"]}') + output_logger.info(f'Operation get_scan_status failed because: {res_json["error"]}. {res_json["details"]}') break @staticmethod diff --git a/theHarvester/discovery/rapiddns.py b/theHarvester/discovery/rapiddns.py index 9d3c9876..caa523a9 100644 --- a/theHarvester/discovery/rapiddns.py +++ b/theHarvester/discovery/rapiddns.py @@ -2,6 +2,7 @@ from bs4 import BeautifulSoup from bs4.element import Tag from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchRapidDns: @@ -42,7 +43,7 @@ class SearchRapidDns: self.total_results.append(f'{subdomain}:{str(cells[1].get_text()).strip()}') self.total_results = list({domain for domain in self.total_results}) except Exception as e: - print(f'An exception has occurred: {e!s}') + output_logger.info(f'An exception has occurred: {e!s}') async def process(self, proxy: bool = False) -> None: self.proxy = proxy diff --git a/theHarvester/discovery/robtex.py b/theHarvester/discovery/robtex.py index 7ac842cc..27655c09 100644 --- a/theHarvester/discovery/robtex.py +++ b/theHarvester/discovery/robtex.py @@ -4,6 +4,7 @@ from types import ModuleType import aiohttp from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger json: ModuleType = _stdlib_json try: @@ -11,9 +12,9 @@ try: json = _ujson except ImportError as e: - print(f"'ujson' not available. Falling back to standard 'json' module. Reason: {e}") + output_logger.info(f"'ujson' not available. Falling back to standard 'json' module. Reason: {e}") except (AttributeError, OSError, RuntimeError, SystemError, ValueError) as e: - print(f"Unexpected error while importing 'ujson'. Falling back to standard 'json'. Reason: {e}") + output_logger.info(f"Unexpected error while importing 'ujson'. Falling back to standard 'json'. Reason: {e}") class SearchRobtex: @@ -50,13 +51,13 @@ class SearchRobtex: response = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy) if not response or not isinstance(response, list) or not response[0]: - print(f'No response from Robtex API for: {url}') + output_logger.info(f'No response from Robtex API for: {url}') return try: data = self._safe_parse_json_lines(response[0]) except (TypeError, ValueError) as e: - print(f'Failed to parse JSON lines from Robtex response: {e}') + output_logger.info(f'Failed to parse JSON lines from Robtex response: {e}') return # Extract subdomains from DNS records @@ -99,10 +100,10 @@ class SearchRobtex: if rrdata and (rrdata.endswith(self.word) or f'.{self.word}' in rrdata): self.totalhosts.add(rrdata.rstrip('.')) except (TypeError, ValueError) as e: - print(f'Failed to parse reverse DNS data from Robtex: {e}') + output_logger.info(f'Failed to parse reverse DNS data from Robtex: {e}') except (aiohttp.ClientError, TimeoutError, OSError, TypeError, ValueError) as e: - print(f'Robtex API error: {e}') + output_logger.info(f'Robtex API error: {e}') async def get_hostnames(self) -> set: return self.totalhosts diff --git a/theHarvester/discovery/rocketreach.py b/theHarvester/discovery/rocketreach.py index b9a13681..04bba065 100644 --- a/theHarvester/discovery/rocketreach.py +++ b/theHarvester/discovery/rocketreach.py @@ -2,6 +2,7 @@ import asyncio from theHarvester.discovery.constants import MissingKey, get_delay from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchRocketReach: @@ -49,7 +50,7 @@ class SearchRocketReach: if detail and 'Request was throttled.' in str(detail): # Rate limit has been triggered need to sleep extra - print( + output_logger.info( f'RocketReach requests have been throttled; ' f'{str(detail).split(" ", 3)[-1].replace("available", "availability")}' ) @@ -81,7 +82,7 @@ class SearchRocketReach: await asyncio.sleep(get_delay() + 5) except Exception as e: - print(f'An exception has occurred rocketreach: {e}') + output_logger.info(f'An exception has occurred rocketreach: {e}') async def get_links(self): return self.links diff --git a/theHarvester/discovery/search_dehashed.py b/theHarvester/discovery/search_dehashed.py index 9fc256e0..c45bd65f 100644 --- a/theHarvester/discovery/search_dehashed.py +++ b/theHarvester/discovery/search_dehashed.py @@ -5,6 +5,7 @@ import aiohttp from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import Core +from theHarvester.lib.output import output_logger class SearchDehashed: @@ -24,7 +25,7 @@ class SearchDehashed: self.proxy: bool = False async def do_search(self) -> None: - print(f'\t[+] Performing Dehashed search for: {self.word}') + output_logger.info(f'\t[+] Performing Dehashed search for: {self.word}') page = 1 size = 100 while True: @@ -61,23 +62,23 @@ class SearchDehashed: break self.data.extend(entries) - print(f'\t[+] Page {page} - Retrieved {len(entries)} entries.') + output_logger.info(f'\t[+] Page {page} - Retrieved {len(entries)} entries.') if len(entries) < size: break page += 1 await asyncio.sleep(0.5) except Exception as e: - print(f'\t[!] Dehashed error: {e}') + output_logger.info(f'\t[!] Dehashed error: {e}') break async def print_csv_results(self) -> None: if not self.data: - print('\t[!] No data found.') + output_logger.info('\t[!] No data found.') return - print('\n[Dehashed Results]') - print('Email,Username,Password,Phone,IP,Source') + output_logger.info('\n[Dehashed Results]') + output_logger.info('Email,Username,Password,Phone,IP,Source') for entry in self.data: email = entry.get('email', '') @@ -88,7 +89,7 @@ class SearchDehashed: source = entry.get('database_name', '') csv_line = f'"{email}","{username}","{password}","{phone}","{ip}","{source}"' - print(csv_line) + output_logger.info(csv_line) async def process(self, proxy: bool = False) -> None: self.proxy = proxy diff --git a/theHarvester/discovery/search_dnsdumpster.py b/theHarvester/discovery/search_dnsdumpster.py index b371fcce..30d69e61 100644 --- a/theHarvester/discovery/search_dnsdumpster.py +++ b/theHarvester/discovery/search_dnsdumpster.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchDNSDumpster: @@ -39,7 +40,7 @@ class SearchDNSDumpster: self.ips.add(ip_info['ip']) except Exception as e: - print(f'Error occurred in DNSDumpster search: {e}') + output_logger.info(f'Error occurred in DNSDumpster search: {e}') async def process(self, proxy: bool = False) -> None: await self.do_search() diff --git a/theHarvester/discovery/searchhunterhow.py b/theHarvester/discovery/searchhunterhow.py index e033bdb4..52002ea2 100644 --- a/theHarvester/discovery/searchhunterhow.py +++ b/theHarvester/discovery/searchhunterhow.py @@ -5,6 +5,7 @@ from dateutil.relativedelta import relativedelta from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchHunterHow: @@ -40,11 +41,9 @@ class SearchHunterHow: proxy=self.proxy, ) dct = response[0] - # print(f'json response: ') - # print(dct) if 'code' in dct.keys(): if dct['code'] == 40001: - print(f'Code 40001 indicates for searchhunterhow: {dct["message"]}') + output_logger.info(f'Code 40001 indicates for searchhunterhow: {dct["message"]}') return # total = dct['data']['total'] # TODO determine if total is ever 100 how to get more subdomains? diff --git a/theHarvester/discovery/securityscorecard.py b/theHarvester/discovery/securityscorecard.py index adb114dc..237e36fa 100644 --- a/theHarvester/discovery/securityscorecard.py +++ b/theHarvester/discovery/securityscorecard.py @@ -2,6 +2,7 @@ import aiohttp from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchSecurityScorecard: @@ -36,7 +37,7 @@ class SearchSecurityScorecard: data = await response.json() self._extract_data(data) except Exception as e: - print(f'Error in SecurityScorecard search: {e}') + output_logger.info(f'Error in SecurityScorecard search: {e}') def _extract_data(self, data: dict) -> None: """Extract and categorize security scorecard information.""" diff --git a/theHarvester/discovery/securitytrailssearch.py b/theHarvester/discovery/securitytrailssearch.py index be671a2e..62e7ff5a 100644 --- a/theHarvester/discovery/securitytrailssearch.py +++ b/theHarvester/discovery/securitytrailssearch.py @@ -2,6 +2,7 @@ import asyncio from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger from theHarvester.parsers import securitytrailsparser @@ -27,7 +28,7 @@ class SearchSecuritytrail: auth_responses = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy) auth_responses = auth_responses[0] if 'False' in auth_responses or 'Invalid authentication' in auth_responses: - print('\tKey could not be authenticated exiting program.') + output_logger.info('\tKey could not be authenticated exiting program.') await asyncio.sleep(5) async def do_search(self) -> None: @@ -42,7 +43,7 @@ class SearchSecuritytrail: if domain_response and isinstance(domain_response[0], dict | list): self.domain_data = domain_response[0] if isinstance(domain_response[0], dict) else {} else: - print('SecurityTrails: No JSON response received for domain query') + output_logger.info('SecurityTrails: No JSON response received for domain query') # keep legacy string totalresults for any downstream reliance if domain_response and domain_response[0]: self.results = str(domain_response[0]) @@ -57,12 +58,12 @@ class SearchSecuritytrail: if subdomain_response and isinstance(subdomain_response[0], dict | list): self.subdomains_data = subdomain_response[0] if isinstance(subdomain_response[0], dict) else {} else: - print('SecurityTrails: No JSON response received for subdomain query') + output_logger.info('SecurityTrails: No JSON response received for subdomain query') if subdomain_response and subdomain_response[0]: self.results = str(subdomain_response[0]) self.totalresults += self.results except Exception as e: - print(f'SecurityTrails API error: {e}') + output_logger.info(f'SecurityTrails API error: {e}') return async def process(self, proxy: bool = False) -> None: diff --git a/theHarvester/discovery/sherlockeye.py b/theHarvester/discovery/sherlockeye.py index bf3eb4c2..e4dbbbc4 100644 --- a/theHarvester/discovery/sherlockeye.py +++ b/theHarvester/discovery/sherlockeye.py @@ -6,6 +6,7 @@ import aiohttp from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import Core +from theHarvester.lib.output import output_logger class SearchSherlockeye: @@ -95,7 +96,7 @@ class SearchSherlockeye: def _extract_response(self, response: dict[str, Any]) -> None: if response.get('success') is False: message = response.get('message', 'Unknown Sherlockeye API error') - print(f'Sherlockeye API error: {message}') + output_logger.info(f'Sherlockeye API error: {message}') return data = response.get('data') @@ -128,14 +129,14 @@ class SearchSherlockeye: ) as response: if response.status != 200: error_body = await response.text() - print(f'Sherlockeye API error ({response.status}): {error_body[:200]}') + output_logger.info(f'Sherlockeye API error ({response.status}): {error_body[:200]}') return response_data = await response.json() if isinstance(response_data, dict): self._extract_response(response_data) except Exception as error: - print(f'Sherlockeye API error: {error}') + output_logger.info(f'Sherlockeye API error: {error}') async def get_hostnames(self) -> set[str]: return self.totalhosts diff --git a/theHarvester/discovery/shodan_internetdb.py b/theHarvester/discovery/shodan_internetdb.py index 590cabdb..282d38d5 100644 --- a/theHarvester/discovery/shodan_internetdb.py +++ b/theHarvester/discovery/shodan_internetdb.py @@ -2,6 +2,7 @@ import asyncio import socket from theHarvester.lib.core import AsyncFetcher +from theHarvester.lib.output import output_logger class SearchShodanInternetDB: @@ -31,7 +32,7 @@ class SearchShodanInternetDB: try: addr_infos = await asyncio.to_thread(socket.getaddrinfo, self.word, None, socket.AF_UNSPEC, socket.SOCK_STREAM) except socket.gaierror: - print(f'Shodan InternetDB: Could not resolve domain {self.word}') + output_logger.info(f'Shodan InternetDB: Could not resolve domain {self.word}') return # Deduplicate IPs from the resolution results @@ -42,7 +43,7 @@ class SearchShodanInternetDB: resolved_ips.add(ip) if not resolved_ips: - print(f'Shodan InternetDB: No IPs resolved for {self.word}') + output_logger.info(f'Shodan InternetDB: No IPs resolved for {self.word}') return # Query InternetDB for each resolved IP diff --git a/theHarvester/discovery/shodansearch.py b/theHarvester/discovery/shodansearch.py index 0fdd74c9..2d335aca 100644 --- a/theHarvester/discovery/shodansearch.py +++ b/theHarvester/discovery/shodansearch.py @@ -4,6 +4,7 @@ from shodan import Shodan, exception from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import Core +from theHarvester.lib.output import output_logger class SearchShodan: @@ -21,7 +22,7 @@ class SearchShodan: results = self.api.host(ipaddress) if not results or 'data' not in results or not results['data']: - print(f'Shodan: No data found for IP {ip}') + output_logger.info(f'Shodan: No data found for IP {ip}') return OrderedDict() asn = '' domains: list = list() @@ -106,10 +107,9 @@ class SearchShodan: return self.tracker except exception.APIError: - print(f'{ip}: Not in Shodan') + output_logger.info(f'{ip}: Not in Shodan') self.tracker[ip] = 'Not in Shodan' except Exception as e: - # print(f'Error occurred in the Shodan IP search module: {e}') self.tracker[ip] = f'Error occurred in the Shodan IP search module: {e}' return self.tracker diff --git a/theHarvester/discovery/subdomaincenter.py b/theHarvester/discovery/subdomaincenter.py index 7785cf1f..1dc49487 100644 --- a/theHarvester/discovery/subdomaincenter.py +++ b/theHarvester/discovery/subdomaincenter.py @@ -1,4 +1,5 @@ from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SubdomainCenter: @@ -16,7 +17,7 @@ class SubdomainCenter: self.results = resp[0] self.results = {sub[4:] if sub[:4] == 'www.' and sub[4:] else sub for sub in self.results} except Exception as e: - print(f'An exception has occurred in SubdomainCenter on : {e}') + output_logger.info(f'An exception has occurred in SubdomainCenter on : {e}') async def get_hostnames(self): return self.results diff --git a/theHarvester/discovery/subdomainfinderc99.py b/theHarvester/discovery/subdomainfinderc99.py index dee44a64..c9d2562a 100644 --- a/theHarvester/discovery/subdomainfinderc99.py +++ b/theHarvester/discovery/subdomainfinderc99.py @@ -30,16 +30,7 @@ class SearchSubdomainfinderc99: await asyncio.sleep(get_delay()) second_resp = await AsyncFetcher.post_fetch(self.server, headers=headers, proxy=self.proxy, data=ujson.dumps(data)) - # print(second_resp) self.totalresults += second_resp - # y = await self.get_hostnames() - # print(list(sorted(y))) - # print(f'Found: {len(y)} subdomains') - - # regex = r"value='(https://subdomainfinder\.c99\.nl/scans/\d{4}-\d{2}-\d{2}/" + self.word + r")'" - # match = re.search(regex, second_resp) - # if match: - # print(match.group(1)) async def get_hostnames(self): rawres = myparser.Parser(self.totalresults, self.word) diff --git a/theHarvester/discovery/takeover.py b/theHarvester/discovery/takeover.py index 478a4703..5bff5a50 100644 --- a/theHarvester/discovery/takeover.py +++ b/theHarvester/discovery/takeover.py @@ -5,6 +5,7 @@ from random import shuffle import ujson from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class TakeOver: @@ -32,7 +33,7 @@ class TakeOver: if unparsed_fingerprint['status'] == 'Vulnerable' or unparsed_fingerprint['status'] == 'Edge case': self.fingerprints[unparsed_fingerprint['fingerprint']] = unparsed_fingerprint['service'] except Exception as e: - print(f'An exception has occurred populating takeover fingerprints: {e}, defaulting to static list') + output_logger.info(f'An exception has occurred populating takeover fingerprints: {e}, defaulting to static list') self.fingerprints = { "'Trying to access your account?'": 'Campaign Monitor', '404 Not Found': 'Fly.io', @@ -66,11 +67,11 @@ class TakeOver: matches = re.findall(regex, resp) matches = list(set(matches)) for match in matches: - print(f'\t Takeover detected: {url}') + output_logger.info(f'\t Takeover detected: {url}') if match in self.fingerprints.keys(): # Validation check as to not error out service = self.fingerprints[match] - print(f'\t Type of takeover is: {service} with match: {match}') + output_logger.info(f'\t Type of takeover is: {service} with match: {match}') self.results[url].append({match: service}) async def do_take(self) -> None: @@ -88,15 +89,15 @@ class TakeOver: else: return except IndexError: - print('Response was empty — possible network error or invalid URL.') + output_logger.info('Response was empty — possible network error or invalid URL.') except ujson.JSONDecodeError: - print('Failed to parse JSON — cert fingerprints might be unavailable.') + output_logger.info('Failed to parse JSON — cert fingerprints might be unavailable.') except KeyError as ke: - print(f'Missing expected field in fingerprint: {ke}') + output_logger.info(f'Missing expected field in fingerprint: {ke}') except TypeError as te: - print(f'Invalid response structure: {te}') + output_logger.info(f'Invalid response structure: {te}') except Exception as e: - print(f'Unexpected error: {e}') + output_logger.info(f'Unexpected error: {e}') async def process(self, proxy: bool = False) -> None: self.proxy = proxy diff --git a/theHarvester/discovery/thc.py b/theHarvester/discovery/thc.py index ab98fdc5..0888efc3 100644 --- a/theHarvester/discovery/thc.py +++ b/theHarvester/discovery/thc.py @@ -3,6 +3,7 @@ import asyncio import aiohttp from theHarvester.lib.core import Core +from theHarvester.lib.output import output_logger class SearchThc: @@ -27,12 +28,14 @@ class SearchThc: if response.status == 429: rate_remaining = response.headers.get('x-ratelimit-remaining', '0') wait_time = self.base_delay * (attempt + 1) - print(f'THC rate limit hit (remaining: {rate_remaining}). Waiting {wait_time}s before retry...') + output_logger.info( + f'THC rate limit hit (remaining: {rate_remaining}). Waiting {wait_time}s before retry...' + ) await asyncio.sleep(wait_time) continue if response.status != 200: - print(f'THC returned status {response.status}') + output_logger.info(f'THC returned status {response.status}') return text = await response.text() @@ -47,10 +50,10 @@ class SearchThc: error_msg = str(e).lower() if '429' in error_msg or 'rate' in error_msg: wait_time = self.base_delay * (attempt + 1) - print(f'THC rate limit detected. Waiting {wait_time}s before retry...') + output_logger.info(f'THC rate limit detected. Waiting {wait_time}s before retry...') await asyncio.sleep(wait_time) continue - print(f'An exception has occurred in THC: {e}') + output_logger.info(f'An exception has occurred in THC: {e}') return async def get_hostnames(self) -> set: diff --git a/theHarvester/discovery/threatcrowd.py b/theHarvester/discovery/threatcrowd.py index d0e9cf1c..5c9c01f6 100644 --- a/theHarvester/discovery/threatcrowd.py +++ b/theHarvester/discovery/threatcrowd.py @@ -2,6 +2,7 @@ import json as _stdlib_json from types import ModuleType from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger json: ModuleType = _stdlib_json try: @@ -45,7 +46,7 @@ class SearchThreatcrowd: response = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy) if not response or not isinstance(response, list) or not response[0]: - print(f'No response from ThreatCrowd API for: {self.word}') + output_logger.info(f'No response from ThreatCrowd API for: {self.word}') return try: @@ -55,7 +56,7 @@ class SearchThreatcrowd: # Check response code - '1' means success in ThreatCrowd API response_code = data.get('response_code', '') if response_code and response_code != '1': - print(f'ThreatCrowd API returned error code: {response_code}') + output_logger.info(f'ThreatCrowd API returned error code: {response_code}') return # Extract subdomains - direct list in response @@ -82,10 +83,10 @@ class SearchThreatcrowd: self.totalips.add(resolution.strip()) except Exception as e: - print(f'Failed to parse ThreatCrowd response: {e}') + output_logger.info(f'Failed to parse ThreatCrowd response: {e}') except Exception as e: - print(f'ThreatCrowd API error: {e}') + output_logger.info(f'ThreatCrowd API error: {e}') async def get_hostnames(self) -> set: return self.totalhosts diff --git a/theHarvester/discovery/tombasearch.py b/theHarvester/discovery/tombasearch.py index a3df0db5..33e877df 100644 --- a/theHarvester/discovery/tombasearch.py +++ b/theHarvester/discovery/tombasearch.py @@ -2,6 +2,7 @@ import asyncio from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchTomba: @@ -53,9 +54,11 @@ class SearchTomba: total_number_reqs = response[0]['data']['total'] // 100 # Parse out meta field within initial JSON response to determine the total number of results if total_requests_avail < total_number_reqs: - print('WARNING: The account does not have enough requests to gather all the emails.') - print(f'Total requests available: {total_requests_avail}, total requests needed to be made: {total_number_reqs}') - print( + output_logger.info('WARNING: The account does not have enough requests to gather all the emails.') + output_logger.info( + f'Total requests available: {total_requests_avail}, total requests needed to be made: {total_number_reqs}' + ) + output_logger.info( 'RETURNING current results, If you still wish to run this module despite the current results, please comment out the "if request" line.' ) return diff --git a/theHarvester/discovery/venacussearch.py b/theHarvester/discovery/venacussearch.py index f73703c8..82ac125f 100644 --- a/theHarvester/discovery/venacussearch.py +++ b/theHarvester/discovery/venacussearch.py @@ -4,6 +4,7 @@ import aiohttp from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import Core +from theHarvester.lib.output import output_logger from theHarvester.parsers import venacusparser @@ -48,7 +49,7 @@ class SearchVenacus: current_results = search_data.get('data', []) if not current_results: - print('No more results found.') + output_logger.info('No more results found.') break total_results.extend(current_results) @@ -61,10 +62,10 @@ class SearchVenacus: self.results = total_results if not self.results: - print('No results found.') + output_logger.info('No results found.') except Exception as e: - print(f'An exception has occurred in Venacus: {e}') + output_logger.info(f'An exception has occurred in Venacus: {e}') async def process(self, proxy: bool = False): self.proxy = proxy diff --git a/theHarvester/discovery/waybackarchive.py b/theHarvester/discovery/waybackarchive.py index 9419c27d..1e1b31d1 100644 --- a/theHarvester/discovery/waybackarchive.py +++ b/theHarvester/discovery/waybackarchive.py @@ -1,6 +1,7 @@ import re from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchWaybackarchive: @@ -61,11 +62,11 @@ class SearchWaybackarchive: self.totalhosts.add(domain) except Exception as e: - print(f'Wayback Archive API error for URL {url}: {e}') + output_logger.info(f'Wayback Archive API error for URL {url}: {e}') continue except Exception as e: - print(f'Wayback Archive API error: {e}') + output_logger.info(f'Wayback Archive API error: {e}') async def get_hostnames(self) -> set: return self.totalhosts diff --git a/theHarvester/discovery/whoisxml.py b/theHarvester/discovery/whoisxml.py index ef1aad56..fd552721 100644 --- a/theHarvester/discovery/whoisxml.py +++ b/theHarvester/discovery/whoisxml.py @@ -1,5 +1,6 @@ from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger class SearchWhoisXML: @@ -25,7 +26,7 @@ class SearchWhoisXML: # Parse the response according to the example JSON structure: # {"search":"example.com.com","result":{"count":10000,"records":[{"domain":"test.example.com","firstSeen":1678169400,"lastSeen":1678169400}]}} self.total_results = [] - print(response[0]) + output_logger.info(response[0]) if response and response[0]: # Extract domains from the records array if 'result' in response[0] and 'records' in response[0]['result']: diff --git a/theHarvester/discovery/windvane.py b/theHarvester/discovery/windvane.py index 7d60f5bb..0be5519f 100644 --- a/theHarvester/discovery/windvane.py +++ b/theHarvester/discovery/windvane.py @@ -2,6 +2,7 @@ import json as _stdlib_json from types import ModuleType from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger json: ModuleType = _stdlib_json try: @@ -76,11 +77,11 @@ class SearchWindvane: await self._search_emails(headers) else: # Without API key, try alternative/limited approaches - print('[*] Windvane API key not found. Using limited unauthenticated access.') + output_logger.info('[*] Windvane API key not found. Using limited unauthenticated access.') await self._search_subdomains_limited(headers) except Exception as e: - print(f'Windvane API error: {e}') + output_logger.info(f'Windvane API error: {e}') async def _search_subdomains(self, headers: dict) -> None: """Search for subdomains using /ListSubDomain endpoint""" @@ -112,15 +113,15 @@ class SearchWindvane: else: # API error - stop pagination if response_data.get('code') != 0: - print(f'Windvane subdomain API error: {response_data.get("msg", "Unknown error")}') + output_logger.info(f'Windvane subdomain API error: {response_data.get("msg", "Unknown error")}') break except Exception as e: - print(f'Windvane subdomain request failed: {e}') + output_logger.info(f'Windvane subdomain request failed: {e}') break except Exception as e: - print(f'Windvane subdomain search error: {e}') + output_logger.info(f'Windvane subdomain search error: {e}') async def _search_dns_history(self, headers: dict) -> None: """Search DNS history using /ListDNS endpoint for additional subdomains and IPs""" @@ -160,11 +161,11 @@ class SearchWindvane: break except Exception as e: - print(f'Windvane DNS history request failed: {e}') + output_logger.info(f'Windvane DNS history request failed: {e}') break except Exception as e: - print(f'Windvane DNS history search error: {e}') + output_logger.info(f'Windvane DNS history search error: {e}') async def _search_emails(self, headers: dict) -> None: """Search for emails using /ListEmail endpoint""" @@ -194,10 +195,10 @@ class SearchWindvane: self.totalhosts.add(domain.lower()) except Exception as e: - print(f'Windvane email search request failed: {e}') + output_logger.info(f'Windvane email search request failed: {e}') except Exception as e: - print(f'Windvane email search error: {e}') + output_logger.info(f'Windvane email search error: {e}') async def _search_subdomains_limited(self, headers: dict) -> None: """Limited subdomain search without API key - tries simpler approaches""" @@ -229,22 +230,22 @@ class SearchWindvane: if domain and domain.endswith(self.word): self.totalhosts.add(domain.lower()) - print(f'[*] Found {len(subdomains)} subdomains with limited access') + output_logger.info(f'[*] Found {len(subdomains)} subdomains with limited access') else: # If API call fails, try fallback approaches await self._fallback_search() except Exception as e: - print(f'Windvane limited API failed: {e}') + output_logger.info(f'Windvane limited API failed: {e}') await self._fallback_search() except Exception as e: - print(f'Windvane limited search error: {e}') + output_logger.info(f'Windvane limited search error: {e}') async def _fallback_search(self) -> None: """Fallback search using common subdomain patterns when API is unavailable""" try: - print('[*] API unavailable, using fallback subdomain pattern search...') + output_logger.info('[*] API unavailable, using fallback subdomain pattern search...') # Common subdomain prefixes to try common_subdomains = [ @@ -296,12 +297,12 @@ class SearchWindvane: continue if found_count > 0: - print(f'[*] Found {found_count} subdomains using DNS fallback') + output_logger.info(f'[*] Found {found_count} subdomains using DNS fallback') else: - print('[*] No additional subdomains found via fallback methods') + output_logger.info('[*] No additional subdomains found via fallback methods') except Exception as e: - print(f'Fallback search error: {e}') + output_logger.info(f'Fallback search error: {e}') def set_api_key(self, api_key: str) -> None: """Set the API key for authenticated requests diff --git a/theHarvester/discovery/zoomeyesearch.py b/theHarvester/discovery/zoomeyesearch.py index 9f79dea2..3f3f7069 100644 --- a/theHarvester/discovery/zoomeyesearch.py +++ b/theHarvester/discovery/zoomeyesearch.py @@ -6,6 +6,7 @@ from typing import Any from theHarvester.discovery.constants import MissingKey, get_delay from theHarvester.lib.core import AsyncFetcher, Core +from theHarvester.lib.output import output_logger from theHarvester.parsers import myparser @@ -76,7 +77,7 @@ class SearchZoomEye: # some responses put HTTP-like code here return resp.get('status') in (0, 200) except Exception as e: - print(f'An error occurred while trying to parse {resp} : {e}') + output_logger.info(f'An error occurred while trying to parse {resp} : {e}') return False # If no explicit status, assume success and let parsing validate @@ -95,9 +96,9 @@ class SearchZoomEye: try: return int(payload['available']) except ValueError: - print('Payload availablity is not a integer') + output_logger.info('Payload availablity is not a integer') except Exception as e: - print(f'An error occurred in page_total_from_payload : {e}') + output_logger.info(f'An error occurred in page_total_from_payload : {e}') total_results = payload.get('total') or payload.get('count') or payload.get('total_count') if isinstance(total_results, int) and total_results >= 0: size = payload.get('size') or page_size @@ -350,7 +351,7 @@ class SearchZoomEye: except Exception as e: # Continue processing other matches instead of failing completely - print(f'ZoomEye parsing error: {e}') + output_logger.info(f'ZoomEye parsing error: {e}') return hostnames, emails, ips, asns, iurls diff --git a/theHarvester/lib/api/api.py b/theHarvester/lib/api/api.py index 587c8bf9..09f72302 100644 --- a/theHarvester/lib/api/api.py +++ b/theHarvester/lib/api/api.py @@ -17,6 +17,7 @@ from starlette.staticfiles import StaticFiles from theHarvester import __main__ from theHarvester.lib.api.additional_endpoints import router as additional_router +from theHarvester.lib.output import output_logger API_RATE_LIMIT = os.getenv('API_RATE_LIMIT', '5/minute') @@ -185,7 +186,7 @@ async def getsources(request: Request) -> Response: except Exception as e: # Log the error and return a detailed error response error_traceback = traceback.format_exc() - print(f'Error in getsources endpoint: {e!s}\n{error_traceback}') + output_logger.info(f'Error in getsources endpoint: {e!s}\n{error_traceback}') return JSONResponse( { @@ -264,7 +265,7 @@ async def dnsbrute( except Exception as e: # Log the error and return a detailed error response error_traceback = traceback.format_exc() - print(f'Error in dnsbrute endpoint: {e!s}\n{error_traceback}') + output_logger.info(f'Error in dnsbrute endpoint: {e!s}\n{error_traceback}') return JSONResponse( { @@ -383,7 +384,7 @@ async def query( except Exception as e: # Log the error and return a detailed error response error_traceback = traceback.format_exc() - print(f'Error in query endpoint: {e!s}\n{error_traceback}') + output_logger.info(f'Error in query endpoint: {e!s}\n{error_traceback}') return JSONResponse( { diff --git a/theHarvester/lib/api/api_example.py b/theHarvester/lib/api/api_example.py index 62a04caa..25c90397 100644 --- a/theHarvester/lib/api/api_example.py +++ b/theHarvester/lib/api/api_example.py @@ -5,7 +5,7 @@ import asyncio import aiohttp import netaddr -from theHarvester.lib.output import print_section, sorted_unique +from theHarvester.lib.output import output_logger, print_section, sorted_unique async def fetch_json(session, url): @@ -14,7 +14,7 @@ async def fetch_json(session, url): response.raise_for_status() # Raise an exception for 4XX/5XX responses return await response.json() except Exception as e: - print(f'Error fetching data from {url}: {e}') + output_logger.info(f'Error fetching data from {url}: {e}') return {} @@ -24,7 +24,7 @@ async def fetch(session, url): response.raise_for_status() # Raise an exception for 4XX/5XX responses return await response.text() except Exception as e: - print(f'Error fetching data from {url}: {e}') + output_logger.info(f'Error fetching data from {url}: {e}') return '' @@ -57,7 +57,7 @@ async def main() -> None: interesting_urls = sorted_unique(interesting_urls) if len(twitter_people_list_tracker) == 0: - print('\n[*] No Twitter users found.') + output_logger.info('\n[*] No Twitter users found.') elif len(twitter_people_list_tracker) >= 1: print_section( '\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker)), @@ -67,7 +67,7 @@ async def main() -> None: twitter_people_list_tracker = sorted_unique(twitter_people_list_tracker) if len(linkedin_people_list_tracker) == 0: - print('\n[*] No LinkedIn users found.') + output_logger.info('\n[*] No LinkedIn users found.') elif len(linkedin_people_list_tracker) >= 1: print_section( '\n[*] LinkedIn Users found: ' + str(len(linkedin_people_list_tracker)), @@ -77,7 +77,7 @@ async def main() -> None: linkedin_people_list_tracker = sorted_unique(linkedin_people_list_tracker) if len(linkedin_links_tracker) == 0: - print('\n[*] No LinkedIn links found.') + output_logger.info('\n[*] No LinkedIn links found.') else: print_section( f'\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}', linkedin_links_tracker, '---------------------' @@ -86,33 +86,33 @@ async def main() -> None: length_urls = len(trello_urls) if length_urls == 0: - print('\n[*] No Trello URLs found.') + output_logger.info('\n[*] No Trello URLs found.') else: print_section('\n[*] Trello URLs found: ' + str(length_urls), trello_urls, '--------------------') if len(ips) == 0: - print('\n[*] No IPs found.') + output_logger.info('\n[*] No IPs found.') else: - print('\n[*] IPs found: ' + str(len(ips))) - print('-------------------') + output_logger.info('\n[*] IPs found: ' + str(len(ips))) + output_logger.info('-------------------') # use netaddr as the list may contain ipv4 and ipv6 addresses ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(ips)]) - print('\n'.join(map(str, ip_list))) + output_logger.info('\n'.join(map(str, ip_list))) if len(emails) == 0: - print('\n[*] No emails found.') + output_logger.info('\n[*] No emails found.') else: - print('\n[*] Emails found: ' + str(len(emails))) - print('----------------------') + output_logger.info('\n[*] Emails found: ' + str(len(emails))) + output_logger.info('----------------------') all_emails = sorted_unique(emails) - print('\n'.join(all_emails)) + output_logger.info('\n'.join(all_emails)) if len(hosts) == 0: - print('\n[*] No hosts found.\n\n') + output_logger.info('\n[*] No hosts found.\n\n') else: - print('\n[*] Hosts found: ' + str(len(hosts))) - print('---------------------') - print('\n'.join(hosts)) + output_logger.info('\n[*] Hosts found: ' + str(len(hosts))) + output_logger.info('---------------------') + output_logger.info('\n'.join(hosts)) if __name__ == '__main__': diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index f40a147e..528e6ab0 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -16,6 +16,7 @@ import yaml from aiohttp_socks import ProxyConnector from theHarvester import __version__ +from theHarvester.lib.output import output_logger if TYPE_CHECKING: from collections.abc import Sized @@ -77,7 +78,7 @@ class Core: file = path.expanduser() / filename config = file.read_text() if not Core.quiet: - print(f'Read {filename} from {file}') + output_logger.info(f'Read {filename} from {file}') return config # Fallback to creating default in the user's home dir @@ -85,7 +86,7 @@ class Core: dest = CONFIG_DIRS[0].expanduser() / filename dest.parent.mkdir(exist_ok=True) dest.write_text(default) - print(f'Created default {filename} at {dest}') + output_logger.info(f'Created default {filename} at {dest}') return default @staticmethod @@ -259,19 +260,19 @@ class Core: @staticmethod def banner() -> None: - print('*******************************************************************') - print('* _ _ _ *') - print(r'* | |_| |__ ___ /\ /\__ _ _ ____ _____ ___| |_ ___ _ __ *') - print(r"* | __| _ \ / _ \ / /_/ / _` | '__\ \ / / _ \/ __| __/ _ \ '__| *") - print(r'* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *') - print(r'* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *') - print('* *') - print('* theHarvester {version}{filler}*'.format(version=__version__, filler=' ' * (51 - len(__version__)))) - print('* Coded by Christian Martorella *') - print('* Edge-Security Research *') - print('* cmartorella@edge-security.com *') - print('* *') - print('*******************************************************************') + output_logger.info('*******************************************************************') + output_logger.info('* _ _ _ *') + output_logger.info(r'* | |_| |__ ___ /\ /\__ _ _ ____ _____ ___| |_ ___ _ __ *') + output_logger.info(r"* | __| _ \ / _ \ / /_/ / _` | '__\ \ / / _ \/ __| __/ _ \ '__| *") + output_logger.info(r'* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *') + output_logger.info(r'* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *') + output_logger.info('* *') + output_logger.info('* theHarvester {version}{filler}*'.format(version=__version__, filler=' ' * (51 - len(__version__)))) + output_logger.info('* Coded by Christian Martorella *') + output_logger.info('* Edge-Security Research *') + output_logger.info('* cmartorella@edge-security.com *') + output_logger.info('* *') + output_logger.info('*******************************************************************') @staticmethod def get_supportedengines() -> list[str]: @@ -667,7 +668,7 @@ class AsyncFetcher: await asyncio.sleep(5) return url, await response.text() except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError) as e: - print(f'Takeover check error: {e}') + output_logger.info(f'Takeover check error: {e}') return url, '' @classmethod @@ -743,5 +744,5 @@ class AsyncFetcher: def show_default_error_message(engine_name: str, word: str, error) -> None: - print(f"Failed to process {engine_name} search for word: '{word}'") - print(f'Error Message: {error}') + output_logger.info(f"Failed to process {engine_name} search for word: '{word}'") + output_logger.info(f'Error Message: {error}') diff --git a/theHarvester/lib/output.py b/theHarvester/lib/output.py index 580fb24e..e578ed5d 100644 --- a/theHarvester/lib/output.py +++ b/theHarvester/lib/output.py @@ -1,11 +1,43 @@ from __future__ import annotations +import logging +import sys from collections.abc import Hashable, Iterable, Sequence from typing import TypeVar T = TypeVar('T', bound=Hashable) +class _OperatorOutputHandler(logging.Handler): + """Write operator-facing messages to the current stdout stream.""" + + def emit(self, record: logging.LogRecord) -> None: + try: + sys.stdout.write(f'{self.format(record)}\n') + except Exception: + self.handleError(record) + + +output_logger = logging.getLogger('theHarvester.output') +output_logger.setLevel(logging.INFO) +output_logger.propagate = False +if not any(isinstance(handler, _OperatorOutputHandler) for handler in output_logger.handlers): + output_logger.addHandler(_OperatorOutputHandler()) + + +def configure_logging(*, verbose: bool) -> None: + """Configure CLI diagnostics without taking ownership from an embedding host.""" + root_logger = logging.getLogger() + if not root_logger.handlers: + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(levelname)s %(name)s: %(message)s')) + root_logger.addHandler(handler) + root_logger.setLevel(logging.WARNING) + + if verbose: + logging.getLogger('theHarvester').setLevel(logging.INFO) + + def sorted_unique[T: Hashable](items: Iterable[T]) -> list[T]: unique_items = list(dict.fromkeys(items)) unique_items.sort(key=lambda item: str(item)) @@ -13,25 +45,25 @@ def sorted_unique[T: Hashable](items: Iterable[T]) -> list[T]: def print_section(header: str, items: Iterable[str], separator: str) -> None: - print(header) - print(separator) + output_logger.info(header) + output_logger.info(separator) for item in sorted_unique(items): - print(item) + output_logger.info(item) def print_linkedin_sections( engines: Sequence[str], people: Sequence[str], links: Sequence[str], separator: str = '---------------------' ) -> None: if len(people) == 0 and 'linkedin' in engines: - print('\n[*] No LinkedIn users found.\n\n') + output_logger.info('\n[*] No LinkedIn users found.\n\n') elif len(people) >= 1: - print('\n[*] LinkedIn Users found: ' + str(len(people))) - print(separator) + output_logger.info('\n[*] LinkedIn Users found: ' + str(len(people))) + output_logger.info(separator) for usr in sorted_unique(people): - print(usr) + output_logger.info(usr) if 'linkedin' in engines or 'rocketreach' in engines: - print(f'\n[*] LinkedIn Links found: {len(links)}') - print(separator) + output_logger.info(f'\n[*] LinkedIn Links found: {len(links)}') + output_logger.info(separator) for link in sorted_unique(links): - print(link) + output_logger.info(link) diff --git a/theHarvester/lib/stash.py b/theHarvester/lib/stash.py index 9db7d29c..784ec6ff 100644 --- a/theHarvester/lib/stash.py +++ b/theHarvester/lib/stash.py @@ -5,6 +5,8 @@ from sqlite3.dbapi2 import Row import aiosqlite +from theHarvester.lib.output import output_logger + db_path = os.path.expanduser('~/.local/share/theHarvester') if not os.path.isdir(db_path): @@ -56,7 +58,7 @@ class StashManager: ) await db.commit() except Exception as e: - print(f'Unexpected error while storing result: {e}') + output_logger.info(f'Unexpected error while storing result: {e}') async def store_all(self, domain, all, res_type, source) -> None: # people are not stored in the database @@ -77,7 +79,7 @@ class StashManager: ) await db.commit() except Exception as e: - print(f'Unexpected error while storing result: {e}') + output_logger.info(f'Unexpected error while storing result: {e}') async def generatedashboardcode(self, domain): try: @@ -165,7 +167,7 @@ class StashManager: self.latestscandomain['scandetailsshodan'] = scandetailsshodan return self.latestscandomain except Exception as e: - print(f'Unexpected error while generating the dashboard code: {e}') + output_logger.info(f'Unexpected error while generating the dashboard code: {e}') async def getlatestscanresults(self, domain, previousday: bool = False) -> Iterable[Row | str] | None: try: @@ -206,7 +208,7 @@ class StashManager: self.previousscanresults = list(results) return self.previousscanresults except Exception as e: - print(f'Error in getting the previous scan results from the database: {e}') + output_logger.info(f'Error in getting the previous scan results from the database: {e}') else: try: cursor = await conn.execute( @@ -231,9 +233,9 @@ class StashManager: self.latestscanresults = list(results) return self.latestscanresults except Exception as e: - print(f'Error in getting the latest scan results from the database: {e}') + output_logger.info(f'Error in getting the latest scan results from the database: {e}') except Exception as e: - print(f'Error connecting to theHarvester database: {e}') + output_logger.info(f'Error connecting to theHarvester database: {e}') return self.latestscanresults async def getscanboarddata(self): @@ -259,7 +261,7 @@ class StashManager: self.scanboarddata['domains'] = self._col0_int(data) return self.scanboarddata except Exception as e: - print(f'Unexpected error while getting the scanboard data: {e}') + output_logger.info(f'Unexpected error while getting the scanboard data: {e}') async def getscanhistorydomain(self, domain): try: @@ -306,7 +308,7 @@ class StashManager: self.domainscanhistory.append(results) return self.domainscanhistory except Exception as e: - print(f'Unexpected error while getting the scanhistory of a domain: {e}') + output_logger.info(f'Unexpected error while getting the scanhistory of a domain: {e}') async def getpluginscanstatistics(self) -> Iterable[Row] | None: try: @@ -321,7 +323,7 @@ class StashManager: results = await cursor.fetchall() self.scanstats = list(results) except Exception as e: - print(f'Unexpected error while getting a plugins scanstatistics: {e}') + output_logger.info(f'Unexpected error while getting a plugins scanstatistics: {e}') return self.scanstats async def latestscanchartdata(self, domain): @@ -409,6 +411,6 @@ class StashManager: self.latestscandomain['scandetailsshodan'] = scandetailsshodan return self.latestscandomain except aiosqlite.Error as db_err: - print(f"Database error occurred for domain '{domain}': {db_err}") + output_logger.info(f"Database error occurred for domain '{domain}': {db_err}") except Exception as e: - print(f"Unexpected error in latestscanchartdata for domain '{domain}': {e}") + output_logger.info(f"Unexpected error in latestscanchartdata for domain '{domain}': {e}") diff --git a/theHarvester/screenshot/screenshot.py b/theHarvester/screenshot/screenshot.py index f560ed0d..be39a2ab 100644 --- a/theHarvester/screenshot/screenshot.py +++ b/theHarvester/screenshot/screenshot.py @@ -13,6 +13,8 @@ import certifi from aiohttp_socks import ProxyConnector from playwright.async_api import async_playwright +from theHarvester.lib.output import output_logger + class ScreenShotter: def __init__(self, output) -> None: @@ -31,7 +33,7 @@ class ScreenShotter: return False return True except Exception as e: - print(f"An exception has occurred while attempting to verify output path's existence: {e}") + output_logger.info(f"An exception has occurred while attempting to verify output path's existence: {e}") return False @staticmethod @@ -41,9 +43,9 @@ class ScreenShotter: async with async_playwright() as p: browser = await p.chromium.launch() await browser.close() - print('Playwright and Chromium are successfully installed.') + output_logger.info('Playwright and Chromium are successfully installed.') except Exception as e: - print(f'An exception has occurred while attempting to verify installation: {e}') + output_logger.info(f'An exception has occurred while attempting to verify installation: {e}') @staticmethod def chunk_list(items: Collection, chunk_size: int) -> list: @@ -86,13 +88,13 @@ class ScreenShotter: text = await resp.text('UTF-8') return f'http://{url}' if not url.startswith('http') else url, text except Exception as e: - print(f'An exception has occurred while attempting to visit {url} : {e}') + output_logger.info(f'An exception has occurred while attempting to visit {url} : {e}') return '', '' async def take_screenshot(self, url: str) -> None: url = f'http://{url}' if not url.startswith('http') else url url = url.replace('www.', '') - print(f'Attempting to take a screenshot of: {url}') + output_logger.info(f'Attempting to take a screenshot of: {url}') async with async_playwright() as p: browser = await p.chromium.launch(headless=True) # New browser context @@ -106,10 +108,10 @@ class ScreenShotter: await page.goto(url, timeout=35000) await page.screenshot(path=path) except Exception as e: - print(f'An exception has occurred attempting to screenshot: {url} : {e}') + output_logger.info(f'An exception has occurred attempting to screenshot: {url} : {e}') path = '' finally: await page.close() await context.close() await browser.close() - print(date, url, path) + output_logger.info('%s %s %s', date, url, path)