From fa2dedd38becaa68a3fe1daa68e7ba4e47e262f2 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sat, 24 Jan 2026 11:28:20 +0000 Subject: [PATCH] Refactor and standardize output handling with new utilities in `theHarvester.lib.output`, add tests for `print_linkedin_sections` and `sorted_unique`. Fix regex inaccuracies and enhance CORS validation. --- tests/lib/test_output.py | 38 ++++++++++++++++++++ tests/test_security.py | 2 +- theHarvester/__main__.py | 52 +++++++++------------------ theHarvester/lib/api/api_example.py | 55 +++++++++++++---------------- theHarvester/lib/output.py | 37 +++++++++++++++++++ theHarvester/parsers/myparser.py | 2 +- 6 files changed, 117 insertions(+), 69 deletions(-) create mode 100644 tests/lib/test_output.py create mode 100644 theHarvester/lib/output.py diff --git a/tests/lib/test_output.py b/tests/lib/test_output.py new file mode 100644 index 00000000..27c6da7b --- /dev/null +++ b/tests/lib/test_output.py @@ -0,0 +1,38 @@ +from __future__ import annotations + + +from theHarvester.lib.output import print_linkedin_sections, sorted_unique + + +def test_sorted_unique_sorts_and_deduplicates() -> None: + assert sorted_unique(["b", "a", "b"]) == ["a", "b"] + + +def test_print_linkedin_sections_prints_links_when_present(capsys) -> None: + # Regression coverage: the CLI previously never printed LinkedIn links when the list was non-empty. + print_linkedin_sections( + engines=["linkedin"], + people=[], + links=["https://b.example", "https://a.example", "https://a.example"], + ) + + out = capsys.readouterr().out + assert "No LinkedIn users found" in out + assert "LinkedIn Links found: 3" in out + assert "https://a.example" in out + assert "https://b.example" in out + + +def test_print_linkedin_sections_prints_people_and_links(capsys) -> None: + print_linkedin_sections( + engines=["rocketreach"], + people=["bob", "alice", "bob"], + links=["https://z.example", "https://z.example"], + ) + + out = capsys.readouterr().out + assert "LinkedIn Users found: 3" in out + assert "alice" in out + assert "bob" in out + assert "LinkedIn Links found: 2" in out + assert "https://z.example" in out diff --git a/tests/test_security.py b/tests/test_security.py index c143ea90..39991f3c 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -36,7 +36,7 @@ class TestCORSConfiguration: allow_origins = options.get('allow_origins', []) allow_credentials = options.get('allow_credentials', False) - if '*' in allow_origins: + if isinstance(allow_origins, (list, tuple, set)) and '*' in allow_origins: assert ( allow_credentials is False ), 'CRITICAL: CORS must not allow credentials with wildcard origins (CVE risk)' diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index b8416355..4436fea1 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -72,6 +72,7 @@ from theHarvester.discovery import ( from theHarvester.discovery.constants import MissingKey from theHarvester.lib import hostchecker, stash from theHarvester.lib.core import DATA_DIR, Core, show_default_error_message +from theHarvester.lib.output import print_linkedin_sections, print_section, sorted_unique from theHarvester.screenshot.screenshot import ScreenShotter if TYPE_CHECKING: @@ -1330,43 +1331,26 @@ async def start(rest_args: argparse.Namespace | None = None): # Results if len(total_asns) > 0: - print(f'\n[*] ASNS found: {len(total_asns)}') - print('--------------------') - total_asns = list(sorted(set(total_asns))) - for asn in total_asns: - print(asn) + print_section(f'\n[*] ASNS found: {len(total_asns)}', total_asns, '--------------------') + total_asns = sorted_unique(total_asns) if len(interesting_urls) > 0: - print(f'\n[*] Interesting Urls found: {len(interesting_urls)}') - print('--------------------') - interesting_urls = list(sorted(set(interesting_urls))) - for iurl in interesting_urls: - print(iurl) + print_section(f'\n[*] Interesting Urls found: {len(interesting_urls)}', interesting_urls, '--------------------') + interesting_urls = sorted_unique(interesting_urls) if len(twitter_people_list_tracker) == 0 and 'twitter' in engines: print('\n[*] No Twitter users found.\n\n') elif len(twitter_people_list_tracker) >= 1: - print('\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker))) - print('---------------------') - twitter_people_list_tracker = list(sorted(set(twitter_people_list_tracker))) - for usr in twitter_people_list_tracker: - print(usr) + print_section( + '\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker)), + twitter_people_list_tracker, + '---------------------', + ) + twitter_people_list_tracker = sorted_unique(twitter_people_list_tracker) - if len(linkedin_people_list_tracker) == 0 and 'linkedin' in engines: - print('\n[*] No LinkedIn users found.\n\n') - elif len(linkedin_people_list_tracker) >= 1: - print('\n[*] LinkedIn Users found: ' + str(len(linkedin_people_list_tracker))) - print('---------------------') - linkedin_people_list_tracker = list(sorted(set(linkedin_people_list_tracker))) - for usr in linkedin_people_list_tracker: - print(usr) - - if len(linkedin_links_tracker) == 0 and ('linkedin' in engines or 'rocketreach' in engines): - print(f'\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}') - linkedin_links_tracker = list(sorted(set(linkedin_links_tracker))) - print('---------------------') - for link in linkedin_people_list_tracker: - print(link) + print_linkedin_sections(engines, linkedin_people_list_tracker, linkedin_links_tracker) + linkedin_people_list_tracker = sorted_unique(linkedin_people_list_tracker) + linkedin_links_tracker = sorted_unique(linkedin_links_tracker) length_urls = len(all_urls) if length_urls == 0: @@ -1374,11 +1358,8 @@ async def start(rest_args: argparse.Namespace | None = None): print('\n[*] No Trello URLs found.') else: total = length_urls - print('\n[*] Trello URLs found: ' + str(total)) - print('--------------------') - all_urls = list(sorted(set(all_urls))) - for url in sorted(all_urls): - print(url) + print_section('\n[*] Trello URLs found: ' + str(total), all_urls, '--------------------') + all_urls = sorted_unique(all_urls) if len(all_ip) == 0: print('\n[*] No IPs found.') @@ -1847,7 +1828,6 @@ async def start(rest_args: argparse.Namespace | None = None): try: print('\n[*] Performing BuiltWith scan...') builtwith_scanner = builtwith.SearchBuiltWith(word) - # Use the process method according to the original structure of this module await builtwith_scanner.process(use_proxy) hosts = await builtwith_scanner.get_hostnames() diff --git a/theHarvester/lib/api/api_example.py b/theHarvester/lib/api/api_example.py index 758ad062..2e98fe4b 100644 --- a/theHarvester/lib/api/api_example.py +++ b/theHarvester/lib/api/api_example.py @@ -7,6 +7,8 @@ import asyncio import aiohttp import netaddr +from theHarvester.lib.output import print_section, sorted_unique + async def fetch_json(session, url): try: @@ -50,55 +52,46 @@ async def main() -> None: hosts = fetched_json.get('hosts', []) if len(total_asns) > 0: - print(f'\n[*] ASNS found: {len(total_asns)}') - print('--------------------') - total_asns = list(sorted(set(total_asns))) - for asn in total_asns: - print(asn) + print_section(f'\n[*] ASNS found: {len(total_asns)}', total_asns, '--------------------') + total_asns = sorted_unique(total_asns) if len(interesting_urls) > 0: - print(f'\n[*] Interesting Urls found: {len(interesting_urls)}') - print('--------------------') - interesting_urls = list(sorted(set(interesting_urls))) - for iurl in interesting_urls: - print(iurl) + print_section(f'\n[*] Interesting Urls found: {len(interesting_urls)}', interesting_urls, '--------------------') + interesting_urls = sorted_unique(interesting_urls) if len(twitter_people_list_tracker) == 0: print('\n[*] No Twitter users found.') elif len(twitter_people_list_tracker) >= 1: - print('\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker))) - print('---------------------') - twitter_people_list_tracker = list(sorted(set(twitter_people_list_tracker))) - for usr in twitter_people_list_tracker: - print(usr) + print_section( + '\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker)), + twitter_people_list_tracker, + '---------------------', + ) + twitter_people_list_tracker = sorted_unique(twitter_people_list_tracker) if len(linkedin_people_list_tracker) == 0: print('\n[*] No LinkedIn users found.') elif len(linkedin_people_list_tracker) >= 1: - print('\n[*] LinkedIn Users found: ' + str(len(linkedin_people_list_tracker))) - print('---------------------') - linkedin_people_list_tracker = list(sorted(set(linkedin_people_list_tracker))) - for usr in linkedin_people_list_tracker: - print(usr) + print_section( + '\n[*] LinkedIn Users found: ' + str(len(linkedin_people_list_tracker)), + linkedin_people_list_tracker, + '---------------------', + ) + linkedin_people_list_tracker = sorted_unique(linkedin_people_list_tracker) if len(linkedin_links_tracker) == 0: print('\n[*] No LinkedIn links found.') else: - print(f'\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}') - print('---------------------') - linkedin_links_tracker = list(sorted(set(linkedin_links_tracker))) - for link in linkedin_links_tracker: - print(link) + print_section( + f'\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}', linkedin_links_tracker, '---------------------' + ) + linkedin_links_tracker = sorted_unique(linkedin_links_tracker) length_urls = len(trello_urls) if length_urls == 0: print('\n[*] No Trello URLs found.') else: - print('\n[*] Trello URLs found: ' + str(length_urls)) - print('--------------------') - all_urls = list(sorted(set(trello_urls))) - for url in sorted(all_urls): - print(url) + print_section('\n[*] Trello URLs found: ' + str(length_urls), trello_urls, '--------------------') if len(ips) == 0: print('\n[*] No IPs found.') @@ -114,7 +107,7 @@ async def main() -> None: else: print('\n[*] Emails found: ' + str(len(emails))) print('----------------------') - all_emails = sorted(list(set(emails))) + all_emails = sorted_unique(emails) print('\n'.join(all_emails)) if len(hosts) == 0: diff --git a/theHarvester/lib/output.py b/theHarvester/lib/output.py new file mode 100644 index 00000000..af27b107 --- /dev/null +++ b/theHarvester/lib/output.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from collections.abc import Hashable, Iterable, Sequence +from typing import TypeVar + +T = TypeVar('T', bound=Hashable) + + +def sorted_unique[T: Hashable](items: Iterable[T]) -> list[T]: + # `T` is only required to be hashable, not orderable. + # Sorting by `str` keeps output deterministic without requiring rich comparison support. + return list(sorted(set(items), key=str)) + + +def print_section(header: str, items: Iterable[str], separator: str) -> None: + print(header) + print(separator) + for item in sorted_unique(items): + print(item) + + +def print_linkedin_sections( + engines: Sequence[str], people: Sequence[str], links: Sequence[str], separator: str = '---------------------' +) -> None: + if len(people) == 0 and 'linkedin' in engines: + print('\n[*] No LinkedIn users found.\n\n') + elif len(people) >= 1: + print('\n[*] LinkedIn Users found: ' + str(len(people))) + print(separator) + for usr in sorted_unique(people): + print(usr) + + if 'linkedin' in engines or 'rocketreach' in engines: + print(f'\n[*] LinkedIn Links found: {len(links)}') + print(separator) + for link in sorted_unique(links): + print(link) diff --git a/theHarvester/parsers/myparser.py b/theHarvester/parsers/myparser.py index 150231c9..0fbbefba 100644 --- a/theHarvester/parsers/myparser.py +++ b/theHarvester/parsers/myparser.py @@ -110,7 +110,7 @@ class Parser: return sets async def urls(self) -> Set[str]: - found = re.finditer(r'(http|https)://(www\.)?trello.com/([a-zA-Z\d\-_\.]+/?)*', self.results) + found = re.finditer(r'(http|https)://(www\.)?trello.com/([a-zA-Z\d\-_.]+/?)*', self.results) urls = {match.group().strip() for match in found} return urls