Refactor and standardize output handling with new utilities in theHarvester.lib.output, add tests for print_linkedin_sections and sorted_unique. Fix regex inaccuracies and enhance CORS validation.

tests/lib/test_output.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+
+from theHarvester.lib.output import print_linkedin_sections, sorted_unique
+
+
+def test_sorted_unique_sorts_and_deduplicates() -> None:
+    assert sorted_unique(["b", "a", "b"]) == ["a", "b"]
+
+
+def test_print_linkedin_sections_prints_links_when_present(capsys) -> None:
+    # Regression coverage: the CLI previously never printed LinkedIn links when the list was non-empty.
+    print_linkedin_sections(
+        engines=["linkedin"],
+        people=[],
+        links=["https://b.example", "https://a.example", "https://a.example"],
+    )
+
+    out = capsys.readouterr().out
+    assert "No LinkedIn users found" in out
+    assert "LinkedIn Links found: 3" in out
+    assert "https://a.example" in out
+    assert "https://b.example" in out
+
+
+def test_print_linkedin_sections_prints_people_and_links(capsys) -> None:
+    print_linkedin_sections(
+        engines=["rocketreach"],
+        people=["bob", "alice", "bob"],
+        links=["https://z.example", "https://z.example"],
+    )
+
+    out = capsys.readouterr().out
+    assert "LinkedIn Users found: 3" in out
+    assert "alice" in out
+    assert "bob" in out
+    assert "LinkedIn Links found: 2" in out
+    assert "https://z.example" in out

tests/test_security.py

@@ -36,7 +36,7 @@ class TestCORSConfiguration:
         allow_origins = options.get('allow_origins', [])
         allow_credentials = options.get('allow_credentials', False)
 
-        if '*' in allow_origins:
+        if isinstance(allow_origins, (list, tuple, set)) and '*' in allow_origins:
             assert (
                 allow_credentials is False
             ), 'CRITICAL: CORS must not allow credentials with wildcard origins (CVE risk)'

theHarvester/__main__.py

@@ -72,6 +72,7 @@ from theHarvester.discovery import (
 from theHarvester.discovery.constants import MissingKey
 from theHarvester.lib import hostchecker, stash
 from theHarvester.lib.core import DATA_DIR, Core, show_default_error_message
+from theHarvester.lib.output import print_linkedin_sections, print_section, sorted_unique
 from theHarvester.screenshot.screenshot import ScreenShotter
 
 if TYPE_CHECKING:
@@ -1330,43 +1331,26 @@ async def start(rest_args: argparse.Namespace | None = None):
 
     # Results
     if len(total_asns) > 0:
-        print(f'\n[*] ASNS found: {len(total_asns)}')
-        print('--------------------')
-        total_asns = list(sorted(set(total_asns)))
-        for asn in total_asns:
-            print(asn)
+        print_section(f'\n[*] ASNS found: {len(total_asns)}', total_asns, '--------------------')
+        total_asns = sorted_unique(total_asns)
 
     if len(interesting_urls) > 0:
-        print(f'\n[*] Interesting Urls found: {len(interesting_urls)}')
-        print('--------------------')
-        interesting_urls = list(sorted(set(interesting_urls)))
-        for iurl in interesting_urls:
-            print(iurl)
+        print_section(f'\n[*] Interesting Urls found: {len(interesting_urls)}', interesting_urls, '--------------------')
+        interesting_urls = sorted_unique(interesting_urls)
 
     if len(twitter_people_list_tracker) == 0 and 'twitter' in engines:
         print('\n[*] No Twitter users found.\n\n')
     elif len(twitter_people_list_tracker) >= 1:
-        print('\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker)))
-        print('---------------------')
-        twitter_people_list_tracker = list(sorted(set(twitter_people_list_tracker)))
-        for usr in twitter_people_list_tracker:
-            print(usr)
+        print_section(
+            '\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker)),
+            twitter_people_list_tracker,
+            '---------------------',
+        )
+        twitter_people_list_tracker = sorted_unique(twitter_people_list_tracker)
 
-    if len(linkedin_people_list_tracker) == 0 and 'linkedin' in engines:
-        print('\n[*] No LinkedIn users found.\n\n')
-    elif len(linkedin_people_list_tracker) >= 1:
-        print('\n[*] LinkedIn Users found: ' + str(len(linkedin_people_list_tracker)))
-        print('---------------------')
-        linkedin_people_list_tracker = list(sorted(set(linkedin_people_list_tracker)))
-        for usr in linkedin_people_list_tracker:
-            print(usr)
-
-    if len(linkedin_links_tracker) == 0 and ('linkedin' in engines or 'rocketreach' in engines):
-        print(f'\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}')
-        linkedin_links_tracker = list(sorted(set(linkedin_links_tracker)))
-        print('---------------------')
-        for link in linkedin_people_list_tracker:
-            print(link)
+    print_linkedin_sections(engines, linkedin_people_list_tracker, linkedin_links_tracker)
+    linkedin_people_list_tracker = sorted_unique(linkedin_people_list_tracker)
+    linkedin_links_tracker = sorted_unique(linkedin_links_tracker)
 
     length_urls = len(all_urls)
     if length_urls == 0:
@@ -1374,11 +1358,8 @@ async def start(rest_args: argparse.Namespace | None = None):
             print('\n[*] No Trello URLs found.')
     else:
         total = length_urls
-        print('\n[*] Trello URLs found: ' + str(total))
-        print('--------------------')
-        all_urls = list(sorted(set(all_urls)))
-        for url in sorted(all_urls):
-            print(url)
+        print_section('\n[*] Trello URLs found: ' + str(total), all_urls, '--------------------')
+        all_urls = sorted_unique(all_urls)
 
     if len(all_ip) == 0:
         print('\n[*] No IPs found.')
@@ -1847,7 +1828,6 @@ async def start(rest_args: argparse.Namespace | None = None):
         try:
             print('\n[*] Performing BuiltWith scan...')
             builtwith_scanner = builtwith.SearchBuiltWith(word)
-            # Use the process method according to the original structure of this module
             await builtwith_scanner.process(use_proxy)
 
             hosts = await builtwith_scanner.get_hostnames()

theHarvester/lib/api/api_example.py

@@ -7,6 +7,8 @@ import asyncio
 import aiohttp
 import netaddr
 
+from theHarvester.lib.output import print_section, sorted_unique
+
 
 async def fetch_json(session, url):
     try:
@@ -50,55 +52,46 @@ async def main() -> None:
         hosts = fetched_json.get('hosts', [])
 
     if len(total_asns) > 0:
-        print(f'\n[*] ASNS found: {len(total_asns)}')
-        print('--------------------')
-        total_asns = list(sorted(set(total_asns)))
-        for asn in total_asns:
-            print(asn)
+        print_section(f'\n[*] ASNS found: {len(total_asns)}', total_asns, '--------------------')
+        total_asns = sorted_unique(total_asns)
 
     if len(interesting_urls) > 0:
-        print(f'\n[*] Interesting Urls found: {len(interesting_urls)}')
-        print('--------------------')
-        interesting_urls = list(sorted(set(interesting_urls)))
-        for iurl in interesting_urls:
-            print(iurl)
+        print_section(f'\n[*] Interesting Urls found: {len(interesting_urls)}', interesting_urls, '--------------------')
+        interesting_urls = sorted_unique(interesting_urls)
 
     if len(twitter_people_list_tracker) == 0:
         print('\n[*] No Twitter users found.')
     elif len(twitter_people_list_tracker) >= 1:
-        print('\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker)))
-        print('---------------------')
-        twitter_people_list_tracker = list(sorted(set(twitter_people_list_tracker)))
-        for usr in twitter_people_list_tracker:
-            print(usr)
+        print_section(
+            '\n[*] Twitter Users found: ' + str(len(twitter_people_list_tracker)),
+            twitter_people_list_tracker,
+            '---------------------',
+        )
+        twitter_people_list_tracker = sorted_unique(twitter_people_list_tracker)
 
     if len(linkedin_people_list_tracker) == 0:
         print('\n[*] No LinkedIn users found.')
     elif len(linkedin_people_list_tracker) >= 1:
-        print('\n[*] LinkedIn Users found: ' + str(len(linkedin_people_list_tracker)))
-        print('---------------------')
-        linkedin_people_list_tracker = list(sorted(set(linkedin_people_list_tracker)))
-        for usr in linkedin_people_list_tracker:
-            print(usr)
+        print_section(
+            '\n[*] LinkedIn Users found: ' + str(len(linkedin_people_list_tracker)),
+            linkedin_people_list_tracker,
+            '---------------------',
+        )
+        linkedin_people_list_tracker = sorted_unique(linkedin_people_list_tracker)
 
     if len(linkedin_links_tracker) == 0:
         print('\n[*] No LinkedIn links found.')
     else:
-        print(f'\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}')
-        print('---------------------')
-        linkedin_links_tracker = list(sorted(set(linkedin_links_tracker)))
-        for link in linkedin_links_tracker:
-            print(link)
+        print_section(
+            f'\n[*] LinkedIn Links found: {len(linkedin_links_tracker)}', linkedin_links_tracker, '---------------------'
+        )
+        linkedin_links_tracker = sorted_unique(linkedin_links_tracker)
 
     length_urls = len(trello_urls)
     if length_urls == 0:
         print('\n[*] No Trello URLs found.')
     else:
-        print('\n[*] Trello URLs found: ' + str(length_urls))
-        print('--------------------')
-        all_urls = list(sorted(set(trello_urls)))
-        for url in sorted(all_urls):
-            print(url)
+        print_section('\n[*] Trello URLs found: ' + str(length_urls), trello_urls, '--------------------')
 
     if len(ips) == 0:
         print('\n[*] No IPs found.')
@@ -114,7 +107,7 @@ async def main() -> None:
     else:
         print('\n[*] Emails found: ' + str(len(emails)))
         print('----------------------')
-        all_emails = sorted(list(set(emails)))
+        all_emails = sorted_unique(emails)
         print('\n'.join(all_emails))
 
     if len(hosts) == 0:

theHarvester/lib/output.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from collections.abc import Hashable, Iterable, Sequence
+from typing import TypeVar
+
+T = TypeVar('T', bound=Hashable)
+
+
+def sorted_unique[T: Hashable](items: Iterable[T]) -> list[T]:
+    # `T` is only required to be hashable, not orderable.
+    # Sorting by `str` keeps output deterministic without requiring rich comparison support.
+    return list(sorted(set(items), key=str))
+
+
+def print_section(header: str, items: Iterable[str], separator: str) -> None:
+    print(header)
+    print(separator)
+    for item in sorted_unique(items):
+        print(item)
+
+
+def print_linkedin_sections(
+    engines: Sequence[str], people: Sequence[str], links: Sequence[str], separator: str = '---------------------'
+) -> None:
+    if len(people) == 0 and 'linkedin' in engines:
+        print('\n[*] No LinkedIn users found.\n\n')
+    elif len(people) >= 1:
+        print('\n[*] LinkedIn Users found: ' + str(len(people)))
+        print(separator)
+        for usr in sorted_unique(people):
+            print(usr)
+
+    if 'linkedin' in engines or 'rocketreach' in engines:
+        print(f'\n[*] LinkedIn Links found: {len(links)}')
+        print(separator)
+        for link in sorted_unique(links):
+            print(link)

theHarvester/parsers/myparser.py

@@ -110,7 +110,7 @@ class Parser:
         return sets
 
     async def urls(self) -> Set[str]:
-        found = re.finditer(r'(http|https)://(www\.)?trello.com/([a-zA-Z\d\-_\.]+/?)*', self.results)
+        found = re.finditer(r'(http|https)://(www\.)?trello.com/([a-zA-Z\d\-_.]+/?)*', self.results)
         urls = {match.group().strip() for match in found}
         return urls