From 3edfb411c39336a1a289db5db6c5e593717f7efc Mon Sep 17 00:00:00 2001 From: dasokkk <60160607+dasokkk@users.noreply.github.com> Date: Fri, 26 Jun 2026 23:32:36 +0300 Subject: [PATCH 1/2] Add unit tests for the crtsh discovery module Covers hostname extraction, wildcard prefix stripping, multi-line name_value splitting, numeric-prefix filtering, and the empty-response and missing-key error paths. AsyncFetcher is mocked so the tests run without network access. --- tests/discovery/test_crtsh.py | 80 +++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tests/discovery/test_crtsh.py diff --git a/tests/discovery/test_crtsh.py b/tests/discovery/test_crtsh.py new file mode 100644 index 00000000..efb9bd6a --- /dev/null +++ b/tests/discovery/test_crtsh.py @@ -0,0 +1,80 @@ +import pytest + +from theHarvester.discovery import crtsh + + +def _patch_fetch(monkeypatch, payload): + import theHarvester.lib.core as core_module + + async def fake_fetch_all(urls, headers=None, proxy=False, json=False): + return [payload] + + monkeypatch.setattr(core_module.AsyncFetcher, 'fetch_all', staticmethod(fake_fetch_all), raising=True) + + +class TestCrtshSearch: + def test_init_sets_state(self): + search = crtsh.SearchCrtsh('example.com') + assert search.word == 'example.com' + assert search.data == [] + assert search.proxy is False + + @pytest.mark.asyncio + async def test_process_collects_hostnames(self, monkeypatch): + _patch_fetch(monkeypatch, [{'name_value': 'www.example.com'}, {'name_value': 'mail.example.com'}]) + search = crtsh.SearchCrtsh('example.com') + await search.process(proxy=True) + assert search.proxy is True + assert set(await search.get_hostnames()) == {'www.example.com', 'mail.example.com'} + + @pytest.mark.asyncio + async def test_wildcard_prefix_is_stripped(self, monkeypatch): + _patch_fetch(monkeypatch, [{'name_value': '*.example.com'}]) + search = crtsh.SearchCrtsh('example.com') + await search.process() + assert 'example.com' in await search.get_hostnames() + + @pytest.mark.asyncio + async def test_multiline_name_value_is_split(self, monkeypatch): + # crt.sh packs several names into one name_value separated by newlines. + _patch_fetch(monkeypatch, [{'name_value': 'a.example.com\nb.example.com'}]) + search = crtsh.SearchCrtsh('example.com') + await search.process() + hostnames = await search.get_hostnames() + assert 'a.example.com' in hostnames + assert 'b.example.com' in hostnames + + @pytest.mark.asyncio + async def test_numeric_prefixed_entries_are_filtered(self, monkeypatch): + _patch_fetch(monkeypatch, [{'name_value': '1234.example.com'}, {'name_value': 'good.example.com'}]) + search = crtsh.SearchCrtsh('example.com') + await search.process() + hostnames = await search.get_hostnames() + assert 'good.example.com' in hostnames + assert '1234.example.com' not in hostnames + + @pytest.mark.asyncio + async def test_empty_response_returns_no_hostnames(self, monkeypatch): + import theHarvester.lib.core as core_module + + async def fake_fetch_all(urls, headers=None, proxy=False, json=False): + return [] + + monkeypatch.setattr(core_module.AsyncFetcher, 'fetch_all', staticmethod(fake_fetch_all), raising=True) + search = crtsh.SearchCrtsh('example.com') + await search.process() + assert await search.get_hostnames() == [] + + @pytest.mark.asyncio + async def test_missing_name_value_key_is_handled(self, monkeypatch): + _patch_fetch(monkeypatch, [{'issuer_ca_id': 1}]) + search = crtsh.SearchCrtsh('example.com') + await search.process() + assert await search.get_hostnames() == [] + + +class TestCrtshIntegration: + def test_supportedengines_lists_crtsh(self): + from theHarvester.lib.core import Core + + assert 'crtsh' in Core.get_supportedengines() From c94435166dcc760adabfab6da224981ccadb3ee6 Mon Sep 17 00:00:00 2001 From: dasokkk <60160607+dasokkk@users.noreply.github.com> Date: Sat, 27 Jun 2026 00:33:05 +0300 Subject: [PATCH 2/2] Compare crtsh test hostnames with set equality Replaces substring membership assertions with exact set comparisons so the tests no longer trip CodeQL's URL substring sanitization check. --- tests/discovery/test_crtsh.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/discovery/test_crtsh.py b/tests/discovery/test_crtsh.py index efb9bd6a..20010edd 100644 --- a/tests/discovery/test_crtsh.py +++ b/tests/discovery/test_crtsh.py @@ -32,7 +32,7 @@ class TestCrtshSearch: _patch_fetch(monkeypatch, [{'name_value': '*.example.com'}]) search = crtsh.SearchCrtsh('example.com') await search.process() - assert 'example.com' in await search.get_hostnames() + assert set(await search.get_hostnames()) == {'example.com'} @pytest.mark.asyncio async def test_multiline_name_value_is_split(self, monkeypatch): @@ -40,18 +40,15 @@ class TestCrtshSearch: _patch_fetch(monkeypatch, [{'name_value': 'a.example.com\nb.example.com'}]) search = crtsh.SearchCrtsh('example.com') await search.process() - hostnames = await search.get_hostnames() - assert 'a.example.com' in hostnames - assert 'b.example.com' in hostnames + assert set(await search.get_hostnames()) == {'a.example.com', 'b.example.com'} @pytest.mark.asyncio async def test_numeric_prefixed_entries_are_filtered(self, monkeypatch): _patch_fetch(monkeypatch, [{'name_value': '1234.example.com'}, {'name_value': 'good.example.com'}]) search = crtsh.SearchCrtsh('example.com') await search.process() - hostnames = await search.get_hostnames() - assert 'good.example.com' in hostnames - assert '1234.example.com' not in hostnames + # The numeric-prefixed entry is filtered out, leaving only the valid hostname. + assert set(await search.get_hostnames()) == {'good.example.com'} @pytest.mark.asyncio async def test_empty_response_returns_no_hostnames(self, monkeypatch):