Fixes all over the place

tests/discovery/test_api_endpoints.py

@@ -1276,3 +1276,10 @@ async def test_detect_schema_reuses_session_with_configured_request_policy(monke
             {'ssl': True, 'allow_redirects': False},
         )
     ]
+
+
+def test_default_wordlist_is_the_packaged_api_wordlist() -> None:
+    search = api_endpoints.SearchApiEndpoints('example.com')
+
+    assert Path(search.wordlist).is_file()
+    assert search._load_wordlist()

tests/discovery/test_arquivo.py

@@ -17,7 +17,8 @@ async def test_process_collects_scoped_hosts_from_one_cdx_request(monkeypatch: p
     async def fake_fetch_all(urls: list[str], **kwargs: Any) -> list[FetcherResponse]:
         requested_urls.extend(urls)
         assert kwargs['include_metadata'] is True
-        assert kwargs['headers']['User-agent']
+        assert 'headers' not in kwargs
+        assert kwargs['session'].headers['User-agent']
         return [
             FetcherResponse(
                 body='\n'.join(

tests/discovery/test_certspotter.py

@@ -196,9 +196,10 @@ class TestCertspotterSearch:
         [
             ([], 'no-response'),
             ([FetcherResponse('not a list', 200, {})], 'invalid-response'),
+            ([None], 'transport-error'),
         ],
     )
-    async def test_search_reports_invalid_response_as_incomplete(
+    async def test_search_reports_first_page_failure_as_failed(
         self,
         monkeypatch: pytest.MonkeyPatch,
         caplog: pytest.LogCaptureFixture,
@@ -214,7 +215,7 @@ class TestCertspotterSearch:
             report = await search.process()
 
         assert await search.get_hostnames() == set()
-        assert report.status == 'partial'
+        assert report.status == 'failed'
         assert report.stop_reason == stop_reason
         assert 'results may be incomplete' in caplog.text
 

tests/discovery/test_githubcode_contract.py

@@ -337,6 +337,36 @@ async def test_github_code_forbidden_fails_immediately_as_access_denied(install_
     assert report == githubcode.SourceExecutionReport('failed', 'access-denied')
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize('headers', [{'x-ratelimit-remaining': '0'}, {'retry-after': '60'}])
+async def test_github_code_rate_limit_forbidden_retries_then_succeeds(
+    install_github_responses,
+    monkeypatch: pytest.MonkeyPatch,
+    headers: dict[str, str],
+) -> None:
+    class RateLimitedResponse(FakeResponse):
+        status = 403
+
+    limited = RateLimitedResponse({}, {})
+    limited.headers = headers
+    requested_urls = install_github_responses(
+        limited,
+        FakeResponse({'items': [{'text_matches': [{'fragment': 'api.example.com'}]}]}, {}),
+    )
+
+    async def no_sleep(_delay: float) -> None:
+        return None
+
+    monkeypatch.setattr(githubcode.asyncio, 'sleep', no_sleep)
+    search = githubcode.SearchGithubCode('example.com', limit=None)
+
+    report = await search.process()
+
+    assert len(requested_urls) == 2
+    assert report is None
+    assert search.counter == 1
+
+
 @pytest.mark.asyncio
 async def test_github_code_rate_limited_retries_then_reports(
     install_github_responses,

tests/discovery/test_tombasearch.py

@@ -343,6 +343,7 @@ async def test_free_tomba_search_rejects_out_of_scope_source_websites(monkeypatc
                             'sources': [
                                 {'website_url': 'api.example.test'},
                                 {'website_url': 'https://portal.example.test'},
+                                {'website_url': 'Docs.Example.Test/contact'},
                                 {'website_url': 'notexample.test'},
                                 {'website_url': 'example.test.evil.net'},
                             ],
@@ -364,7 +365,7 @@ async def test_free_tomba_search_rejects_out_of_scope_source_websites(monkeypatc
     await search.process()
 
     assert await search.get_emails() == ['alice@example.test']
-    assert set(await search.get_hostnames()) == {'api.example.test', 'portal.example.test'}
+    assert set(await search.get_hostnames()) == {'api.example.test', 'portal.example.test', 'docs.example.test'}
 
 
 @pytest.mark.asyncio

tests/discovery/test_windvane.py

@@ -42,9 +42,9 @@ async def test_authenticated_results_are_normalized_and_scoped(monkeypatch) -> N
         },
     }
 
-    async def fake_post_fetch(url, headers=None, data=None, session=None):
+    async def fake_post_fetch(url, headers=None, json_body=None, session=None):
         endpoint = url.rsplit('/', 1)[-1]
-        page = json.loads(data)['page_request']['page']
+        page = json_body['page_request']['page']
         assert headers['X-Api-Key'] == 'test-key'
         return json.dumps(responses[(endpoint, page)])
 
@@ -87,9 +87,9 @@ async def test_authenticated_unlimited_search_follows_all_endpoint_pagination(mo
     monkeypatch.setattr(windvane.Core, 'windvane_key', lambda: 'test-key')
     requests: list[tuple[str, int, int]] = []
 
-    async def fake_post_fetch(url, headers=None, data=None, session=None):
+    async def fake_post_fetch(url, headers=None, json_body=None, session=None):
         endpoint = url.rsplit('/', 1)[-1]
-        page_request = json.loads(data)['page_request']
+        page_request = json_body['page_request']
         page = page_request['page']
         requests.append((endpoint, page, page_request['count']))
         last_pages = {'ListSubDomain': 4, 'ListDNS': 3, 'ListEmail': 2}
@@ -172,9 +172,9 @@ async def test_finite_limit_stops_each_windvane_endpoint(monkeypatch) -> None:
     monkeypatch.setattr(windvane.Core, 'windvane_key', lambda: 'test-key')
     requests: list[tuple[str, int, int]] = []
 
-    async def fake_post_fetch(url, headers=None, data=None, session=None):
+    async def fake_post_fetch(url, headers=None, json_body=None, session=None):
         endpoint = url.rsplit('/', 1)[-1]
-        page_request = json.loads(data)['page_request']
+        page_request = json_body['page_request']
         requests.append((endpoint, page_request['page'], page_request['count']))
         return json.dumps({'code': 0, 'data': {'list': [{}], 'has_more': True}})
 

tests/lib/test_recursive_dns.py

@@ -138,6 +138,23 @@ async def test_recursive_dns_skips_candidates_rejected_by_validation() -> None:
     assert result.stop_reason == 'depth-limit'
 
 
+@pytest.mark.asyncio
+async def test_recursive_dns_invalid_labels_do_not_count_as_zero_yield_batches() -> None:
+    current = {'example.com', 'api.example.com'}
+    resolvers = tuple(FakeResolver(f'resolver-{index}', current) for index in range(3))
+
+    result = await discover_recursive_dns(
+        'example.com',
+        ('example.com',),
+        (*(f'_invalid{index}' for index in range(50)), 'api'),
+        resolvers,
+        RecursiveDNSLimits(depth=1, query_limit=1_000, runtime_seconds=5),
+    )
+
+    assert [finding.hostname for finding in result.findings] == ['api.example.com']
+    assert result.zero_yield_batches == 0
+
+
 @pytest.mark.asyncio
 async def test_recursive_dns_advances_only_current_candidates_breadth_first() -> None:
     current = {'api.example.com', 'dev.api.example.com', 'v2.dev.api.example.com'}

tests/lib/test_run_backend.py

@@ -613,6 +613,49 @@ def test_orphan_recovery_survives_unreadable_evidence_for_one_run(tmp_path, monk
     assert healthy['results'] == [{'type': 'email', 'value': 'saved@second.example', 'sources': [], 'actions': []}]
 
 
+def test_orphan_recovery_storage_failure_keeps_run_recoverable(tmp_path, monkeypatch) -> None:
+    from sqlalchemy.exc import OperationalError
+
+    from theHarvester.lib.api.run_artifacts import ensure_private_directory, write_child_evidence
+    from theHarvester.lib.api.run_models import RunRequest
+    from theHarvester.lib.api.run_store import RunStore
+    from theHarvester.lib.completed_result import CompletedResult
+
+    monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite'))
+    monkeypatch.setenv('THEHARVESTER_RUN_ARTIFACTS', str(tmp_path / 'artifacts'))
+
+    async def locked_save(_completed):
+        raise OperationalError('INSERT', {}, Exception('database is locked'))
+
+    async def scenario():
+        store = RunStore()
+        run = await store.create(RunRequest(target='locked.example', sources=['crtsh']))
+        assert await store.claim_next() is not None
+        now = datetime.now(UTC)
+        artifact_dir = store.artifact_directory(run['run_id'])
+        ensure_private_directory(artifact_dir)
+        write_child_evidence(
+            artifact_dir,
+            CompletedResult.finish(
+                target='locked.example',
+                started_at=now,
+                completed_at=now,
+                groups={'email': ['saved@locked.example']},
+            ),
+            partial=True,
+        )
+        monkeypatch.setattr(store.results, 'save_run', locked_save)
+
+        with pytest.raises(OperationalError):
+            await store.recover_orphans()
+        return await store.get(run['run_id'])
+
+    recovered = asyncio.run(scenario())
+
+    assert recovered is not None
+    assert recovered['status'] == 'running'
+
+
 def test_orphan_recovery_prefers_immutable_persisted_evidence_over_a_newer_checkpoint(tmp_path, monkeypatch) -> None:
     from theHarvester.lib.api.run_artifacts import ensure_private_directory, write_child_evidence
     from theHarvester.lib.api.run_models import RunRequest

theHarvester/discovery/api_endpoints.py

@@ -9,14 +9,13 @@ import re
 from dataclasses import asdict, dataclass, field
 from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
-from pathlib import Path
 from typing import Any
 from urllib.parse import urljoin, urlparse
 
 import aiohttp
 
 from theHarvester.lib.cancellation import drain_tasks_after_cancellation
-from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse, ResponseStreamError
+from theHarvester.lib.core import DATA_DIR, AsyncFetcher, Core, FetcherResponse, ResponseStreamError
 
 logger = logging.getLogger(__name__)
 _DIAGNOSTIC_RESPONSE_HEADERS = {
@@ -151,7 +150,7 @@ class SearchApiEndpoints:
         self.stop_reason: str | None = None
 
         # Set default wordlist path
-        default_wordlist = Path(__file__).resolve().parents[2] / 'wordlists' / 'api_endpoints.txt'
+        default_wordlist = DATA_DIR / 'wordlists' / 'api_endpoints.txt'
         self.wordlist = wordlist or str(default_wordlist)
         self.exact_paths = exact_paths
 

theHarvester/discovery/arquivo.py

@@ -22,10 +22,10 @@ class SearchArquivo:
     async def process(self, proxy: bool = False) -> SourceExecutionReport | None:
         self.proxy = proxy
         headers = {'User-agent': Core.get_user_agent()}
-        async with AsyncFetcher.open_session(headers=headers, proxy=self.proxy) as session:
-            return await self._search(session, headers)
+        async with AsyncFetcher.open_session(headers=headers, proxy=self.proxy, request_timeout=60) as session:
+            return await self._search(session)
 
-    async def _search(self, session, headers: dict[str, str]) -> SourceExecutionReport | None:
+    async def _search(self, session) -> SourceExecutionReport | None:
         offset = 0
         previous_page = None
         report = None
@@ -44,7 +44,6 @@ class SearchArquivo:
             try:
                 responses: list[FetcherResponse | None] = await AsyncFetcher.fetch_all(
                     [f'https://arquivo.pt/wayback/cdx?{query}'],
-                    headers=headers,
                     session=session,
                     include_metadata=True,
                 )

theHarvester/discovery/certspottersearch.py

@@ -140,7 +140,9 @@ class SearchCertspoter:
     async def process(self, proxy: bool = False) -> SourceExecutionReport | None:
         self.proxy = proxy
         self._report = None
-        async with AsyncFetcher.open_session(proxy=self.proxy) as session:
+        async with AsyncFetcher.open_session(proxy=self.proxy, request_timeout=60) as session:
             await self.do_search(session)
         logger.info('\tSearching results.')
+        if self._report is not None and self._report.status == 'partial' and not self.totalhosts:
+            return SourceExecutionReport('failed', self._report.stop_reason)
         return self._report

theHarvester/discovery/commoncrawl.py

@@ -111,10 +111,7 @@ class SearchCommoncrawl:
             if self.limit == 0:
                 return None
 
-            headers = {'User-agent': Core.get_user_agent()}
-            catalog_response = await AsyncFetcher.fetch_all(
-                [f'{self.hostname}/collinfo.json'], headers=headers, session=session, json=True
-            )
+            catalog_response = await AsyncFetcher.fetch_all([f'{self.hostname}/collinfo.json'], session=session, json=True)
             if not catalog_response or not isinstance(catalog_response[0], list) or not catalog_response[0]:
                 logger.error('Common Crawl API error: invalid index catalog')
                 return SourceExecutionReport('failed', 'invalid-catalog')
@@ -149,7 +146,7 @@ class SearchCommoncrawl:
                     try:
                         query_had_errors = False
                         count_url = f'{endpoint}?{urlencode({"url": query, "output": "json", "pageSize": self.PAGE_SIZE, "showNumPages": "true"})}'
-                        count_response = await AsyncFetcher.fetch_all([count_url], headers=headers, session=session)
+                        count_response = await AsyncFetcher.fetch_all([count_url], session=session)
                         count_payload = json.loads(count_response[0])
                         page_count = count_payload.get('pages') if isinstance(count_payload, dict) else None
                         if isinstance(page_count, bool) or not isinstance(page_count, int) or page_count < 0:
@@ -165,7 +162,7 @@ class SearchCommoncrawl:
                                 return None
                             page_url = f'{endpoint}?{urlencode({"url": query, "output": "json", "pageSize": self.PAGE_SIZE, "page": first_page, "limit": min(remaining, self.MAX_RECORDS_PER_REQUEST)})}'
                             first_page += 1
-                            responses = await AsyncFetcher.fetch_all([page_url], headers=headers, session=session)
+                            responses = await AsyncFetcher.fetch_all([page_url], session=session)
                             if not isinstance(responses, list) or not responses:
                                 raise ValueError('invalid page response')
                             try:
@@ -217,7 +214,9 @@ class SearchCommoncrawl:
         self.proxy = proxy
         try:
             async with (
-                AsyncFetcher.open_session(headers={'User-agent': Core.get_user_agent()}, proxy=self.proxy) as session,
+                AsyncFetcher.open_session(
+                    headers={'User-agent': Core.get_user_agent()}, proxy=self.proxy, request_timeout=60
+                ) as session,
                 asyncio.timeout(self.RUNTIME_SECONDS),
             ):
                 return await self.do_search(session)

theHarvester/discovery/githubcode.py

@@ -104,6 +104,11 @@ class SearchGithubCode:
             logger.info(f'Error handling response: {e}')
             return ErrorResult(500)
 
+    @staticmethod
+    def _is_rate_limited(status: int, headers: dict[str, str]) -> bool:
+        # GitHub reports primary and secondary rate limits as 403 with these headers.
+        return status == 403 and (headers.get('x-ratelimit-remaining') == '0' or 'retry-after' in headers)
+
     async def do_search(
         self,
         page: int,
@@ -116,7 +121,8 @@ class SearchGithubCode:
             url = f'{self.base_url}&page={page}' if page else self.base_url
             response = await AsyncFetcher.fetch_json(url, session=session)
             body = response.body if isinstance(response.body, dict) else {}
-            return '', body, response.status, response.links
+            status = 429 if self._is_rate_limited(response.status, response.headers) else response.status
+            return '', body, status, response.links
         except ResponseStreamError:
             raise
         except Exception as e:

theHarvester/discovery/gitlabsearch.py

@@ -258,5 +258,5 @@ class SearchGitlab:
 
     async def process(self, proxy: bool = False) -> SourceExecutionReport | None:
         self.proxy = proxy
-        async with AsyncFetcher.open_session(proxy=self.proxy) as session:
+        async with AsyncFetcher.open_session(proxy=self.proxy, request_timeout=60) as session:
             return await self.do_search(session)

theHarvester/discovery/huntersearch.py

@@ -128,7 +128,7 @@ class SearchHunter:
     async def process(self, proxy: bool = False) -> SourceExecutionReport | None:
         self.proxy = proxy
         try:
-            async with AsyncFetcher.open_session(proxy=self.proxy) as session:
+            async with AsyncFetcher.open_session(proxy=self.proxy, request_timeout=60) as session:
                 return await self.do_search(session)  # Only need to do it once.
         except AttributeError, KeyError, TypeError:
             logger.info('Hunter returned malformed data')

theHarvester/discovery/mojeek.py

@@ -186,7 +186,7 @@ class SearchMojeek:
 
     async def process(self, proxy: bool = False) -> SourceExecutionReport | None:
         self.proxy = proxy
-        async with AsyncFetcher.open_session(proxy=self.proxy) as session:
+        async with AsyncFetcher.open_session(proxy=self.proxy, request_timeout=60) as session:
             return await self.do_search(session)
 
     async def get_emails(self) -> list[str]:

theHarvester/discovery/tombasearch.py

@@ -114,12 +114,12 @@ class SearchTomba:
         if not isinstance(value, str) or not value.strip():
             return None
         candidate = value.strip()
-        if '://' in candidate:
-            try:
-                candidate = urlsplit(candidate).hostname or ''
-            except ValueError:
-                return None
-        return normalize_scoped_hostname(candidate, self.word)
+        try:
+            # A network-path prefix lets urlsplit find the host in scheme-less values.
+            hostname = urlsplit(candidate if '://' in candidate else f'//{candidate}').hostname or ''
+        except ValueError:
+            return None
+        return normalize_scoped_hostname(hostname, self.word)
 
     async def parse_resp(self, json_resp: dict) -> tuple[list[str], list[str]]:
         emails = list(sorted({email['email'] for email in json_resp['data']['emails']}))
@@ -138,7 +138,7 @@ class SearchTomba:
     async def process(self, proxy: bool = False) -> SourceExecutionReport | None:
         self.proxy = proxy
         try:
-            async with AsyncFetcher.open_session(proxy=self.proxy) as session:
+            async with AsyncFetcher.open_session(proxy=self.proxy, request_timeout=60) as session:
                 return await self.do_search(session)  # Only need to do it once.
         except AttributeError, KeyError, TypeError:
             logger.info('Tomba returned malformed data')

theHarvester/discovery/windvane.py

@@ -115,7 +115,7 @@ class SearchWindvane:
             response = await AsyncFetcher.post_fetch(
                 url,
                 headers=headers,
-                data=json.dumps(request_data, separators=(',', ':')),
+                json_body=request_data,
                 session=session,
             )
             if not response:

theHarvester/lib/api/run_store.py

@@ -457,9 +457,10 @@ class RunStore:
                     evidence,
                     recovered_at,
                 )
-            except Exception as evidence_failure:
+            except (HTTPException, KeyError, TypeError, ValueError, ResultStoreError) as evidence_failure:
                 # One unreadable checkpoint must not abort recovery of the
                 # remaining orphaned runs or crash every future startup.
+                # Storage failures propagate so the run stays recoverable.
                 logger.warning(
                     'Run %s has unreadable evidence and will be failed without it: %s',
                     run_id,

theHarvester/lib/recursive_dns.py

@@ -127,7 +127,9 @@ async def discover_recursive_dns(
                     stop_reason = 'runtime-limit'
                     return
                 candidate = f'{label}.{parent}'
-                if candidate in seen:
+                if candidate in seen or normalize_scoped_hostname(candidate, normalized_target) is None:
+                    # A label such as _dmarc passes _normalize_label but is not a
+                    # valid scoped hostname; validate_dns_candidates would drop it.
                     continue
                 seen.add(candidate)
                 yield candidate, parent
@@ -197,11 +199,6 @@ async def discover_recursive_dns(
         while batch := tuple(islice(candidates, 50)):
             batch_yield = 0
             for candidate, parent in batch:
-                if normalize_scoped_hostname(candidate, normalized_target) is None:
-                    # A label such as _dmarc passes _normalize_label but is not a
-                    # valid scoped hostname; validate_dns_candidates would drop it,
-                    # so skip it here instead of aborting the whole action.
-                    continue
                 consensus = await validate(candidate)
                 if consensus is None:
                     stopped = True