diff --git a/CHANGELOG.md b/CHANGELOG.md index 58a7be66..a7f6b188 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removed the nonfunctional ThreatCrowd source because its service hostnames terminate at deleted AWS load balancers and return NXDOMAIN; OTX remains available through its separate adapter. ### Fixed +- Retained relevant GitLab project, profile, and website URLs in consolidated JSONL and SQLite results while excluding unrelated user URLs. - Removed BuiltWith's duplicate interesting-URL getter by allowing the shared collector to use either established getter spelling. - Made no-filename REST `/query` executions reach completed-result construction and SQLite persistence without changing the legacy response fields. - Made Chaos reject empty credentials, report HTTP and malformed-response failures, and preserve supported subdomain response shapes. diff --git a/README.md b/README.md index e7fc6742..c05c688b 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ Read the **API key** column as follows: | `fofa` | ✓ | No | ✓ | No | No | No | No | No | ✓ | | `fullhunt` | ✓ | No | No | No | No | No | No | No | ✓ | | `github-code` | ✓ | ✓ | No | No | No | No | No | No | ✓ | -| `gitlab` | ✓ | ✓ | No | No | No | No | No | No | No | +| `gitlab` | ✓ | ✓ | No | No | ✓ | No | No | No | No | | `hackertarget` | ✓ | No | No | No | No | No | No | No | Optional | | `haveibeenpwned` | No | No | No | No | No | No | ✓ | `POST /additional/breaches` response | No | | `hibpverified` | No | ✓ | No | No | No | No | ✓ | No | ✓ | diff --git a/tests/discovery/test_gitlabsearch.py b/tests/discovery/test_gitlabsearch.py index 593227a5..25b9d4ab 100644 --- a/tests/discovery/test_gitlabsearch.py +++ b/tests/discovery/test_gitlabsearch.py @@ -1,9 +1,13 @@ import json +import sys +from pathlib import Path from typing import Any import pytest from theHarvester.discovery import gitlabsearch +from theHarvester import __main__ as theharvester_main +from theHarvester.lib.completed_result import CompletedResult @pytest.mark.asyncio @@ -83,6 +87,11 @@ async def test_public_discovery_normalizes_evidence_and_uses_bounded_requests( 'status.example.test', } assert await search.get_emails() == {'admin@example.test', 'security@example.test'} + assert await search.get_urls() == { + 'https://gitlab.com/example', + 'https://gitlab.com/group/project', + 'https://Portal.Example.TEST./profile', + } @pytest.mark.asyncio @@ -118,3 +127,50 @@ async def test_decoded_pages_are_accepted_without_silent_slicing(monkeypatch: py assert 'project-21.example.test' in hostnames assert len(emails) == 11 assert 'user-11@example.test' in emails + + +@pytest.mark.asyncio +async def test_gitlab_urls_reach_completed_jsonl( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + completed_results: list[CompletedResult] = [] + + class FakeStash: + async def do_init(self) -> None: + return None + + async def store_all(self, *_args: object) -> None: + return None + + async def store_completed_result(self, result: CompletedResult) -> None: + completed_results.append(result) + + class FakeGitlab: + def __init__(self, domain: str) -> None: + assert domain == 'example.test' + + async def process(self, _proxy: bool) -> None: + return None + + async def get_hostnames(self) -> set[str]: + return set() + + async def get_emails(self) -> set[str]: + return set() + + async def get_urls(self) -> set[str]: + return {'https://gitlab.com/group/project'} + + report = tmp_path / 'gitlab-report' + monkeypatch.setattr(theharvester_main.stash, 'StashManager', FakeStash) + monkeypatch.setattr(theharvester_main.gitlabsearch, 'SearchGitlab', FakeGitlab) + monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.test', '-b', 'gitlab', '-f', str(report)]) + + with pytest.raises(SystemExit) as exit_info: + await theharvester_main.start() + + assert exit_info.value.code == 0 + assert completed_results[0].results == (('url', 'https://gitlab.com/group/project'),) + records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()] + assert {'type': 'url', 'value': 'https://gitlab.com/group/project'} in records diff --git a/tests/lib/test_core.py b/tests/lib/test_core.py index 7285f034..3633391a 100644 --- a/tests/lib/test_core.py +++ b/tests/lib/test_core.py @@ -49,6 +49,7 @@ def test_capabilities_and_explicit_sources_form_a_union() -> None: "bevigil", "builtwith", "certspotter", + "gitlab", "intelx", "rocketreach", "urlscan", diff --git a/tests/lib/test_source_catalog.py b/tests/lib/test_source_catalog.py index aaf8f7c8..7fe49e8b 100644 --- a/tests/lib/test_source_catalog.py +++ b/tests/lib/test_source_catalog.py @@ -55,7 +55,9 @@ def test_subdomain_route_drives_subdomain_capability() -> None: def test_source_specs_describe_consolidated_routes_not_getter_presence() -> None: - assert SOURCE_SPECS['gitlab'].routes == frozenset({ResultRoute.SUBDOMAINS, ResultRoute.EMAILS}) + assert SOURCE_SPECS['gitlab'].routes == frozenset( + {ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.URLS} + ) assert SOURCE_SPECS['haveibeenpwned'].routes == frozenset({ResultRoute.BREACHES}) assert SOURCE_SPECS['hibpverified'].routes == frozenset({ResultRoute.EMAILS, ResultRoute.BREACHES}) assert SOURCE_SPECS['urlscan'].routes == frozenset( diff --git a/tests/test_all_source_orchestration.py b/tests/test_all_source_orchestration.py index 1a9ef018..6297c161 100644 --- a/tests/test_all_source_orchestration.py +++ b/tests/test_all_source_orchestration.py @@ -169,6 +169,9 @@ async def test_all_schedules_each_passive_catalog_source_once_and_reports_result async def get_links(self) -> set[str]: return {'https://sub.example.test/profile'} + async def get_urls(self) -> set[str]: + return {'https://gitlab.com/example/project'} + async def get_interestingurls(self) -> set[str]: return {'https://sub.example.test/evidence'} @@ -229,6 +232,7 @@ async def test_all_schedules_each_passive_catalog_source_once_and_reports_result jsonl_records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()] assert {'type': 'hostname', 'value': 'sub.example.test'} in jsonl_records assert {'type': 'email', 'value': 'user@example.test'} in jsonl_records + assert {'type': 'url', 'value': 'https://gitlab.com/example/project'} in jsonl_records completed = await TestStash().load_completed_result(UUID(jsonl_records[0]['run_id'])) assert completed.target == 'example.test' diff --git a/tests/test_readme.py b/tests/test_readme.py index 2c5fe12d..edc8699c 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -14,6 +14,7 @@ ROUTE_COLUMNS = { ResultRoute.IPS: 'IPs', ResultRoute.ASNS: 'ASNs', ResultRoute.LINKS: 'URLs / links', + ResultRoute.URLS: 'URLs / links', ResultRoute.INTERESTING_URLS: 'URLs / links', ResultRoute.PEOPLE: 'People', ResultRoute.BREACHES: 'Breaches', diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 6ec73d0a..d0b2ef6c 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -442,6 +442,12 @@ async def start( if len(links) > 0: await db.store_all(word, links, 'linkedinlinks', source) + if ResultRoute.URLS in routes: + urls = await search_engine.get_urls() + all_urls.extend(urls) + if len(urls) > 0: + await db_stash.store_all(word, urls, 'url', source) + if ResultRoute.INTERESTING_URLS in routes: get_interesting_urls = getattr(search_engine, 'get_interesting_urls', None) iurls = await get_interesting_urls() if get_interesting_urls else await search_engine.get_interestingurls() @@ -1471,7 +1477,7 @@ async def start( output_logger.info('\n[*] No Trello URLs found.') else: total = length_urls - print_section('\n[*] Trello URLs found: ' + str(total), all_urls, '--------------------') + print_section('\n[*] URLs found: ' + str(total), all_urls, '--------------------') all_urls = sorted_unique(all_urls) if len(all_ip) == 0: diff --git a/theHarvester/discovery/gitlabsearch.py b/theHarvester/discovery/gitlabsearch.py index f0f141e7..c533ad11 100644 --- a/theHarvester/discovery/gitlabsearch.py +++ b/theHarvester/discovery/gitlabsearch.py @@ -68,6 +68,13 @@ class SearchGitlab: return emails + def _add_text_evidence(self, text: str) -> bool: + hosts = self._extract_domains_from_text(text) + emails = self._extract_emails_from_text(text) + self.totalhosts.update(hosts) + self.totalemails.update(emails) + return bool(hosts or emails) + async def search_projects(self) -> None: """Search GitLab projects for domain references""" try: @@ -101,12 +108,7 @@ class SearchGitlab: # Look for domains in description and name all_text = f'{description} {name} {path}' - self.totalhosts.update(self._extract_domains_from_text(all_text)) - self.totalemails.update(self._extract_emails_from_text(all_text)) - - # Add the web URL if it contains our domain - if web_url and self.word in web_url: - self.totalurls.add(web_url) + project_is_relevant = self._add_text_evidence(all_text) # Try to get README content for more detailed search project_id = project.get('id') @@ -122,11 +124,13 @@ class SearchGitlab: readme_text = ( readme_response[0] if isinstance(readme_response[0], str) else str(readme_response[0]) ) - self.totalhosts.update(self._extract_domains_from_text(readme_text)) - self.totalemails.update(self._extract_emails_from_text(readme_text)) + project_is_relevant = self._add_text_evidence(readme_text) or project_is_relevant except Exception: pass # README might not exist or be accessible + if project_is_relevant and isinstance(web_url, str) and web_url.strip(): + self.totalurls.add(web_url.strip()) + except Exception as e: logger.info(f'Failed to parse GitLab projects response: {e}') @@ -163,20 +167,23 @@ class SearchGitlab: public_email = user.get('public_email', '') or '' # Look for domains in user info - all_text = f'{name} {username} {bio} {website_url}' - self.totalhosts.update(self._extract_domains_from_text(all_text)) + user_hosts = self._extract_domains_from_text(f'{name} {username} {bio}') + website_hosts = self._extract_domains_from_text(website_url) if isinstance(website_url, str) else set() + user_hosts.update(website_hosts) + self.totalhosts.update(user_hosts) # Check email + user_emails: set[str] = set() if public_email: - self.totalemails.update(self._extract_emails_from_text(public_email)) + user_emails = self._extract_emails_from_text(public_email) + self.totalemails.update(user_emails) - # Check website URL - if website_url and self.word in website_url: - self.totalurls.add(website_url) + user_is_relevant = bool(user_hosts or user_emails) + if website_hosts and isinstance(website_url, str) and website_url.strip(): + self.totalurls.add(website_url.strip()) - # Add user profile URL if relevant - if web_url: - self.totalurls.add(web_url) + if user_is_relevant and isinstance(web_url, str) and web_url.strip(): + self.totalurls.add(web_url.strip()) except Exception as e: logger.info(f'Failed to parse GitLab users response: {e}') diff --git a/theHarvester/lib/api/api.py b/theHarvester/lib/api/api.py index 2a7bcf04..8c98af49 100644 --- a/theHarvester/lib/api/api.py +++ b/theHarvester/lib/api/api.py @@ -48,7 +48,7 @@ class QueryResponse(BaseModel): twitter_people: list[str] = Field(default_factory=list, description='List of Twitter people') linkedin_people: list[dict] = Field(default_factory=list, description='List of LinkedIn people') linkedin_links: list[str] = Field(default_factory=list, description='List of LinkedIn links') - trello_urls: list[str] = Field(default_factory=list, description='List of Trello URLs') + trello_urls: list[str] = Field(default_factory=list, description='List of discovered URLs (legacy field name)') ips: list[str] = Field(default_factory=list, description='List of IPs') emails: list[str] = Field(default_factory=list, description='List of emails') hosts: list[str] = Field(default_factory=list, description='List of hosts') diff --git a/theHarvester/lib/source_catalog.py b/theHarvester/lib/source_catalog.py index 0db214c3..4d63da55 100644 --- a/theHarvester/lib/source_catalog.py +++ b/theHarvester/lib/source_catalog.py @@ -22,6 +22,7 @@ class ResultRoute(Enum): ASNS = auto() PEOPLE = auto() LINKS = auto() + URLS = auto() INTERESTING_URLS = auto() BREACHES = auto() @@ -33,6 +34,7 @@ _ROUTE_CAPABILITIES = { ResultRoute.ASNS: 'asns', ResultRoute.PEOPLE: 'people', ResultRoute.LINKS: 'urls', + ResultRoute.URLS: 'urls', ResultRoute.INTERESTING_URLS: 'urls', ResultRoute.BREACHES: 'breaches', } @@ -88,7 +90,7 @@ _SPECS = ( _spec('fofa', ResultRoute.SUBDOMAINS, ResultRoute.IPS), _spec('fullhunt', ResultRoute.SUBDOMAINS), _spec('github-code', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), - _spec('gitlab', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS), + _spec('gitlab', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.URLS), _spec('hackertarget', ResultRoute.SUBDOMAINS), _spec('haveibeenpwned', ResultRoute.BREACHES), _spec('hibpverified', ResultRoute.EMAILS, ResultRoute.BREACHES),