Retain GitLab evidence URLs (#2492)

CHANGELOG.md

@@ -42,6 +42,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Removed the nonfunctional ThreatCrowd source because its service hostnames terminate at deleted AWS load balancers and return NXDOMAIN; OTX remains available through its separate adapter.
 
 ### Fixed
+- Retained relevant GitLab project, profile, and website URLs in consolidated JSONL and SQLite results while excluding unrelated user URLs.
 - Removed BuiltWith's duplicate interesting-URL getter by allowing the shared collector to use either established getter spelling.
 - Made no-filename REST `/query` executions reach completed-result construction and SQLite persistence without changing the legacy response fields.
 - Made Chaos reject empty credentials, report HTTP and malformed-response failures, and preserve supported subdomain response shapes.

README.md

@@ -152,7 +152,7 @@ Read the **API key** column as follows:
 | `fofa` | ✓ | No | ✓ | No | No | No | No | No | ✓ |
 | `fullhunt` | ✓ | No | No | No | No | No | No | No | ✓ |
 | `github-code` | ✓ | ✓ | No | No | No | No | No | No | ✓ |
-| `gitlab` | ✓ | ✓ | No | No | No | No | No | No | No |
+| `gitlab` | ✓ | ✓ | No | No | ✓ | No | No | No | No |
 | `hackertarget` | ✓ | No | No | No | No | No | No | No | Optional |
 | `haveibeenpwned` | No | No | No | No | No | No | ✓ | `POST /additional/breaches` response | No |
 | `hibpverified` | No | ✓ | No | No | No | No | ✓ | No | ✓ |

tests/discovery/test_gitlabsearch.py

@@ -1,9 +1,13 @@
 import json
+import sys
+from pathlib import Path
 from typing import Any
 
 import pytest
 
 from theHarvester.discovery import gitlabsearch
+from theHarvester import __main__ as theharvester_main
+from theHarvester.lib.completed_result import CompletedResult
 
 
 @pytest.mark.asyncio
@@ -83,6 +87,11 @@ async def test_public_discovery_normalizes_evidence_and_uses_bounded_requests(
         'status.example.test',
     }
     assert await search.get_emails() == {'admin@example.test', 'security@example.test'}
+    assert await search.get_urls() == {
+        'https://gitlab.com/example',
+        'https://gitlab.com/group/project',
+        'https://Portal.Example.TEST./profile',
+    }
 
 
 @pytest.mark.asyncio
@@ -118,3 +127,50 @@ async def test_decoded_pages_are_accepted_without_silent_slicing(monkeypatch: py
     assert 'project-21.example.test' in hostnames
     assert len(emails) == 11
     assert 'user-11@example.test' in emails
+
+
+@pytest.mark.asyncio
+async def test_gitlab_urls_reach_completed_jsonl(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    completed_results: list[CompletedResult] = []
+
+    class FakeStash:
+        async def do_init(self) -> None:
+            return None
+
+        async def store_all(self, *_args: object) -> None:
+            return None
+
+        async def store_completed_result(self, result: CompletedResult) -> None:
+            completed_results.append(result)
+
+    class FakeGitlab:
+        def __init__(self, domain: str) -> None:
+            assert domain == 'example.test'
+
+        async def process(self, _proxy: bool) -> None:
+            return None
+
+        async def get_hostnames(self) -> set[str]:
+            return set()
+
+        async def get_emails(self) -> set[str]:
+            return set()
+
+        async def get_urls(self) -> set[str]:
+            return {'https://gitlab.com/group/project'}
+
+    report = tmp_path / 'gitlab-report'
+    monkeypatch.setattr(theharvester_main.stash, 'StashManager', FakeStash)
+    monkeypatch.setattr(theharvester_main.gitlabsearch, 'SearchGitlab', FakeGitlab)
+    monkeypatch.setattr(sys, 'argv', ['theHarvester', '-d', 'example.test', '-b', 'gitlab', '-f', str(report)])
+
+    with pytest.raises(SystemExit) as exit_info:
+        await theharvester_main.start()
+
+    assert exit_info.value.code == 0
+    assert completed_results[0].results == (('url', 'https://gitlab.com/group/project'),)
+    records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()]
+    assert {'type': 'url', 'value': 'https://gitlab.com/group/project'} in records

tests/lib/test_core.py

@@ -49,6 +49,7 @@ def test_capabilities_and_explicit_sources_form_a_union() -> None:
         "bevigil",
         "builtwith",
         "certspotter",
+        "gitlab",
         "intelx",
         "rocketreach",
         "urlscan",

tests/lib/test_source_catalog.py

@@ -55,7 +55,9 @@ def test_subdomain_route_drives_subdomain_capability() -> None:
 
 
 def test_source_specs_describe_consolidated_routes_not_getter_presence() -> None:
-    assert SOURCE_SPECS['gitlab'].routes == frozenset({ResultRoute.SUBDOMAINS, ResultRoute.EMAILS})
+    assert SOURCE_SPECS['gitlab'].routes == frozenset(
+        {ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.URLS}
+    )
     assert SOURCE_SPECS['haveibeenpwned'].routes == frozenset({ResultRoute.BREACHES})
     assert SOURCE_SPECS['hibpverified'].routes == frozenset({ResultRoute.EMAILS, ResultRoute.BREACHES})
     assert SOURCE_SPECS['urlscan'].routes == frozenset(

tests/test_all_source_orchestration.py

@@ -169,6 +169,9 @@ async def test_all_schedules_each_passive_catalog_source_once_and_reports_result
         async def get_links(self) -> set[str]:
             return {'https://sub.example.test/profile'}
 
+        async def get_urls(self) -> set[str]:
+            return {'https://gitlab.com/example/project'}
+
         async def get_interestingurls(self) -> set[str]:
             return {'https://sub.example.test/evidence'}
 
@@ -229,6 +232,7 @@ async def test_all_schedules_each_passive_catalog_source_once_and_reports_result
     jsonl_records = [json.loads(line) for line in report.with_suffix('.jsonl').read_text().splitlines()]
     assert {'type': 'hostname', 'value': 'sub.example.test'} in jsonl_records
     assert {'type': 'email', 'value': 'user@example.test'} in jsonl_records
+    assert {'type': 'url', 'value': 'https://gitlab.com/example/project'} in jsonl_records
 
     completed = await TestStash().load_completed_result(UUID(jsonl_records[0]['run_id']))
     assert completed.target == 'example.test'

tests/test_readme.py

@@ -14,6 +14,7 @@ ROUTE_COLUMNS = {
     ResultRoute.IPS: 'IPs',
     ResultRoute.ASNS: 'ASNs',
     ResultRoute.LINKS: 'URLs / links',
+    ResultRoute.URLS: 'URLs / links',
     ResultRoute.INTERESTING_URLS: 'URLs / links',
     ResultRoute.PEOPLE: 'People',
     ResultRoute.BREACHES: 'Breaches',

theHarvester/__main__.py

@@ -442,6 +442,12 @@ async def start(
             if len(links) > 0:
                 await db.store_all(word, links, 'linkedinlinks', source)
 
+        if ResultRoute.URLS in routes:
+            urls = await search_engine.get_urls()
+            all_urls.extend(urls)
+            if len(urls) > 0:
+                await db_stash.store_all(word, urls, 'url', source)
+
         if ResultRoute.INTERESTING_URLS in routes:
             get_interesting_urls = getattr(search_engine, 'get_interesting_urls', None)
             iurls = await get_interesting_urls() if get_interesting_urls else await search_engine.get_interestingurls()
@@ -1471,7 +1477,7 @@ async def start(
             output_logger.info('\n[*] No Trello URLs found.')
     else:
         total = length_urls
-        print_section('\n[*] Trello URLs found: ' + str(total), all_urls, '--------------------')
+        print_section('\n[*] URLs found: ' + str(total), all_urls, '--------------------')
         all_urls = sorted_unique(all_urls)
 
     if len(all_ip) == 0:

theHarvester/discovery/gitlabsearch.py

@@ -68,6 +68,13 @@ class SearchGitlab:
 
         return emails
 
+    def _add_text_evidence(self, text: str) -> bool:
+        hosts = self._extract_domains_from_text(text)
+        emails = self._extract_emails_from_text(text)
+        self.totalhosts.update(hosts)
+        self.totalemails.update(emails)
+        return bool(hosts or emails)
+
     async def search_projects(self) -> None:
         """Search GitLab projects for domain references"""
         try:
@@ -101,12 +108,7 @@ class SearchGitlab:
 
                         # Look for domains in description and name
                         all_text = f'{description} {name} {path}'
-                        self.totalhosts.update(self._extract_domains_from_text(all_text))
-                        self.totalemails.update(self._extract_emails_from_text(all_text))
-
-                        # Add the web URL if it contains our domain
-                        if web_url and self.word in web_url:
-                            self.totalurls.add(web_url)
+                        project_is_relevant = self._add_text_evidence(all_text)
 
                         # Try to get README content for more detailed search
                         project_id = project.get('id')
@@ -122,11 +124,13 @@ class SearchGitlab:
                                     readme_text = (
                                         readme_response[0] if isinstance(readme_response[0], str) else str(readme_response[0])
                                     )
-                                    self.totalhosts.update(self._extract_domains_from_text(readme_text))
-                                    self.totalemails.update(self._extract_emails_from_text(readme_text))
+                                    project_is_relevant = self._add_text_evidence(readme_text) or project_is_relevant
                             except Exception:
                                 pass  # README might not exist or be accessible
 
+                        if project_is_relevant and isinstance(web_url, str) and web_url.strip():
+                            self.totalurls.add(web_url.strip())
+
                 except Exception as e:
                     logger.info(f'Failed to parse GitLab projects response: {e}')
 
@@ -163,20 +167,23 @@ class SearchGitlab:
                     public_email = user.get('public_email', '') or ''
 
                     # Look for domains in user info
-                    all_text = f'{name} {username} {bio} {website_url}'
-                    self.totalhosts.update(self._extract_domains_from_text(all_text))
+                    user_hosts = self._extract_domains_from_text(f'{name} {username} {bio}')
+                    website_hosts = self._extract_domains_from_text(website_url) if isinstance(website_url, str) else set()
+                    user_hosts.update(website_hosts)
+                    self.totalhosts.update(user_hosts)
 
                     # Check email
+                    user_emails: set[str] = set()
                     if public_email:
-                        self.totalemails.update(self._extract_emails_from_text(public_email))
+                        user_emails = self._extract_emails_from_text(public_email)
+                        self.totalemails.update(user_emails)
 
-                    # Check website URL
-                    if website_url and self.word in website_url:
-                        self.totalurls.add(website_url)
+                    user_is_relevant = bool(user_hosts or user_emails)
+                    if website_hosts and isinstance(website_url, str) and website_url.strip():
+                        self.totalurls.add(website_url.strip())
 
-                    # Add user profile URL if relevant
-                    if web_url:
-                        self.totalurls.add(web_url)
+                    if user_is_relevant and isinstance(web_url, str) and web_url.strip():
+                        self.totalurls.add(web_url.strip())
 
             except Exception as e:
                 logger.info(f'Failed to parse GitLab users response: {e}')

theHarvester/lib/api/api.py

@@ -48,7 +48,7 @@ class QueryResponse(BaseModel):
     twitter_people: list[str] = Field(default_factory=list, description='List of Twitter people')
     linkedin_people: list[dict] = Field(default_factory=list, description='List of LinkedIn people')
     linkedin_links: list[str] = Field(default_factory=list, description='List of LinkedIn links')
-    trello_urls: list[str] = Field(default_factory=list, description='List of Trello URLs')
+    trello_urls: list[str] = Field(default_factory=list, description='List of discovered URLs (legacy field name)')
     ips: list[str] = Field(default_factory=list, description='List of IPs')
     emails: list[str] = Field(default_factory=list, description='List of emails')
     hosts: list[str] = Field(default_factory=list, description='List of hosts')

theHarvester/lib/source_catalog.py

@@ -22,6 +22,7 @@ class ResultRoute(Enum):
     ASNS = auto()
     PEOPLE = auto()
     LINKS = auto()
+    URLS = auto()
     INTERESTING_URLS = auto()
     BREACHES = auto()
 
@@ -33,6 +34,7 @@ _ROUTE_CAPABILITIES = {
     ResultRoute.ASNS: 'asns',
     ResultRoute.PEOPLE: 'people',
     ResultRoute.LINKS: 'urls',
+    ResultRoute.URLS: 'urls',
     ResultRoute.INTERESTING_URLS: 'urls',
     ResultRoute.BREACHES: 'breaches',
 }
@@ -88,7 +90,7 @@ _SPECS = (
     _spec('fofa', ResultRoute.SUBDOMAINS, ResultRoute.IPS),
     _spec('fullhunt', ResultRoute.SUBDOMAINS),
     _spec('github-code', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS),
-    _spec('gitlab', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS),
+    _spec('gitlab', ResultRoute.SUBDOMAINS, ResultRoute.EMAILS, ResultRoute.URLS),
     _spec('hackertarget', ResultRoute.SUBDOMAINS),
     _spec('haveibeenpwned', ResultRoute.BREACHES),
     _spec('hibpverified', ResultRoute.EMAILS, ResultRoute.BREACHES),