Merge pull request #2416 from NotoriousRebel/codex/upstream-dev-robtex-reverse-ip

Stop invalid Robtex reverse-IP requests

tests/discovery/test_robtex.py

@@ -0,0 +1,22 @@
+from typing import Any
+
+import pytest
+
+from theHarvester.discovery import robtex
+
+
+@pytest.mark.asyncio
+async def test_robtex_does_not_send_domain_to_reverse_ip_endpoint(monkeypatch: pytest.MonkeyPatch) -> None:
+    requested_urls: list[str] = []
+
+    async def fake_fetch_all(urls: list[str], **_kwargs: Any) -> list[str]:
+        requested_urls.extend(urls)
+        return ['{"rrname":"api.example.com","rrtype":"A","rrdata":"192.0.2.1"}']
+
+    monkeypatch.setattr(robtex.AsyncFetcher, 'fetch_all', fake_fetch_all)
+    search = robtex.SearchRobtex('example.com')
+    await search.process()
+
+    assert requested_urls == ['https://freeapi.robtex.com/pdns/forward/example.com']
+    assert await search.get_hostnames() == {'api.example.com'}
+    assert await search.get_ips() == {'192.0.2.1'}

theHarvester/discovery/robtex.py

@@ -89,21 +89,6 @@ class SearchRobtex:
                         except (ValueError, TypeError):
                             pass
 
-            # Also try reverse DNS lookup for additional data
-            reverse_url = f'{self.hostname}/pdns/reverse/{self.word}'
-            reverse_response = await AsyncFetcher.fetch_all([reverse_url], headers=headers, proxy=self.proxy)
-
-            if reverse_response and isinstance(reverse_response, list) and reverse_response[0]:
-                try:
-                    reverse_data = self._safe_parse_json_lines(reverse_response[0])
-                    for record in reverse_data:
-                        if isinstance(record, dict):
-                            rrdata = record.get('rrdata', '')
-                            if rrdata and (rrdata.endswith(self.word) or f'.{self.word}' in rrdata):
-                                self.totalhosts.add(rrdata.rstrip('.'))
-                except (TypeError, ValueError) as e:
-                    logger.info(f'Failed to parse reverse DNS data from Robtex: {e}')
-
         except (aiohttp.ClientError, TimeoutError, OSError, TypeError, ValueError) as e:
             logger.info(f'Robtex API error: {e}')