mirror of
https://github.com/laramies/theHarvester.git
synced 2026-08-17 19:35:40 +02:00
bump ruff to py3.14 and fix all the errors from mypy,ty,pytest and ruff issues
This commit is contained in:
+1
-1
@@ -124,7 +124,7 @@ exclude = [
|
||||
]
|
||||
|
||||
line-length = 130
|
||||
target-version = "py313"
|
||||
target-version = "py314"
|
||||
show-fixes = true
|
||||
|
||||
[tool.ruff.lint]
|
||||
|
||||
@@ -23,7 +23,7 @@ def test_python_version_policy_requires_314() -> None:
|
||||
assert 'Programming Language :: Python :: 3.13' not in project['project']['classifiers']
|
||||
assert project['tool']['mypy']['python_version'] == '3.14'
|
||||
assert project['tool']['uv']['pip']['python-version'] == '3.14'
|
||||
assert project['tool']['ruff']['target-version'] == 'py313'
|
||||
assert project['tool']['ruff']['target-version'] == 'py314'
|
||||
bug_report = (PROJECT_ROOT / '.github' / 'ISSUE_TEMPLATE' / 'bug_report.yml').read_text(encoding='utf-8')
|
||||
assert 'Python version: 3.14.6' in bug_report
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ import secrets
|
||||
import string
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Iterable
|
||||
from contextlib import AsyncExitStack
|
||||
from datetime import UTC, datetime
|
||||
from ipaddress import ip_address, ip_network
|
||||
@@ -83,6 +82,8 @@ from theHarvester.lib.virtual_host import (
|
||||
from theHarvester.screenshot.screenshot import ScreenShotter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Awaitable, Callable, Iterable
|
||||
|
||||
from theHarvester.lib.network_evidence import NetworkObservation
|
||||
from theHarvester.lib.takeover_evidence import TakeoverCandidateOutcome
|
||||
|
||||
@@ -1386,7 +1387,7 @@ async def start(
|
||||
return {}, set()
|
||||
try:
|
||||
outcomes = await search_take.get_takeover_outcomes()
|
||||
except (asyncio.CancelledError, Exception):
|
||||
except asyncio.CancelledError, Exception:
|
||||
if not best_effort:
|
||||
raise
|
||||
return {}, set()
|
||||
@@ -1912,7 +1913,7 @@ async def start(
|
||||
api_scanner = None
|
||||
|
||||
def collect_api_action_groups(
|
||||
scanner: 'api_endpoints.SearchApiEndpoints | None',
|
||||
scanner: api_endpoints.SearchApiEndpoints | None,
|
||||
*,
|
||||
best_effort: bool = False,
|
||||
) -> tuple[set[str], set[str], dict[ResultKind, Iterable[str]]]:
|
||||
|
||||
@@ -582,7 +582,7 @@ class SearchApiEndpoints:
|
||||
if retry_at.tzinfo is None:
|
||||
retry_at = retry_at.replace(tzinfo=UTC)
|
||||
delay = max(0.0, (retry_at - datetime.now(UTC)).total_seconds())
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
except TypeError, ValueError, OverflowError:
|
||||
pass
|
||||
if delay is None:
|
||||
delay = cls.DEFAULT_RETRY_DELAY_SECONDS * 2**attempt
|
||||
|
||||
@@ -151,7 +151,7 @@ class SearchApisGuru:
|
||||
return
|
||||
try:
|
||||
address = Address(addr_spec=candidate)
|
||||
except (HeaderParseError, ValueError):
|
||||
except HeaderParseError, ValueError:
|
||||
return
|
||||
if len(address.username.encode('utf-8')) > 64:
|
||||
return
|
||||
|
||||
@@ -53,7 +53,7 @@ class SearchArquivo:
|
||||
for line in response.body.splitlines():
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
except json.JSONDecodeError, TypeError:
|
||||
continue
|
||||
if not isinstance(item, dict) or not isinstance(url := item.get('url'), str):
|
||||
continue
|
||||
|
||||
@@ -46,7 +46,7 @@ class SearchBufferover:
|
||||
continue
|
||||
try:
|
||||
row = next(csv.reader([result]))
|
||||
except (csv.Error, StopIteration):
|
||||
except csv.Error, StopIteration:
|
||||
malformed = True
|
||||
continue
|
||||
if len(row) != 4:
|
||||
|
||||
@@ -76,7 +76,7 @@ class SearchBuiltWith:
|
||||
if not parsed.path.startswith('/'):
|
||||
return None, True
|
||||
return urlunsplit(('https', hostname, parsed.path, parsed.query, parsed.fragment)), False
|
||||
except (UnicodeError, ValueError):
|
||||
except UnicodeError, ValueError:
|
||||
return None, True
|
||||
|
||||
def _extract_technology(self, technology: object) -> bool:
|
||||
|
||||
@@ -138,5 +138,5 @@ class SearchCensys:
|
||||
return await self.do_search()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, ValueError):
|
||||
except aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, ValueError:
|
||||
return SourceExecutionReport('failed', 'transport-error')
|
||||
|
||||
@@ -90,7 +90,7 @@ class SearchCommoncrawl:
|
||||
continue
|
||||
try:
|
||||
timestamp = datetime.fromisoformat(str(entry.get('to'))).replace(tzinfo=None)
|
||||
except (KeyError, ValueError):
|
||||
except KeyError, ValueError:
|
||||
logger.warning(f'Common Crawl API error for index {index_id}: invalid catalog entry')
|
||||
continue
|
||||
dated_indexes.append((timestamp, entry))
|
||||
|
||||
@@ -10,10 +10,10 @@ import logging
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from ipaddress import IPv4Network
|
||||
from itertools import chain, islice
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from aiodns import DNSResolver
|
||||
|
||||
@@ -21,6 +21,9 @@ from theHarvester.lib import hostchecker
|
||||
from theHarvester.lib.cancellation import drain_tasks_after_cancellation
|
||||
from theHarvester.lib.core import DATA_DIR
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Iterator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#####################################################################
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import urllib.parse as urlparse
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import aiohttp
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
from theHarvester.discovery.constants import MissingKey, get_delay
|
||||
from theHarvester.lib.core import AsyncFetcher, Core
|
||||
from theHarvester.parsers import myparser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import aiohttp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
# theHarvester/discovery/hackertarget.py
|
||||
from ipaddress import ip_address, ip_network
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from theHarvester.lib.core import AsyncFetcher, Core, FetcherResponse
|
||||
from theHarvester.lib.hostnames import normalize_scoped_hostname
|
||||
from theHarvester.lib.source_execution import SourceExecutionReport, SourceReportStatus
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
|
||||
class SearchHackerTarget:
|
||||
"""Class uses the HackerTarget API to gather subdomains and IPs.
|
||||
@@ -25,7 +29,7 @@ class SearchHackerTarget:
|
||||
headers = {'User-agent': Core.get_user_agent()}
|
||||
|
||||
urls = [f'{self.hostname}/hostsearch/?q={self.word}']
|
||||
parsers = [self._parse_hostsearch]
|
||||
parsers: list[Callable[[str], tuple[int, bool]]] = [self._parse_hostsearch]
|
||||
address_query = True
|
||||
try:
|
||||
ip_network(self.word.strip(), strict=False)
|
||||
|
||||
@@ -30,7 +30,7 @@ class SearchHaveIBeenPwned:
|
||||
proxy=proxy,
|
||||
include_metadata=True,
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
except OSError, RuntimeError, ValueError:
|
||||
logger.info('HaveIBeenPwned request failed')
|
||||
return SourceExecutionReport('failed', 'transport-error')
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ class SearchHibpVerified:
|
||||
proxy=proxy,
|
||||
include_metadata=True,
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
except OSError, RuntimeError, ValueError:
|
||||
logger.info('HIBP verified-domain request failed')
|
||||
return
|
||||
|
||||
|
||||
@@ -115,7 +115,7 @@ class SearchHunter:
|
||||
self.proxy = proxy
|
||||
try:
|
||||
await self.do_search() # Only need to do it once.
|
||||
except (AttributeError, KeyError, TypeError):
|
||||
except AttributeError, KeyError, TypeError:
|
||||
logger.info('Hunter returned malformed data')
|
||||
|
||||
async def get_emails(self):
|
||||
|
||||
@@ -88,7 +88,7 @@ class SearchIntelx:
|
||||
continue
|
||||
try:
|
||||
address = Address(addr_spec=email.strip().lower())
|
||||
except (HeaderParseError, ValueError):
|
||||
except HeaderParseError, ValueError:
|
||||
continue
|
||||
if address.username and (normalized_domain := normalize_scoped_hostname(address.domain, self.word)):
|
||||
emails.add(f'{address.username}@{normalized_domain}')
|
||||
|
||||
@@ -54,7 +54,7 @@ class SearchLeakix:
|
||||
logger.info(f'LeakIX rate limited; retrying once in {delay:g} seconds')
|
||||
await asyncio.sleep(delay)
|
||||
response = await self._fetch()
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
except OSError, RuntimeError, ValueError:
|
||||
logger.info('LeakIX request failed')
|
||||
return
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ class SearchOtx:
|
||||
include_metadata=True,
|
||||
)
|
||||
response = response_list[0] if response_list and isinstance(response_list[0], FetcherResponse) else None
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
except OSError, RuntimeError, ValueError:
|
||||
self.totalhosts = set()
|
||||
self.totalips = set()
|
||||
logger.info('OTX request failed')
|
||||
@@ -85,7 +85,7 @@ class SearchOtx:
|
||||
self.totalips.add(str(ip_address(address.strip())))
|
||||
except ValueError:
|
||||
continue
|
||||
except (KeyError, TypeError, ValueError):
|
||||
except KeyError, TypeError, ValueError:
|
||||
self.totalhosts = set()
|
||||
self.totalips = set()
|
||||
return SourceExecutionReport('failed', 'invalid-response')
|
||||
|
||||
@@ -72,7 +72,7 @@ class SearchPentestTools:
|
||||
return
|
||||
try:
|
||||
output_data = json_results['output_data']['subdomains']
|
||||
except (KeyError, TypeError):
|
||||
except KeyError, TypeError:
|
||||
return
|
||||
if not isinstance(output_data, list):
|
||||
return
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from theHarvester.lib.core import FetcherResponse
|
||||
from theHarvester.lib.source_execution import SourceReportStatus
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from theHarvester.lib.source_execution import SourceReportStatus
|
||||
|
||||
|
||||
def provider_http_error(response: object) -> tuple[SourceReportStatus, str] | None:
|
||||
|
||||
@@ -30,7 +30,7 @@ class SearchRobtex:
|
||||
if line.strip():
|
||||
try:
|
||||
results.append(json.loads(line))
|
||||
except (TypeError, ValueError):
|
||||
except TypeError, ValueError:
|
||||
continue
|
||||
return results
|
||||
|
||||
|
||||
@@ -96,7 +96,7 @@ class SearchDehashed:
|
||||
if len(entries) < size:
|
||||
break
|
||||
page += 1
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
except OSError, RuntimeError, ValueError:
|
||||
logger.info('\t[!] Dehashed request failed')
|
||||
break
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ class SearchDNSDumpster:
|
||||
continue
|
||||
try:
|
||||
self.ips.add(str(ip_address(candidate)))
|
||||
except (TypeError, ValueError):
|
||||
except TypeError, ValueError:
|
||||
malformed = True
|
||||
|
||||
if malformed:
|
||||
|
||||
@@ -425,7 +425,7 @@ class SearchShodan:
|
||||
ip = str(ip_address(ip_value))
|
||||
if self._record_host(ip, {**match, 'data': [match]}):
|
||||
error_types.add('InvalidResponseError')
|
||||
except (TypeError, ValueError):
|
||||
except TypeError, ValueError:
|
||||
error_types.add('InvalidResponseError')
|
||||
received += len(matches)
|
||||
if not matches:
|
||||
@@ -463,7 +463,7 @@ class SearchShodan:
|
||||
try:
|
||||
if self._record_host(normalized_ip, response.body):
|
||||
self.error_type = 'InvalidResponseError'
|
||||
except (TypeError, ValueError):
|
||||
except TypeError, ValueError:
|
||||
self.error_type = 'InvalidResponseError'
|
||||
self.tracker[normalized_ip] = 'Shodan request failed'
|
||||
except ResponseStreamError as error:
|
||||
|
||||
@@ -197,7 +197,7 @@ class SearchSourcegraph:
|
||||
return
|
||||
try:
|
||||
failure = self._consume_event(record)
|
||||
except (json.JSONDecodeError, RecursionError, TypeError, ValueError):
|
||||
except json.JSONDecodeError, RecursionError, TypeError, ValueError:
|
||||
self._stop('invalid-response')
|
||||
return
|
||||
except OverflowError:
|
||||
|
||||
@@ -117,7 +117,7 @@ class SearchTomba:
|
||||
self.proxy = proxy
|
||||
try:
|
||||
await self.do_search() # Only need to do it once.
|
||||
except (AttributeError, KeyError, TypeError):
|
||||
except AttributeError, KeyError, TypeError:
|
||||
logger.info('Tomba returned malformed data')
|
||||
|
||||
async def get_emails(self):
|
||||
|
||||
@@ -266,7 +266,7 @@ class SearchWindvane:
|
||||
try:
|
||||
parts = ip.split('.')
|
||||
return len(parts) == 4 and all(0 <= int(part) <= 255 for part in parts)
|
||||
except (ValueError, TypeError):
|
||||
except ValueError, TypeError:
|
||||
return False
|
||||
|
||||
async def get_hostnames(self) -> set:
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Self
|
||||
from typing import TYPE_CHECKING, Self
|
||||
|
||||
from theHarvester.lib.evidence_types import EXECUTION_STATUSES, RESULT_KINDS, ExecutionStatus, ResultKind, format_utc
|
||||
from theHarvester.lib.result_values import normalize_result_value
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
|
||||
@dataclass(frozen=True, order=True, slots=True)
|
||||
class ActionObservation:
|
||||
|
||||
@@ -21,7 +21,7 @@ def _docker_gateway() -> ipaddress.IPv4Address | None:
|
||||
fields = line.split()
|
||||
if len(fields) >= 3 and fields[1] == '00000000':
|
||||
return ipaddress.IPv4Address(bytes.fromhex(fields[2])[::-1])
|
||||
except (OSError, ValueError):
|
||||
except OSError, ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
@@ -330,7 +330,7 @@ class RunStore:
|
||||
existing = None
|
||||
try:
|
||||
existing = await self.results.load_run(completed.run_id)
|
||||
except (LookupError, ResultStoreError):
|
||||
except LookupError, ResultStoreError:
|
||||
pass
|
||||
if record is not None or existing is not None:
|
||||
if record is not None and existing == completed:
|
||||
@@ -529,7 +529,7 @@ class RunStore:
|
||||
async def _existing_evidence(self, run_id: str, target: str) -> CompletedResult | None:
|
||||
try:
|
||||
completed = await self.results.load_run(UUID(run_id))
|
||||
except (LookupError, ResultStoreError, ValueError):
|
||||
except LookupError, ResultStoreError, ValueError:
|
||||
return None
|
||||
return completed if completed.target == target else None
|
||||
|
||||
|
||||
@@ -355,7 +355,7 @@ async def _child_execute(run_id: str, database: Path) -> None:
|
||||
try:
|
||||
loop.add_signal_handler(signal.SIGTERM, task.cancel)
|
||||
signal_handler_installed = True
|
||||
except (NotImplementedError, RuntimeError):
|
||||
except NotImplementedError, RuntimeError:
|
||||
pass
|
||||
try:
|
||||
response = await task
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import json
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Self
|
||||
from typing import TYPE_CHECKING, Self
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from theHarvester.lib.active_evidence import ActiveEvidence
|
||||
@@ -41,6 +39,10 @@ from theHarvester.lib.takeover_evidence import (
|
||||
)
|
||||
from theHarvester.lib.virtual_host import VirtualHostObservation
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Mapping
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def virtual_host_details(observations: Iterable[VirtualHostObservation]) -> list[dict[str, object]]:
|
||||
details: list[dict[str, object]] = []
|
||||
|
||||
@@ -4,11 +4,14 @@ Production access retains Core's existing file precedence. In-memory access lets
|
||||
tests and embedded callers provide credentials without filesystem or global state.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from theHarvester.lib.core import Core
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping
|
||||
|
||||
|
||||
class FileSystemCredentialAdapter:
|
||||
"""Read production credentials through Core's existing file precedence."""
|
||||
|
||||
@@ -24,6 +24,8 @@ from theHarvester.lib.source_catalog import SOURCE_SPECS, resolve_sources
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator, Sized
|
||||
|
||||
from aiohttp.abc import AbstractCookieJar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DATA_DIR = Path(__file__).parents[1] / 'data'
|
||||
@@ -474,7 +476,7 @@ class AsyncFetcher:
|
||||
if isinstance(proxy, bool) and proxy:
|
||||
try:
|
||||
return cls._get_random_proxy(cls().proxy_list)
|
||||
except (IndexError, TypeError, ValueError):
|
||||
except IndexError, TypeError, ValueError:
|
||||
return None, None
|
||||
return None, None
|
||||
|
||||
@@ -486,7 +488,7 @@ class AsyncFetcher:
|
||||
proxy_url: str | None = None,
|
||||
proxy_type: str | None = None,
|
||||
ssl_context: ssl.SSLContext | bool | None = None,
|
||||
cookie_jar: aiohttp.abc.AbstractCookieJar | None = None,
|
||||
cookie_jar: AbstractCookieJar | None = None,
|
||||
) -> aiohttp.ClientSession:
|
||||
connector = None
|
||||
if proxy_url is not None or proxy_type is not None or ssl_context is not None:
|
||||
@@ -510,7 +512,7 @@ class AsyncFetcher:
|
||||
headers: dict[str, str] | None = None,
|
||||
proxy: str | bool | None = '',
|
||||
request_timeout: int | None = None,
|
||||
cookie_jar: aiohttp.abc.AbstractCookieJar | None = None,
|
||||
cookie_jar: AbstractCookieJar | None = None,
|
||||
) -> AsyncIterator[aiohttp.ClientSession]:
|
||||
"""Own one connection pool, proxy identity, and cookie jar for a provider conversation."""
|
||||
proxy_url, proxy_type = cls._resolve_proxy(proxy)
|
||||
@@ -580,7 +582,7 @@ class AsyncFetcher:
|
||||
else:
|
||||
try:
|
||||
body = await response.json()
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
except aiohttp.ContentTypeError, ValueError:
|
||||
body = text_body
|
||||
else:
|
||||
body = await response.json()
|
||||
@@ -702,7 +704,7 @@ class AsyncFetcher:
|
||||
include_metadata=include_metadata,
|
||||
**request_kwargs,
|
||||
)
|
||||
except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError):
|
||||
except aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError:
|
||||
return None if include_metadata else ''
|
||||
|
||||
@classmethod
|
||||
@@ -768,7 +770,7 @@ class AsyncFetcher:
|
||||
finally:
|
||||
if owns_session:
|
||||
await session.close()
|
||||
except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError):
|
||||
except aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError:
|
||||
return None if include_metadata else ''
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -4,7 +4,6 @@ import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
from collections.abc import AsyncIterator, Iterable
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
@@ -41,8 +40,6 @@ from theHarvester.lib.active_evidence import (
|
||||
)
|
||||
from theHarvester.lib.asn_attribution import (
|
||||
AsnAttributionObservation,
|
||||
ProducerKind,
|
||||
SubjectKind,
|
||||
canonical_asn_attributions,
|
||||
)
|
||||
from theHarvester.lib.completed_result import (
|
||||
@@ -70,6 +67,9 @@ from theHarvester.lib.takeover_evidence import (
|
||||
from theHarvester.lib.virtual_host import VirtualHostObservation
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator, Iterable
|
||||
|
||||
from theHarvester.lib.asn_attribution import ProducerKind, SubjectKind
|
||||
from theHarvester.lib.evidence_types import EvidenceStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -1065,14 +1065,16 @@ class ResultStore:
|
||||
or subject_result.kind not in {'hostname', 'ip'}
|
||||
):
|
||||
raise ResultStoreError('Persisted ASN attribution is invalid')
|
||||
producer_kind: ProducerKind = 'source' if attribution_execution.producer_kind == 'source' else 'action'
|
||||
subject_kind: SubjectKind = 'hostname' if subject_result.kind == 'hostname' else 'ip'
|
||||
try:
|
||||
asn_attributions.append(
|
||||
AsnAttributionObservation(
|
||||
cast('ProducerKind', attribution_execution.producer_kind),
|
||||
producer_kind,
|
||||
attribution_execution.name,
|
||||
asn_result.value,
|
||||
attribution.organization_label,
|
||||
cast('SubjectKind', subject_result.kind),
|
||||
subject_kind,
|
||||
subject_result.value,
|
||||
datetime.datetime.fromisoformat(attribution.collected_at),
|
||||
)
|
||||
|
||||
@@ -60,7 +60,7 @@ async def resolve_ip_addresses(hostname: str, *, family: socket.AddressFamily =
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode('ascii')
|
||||
address = ipaddress.ip_address(value)
|
||||
except (AttributeError, IndexError, TypeError, UnicodeDecodeError, ValueError):
|
||||
except AttributeError, IndexError, TypeError, UnicodeDecodeError, ValueError:
|
||||
continue
|
||||
if family == socket.AF_INET and address.version != 4:
|
||||
continue
|
||||
|
||||
@@ -277,13 +277,18 @@ class NetworkEvidenceAccumulator:
|
||||
|
||||
|
||||
def network_observation_sort_key(observation: NetworkObservation) -> tuple[object, ...]:
|
||||
type_order = {PrefixOriginObservation: 0, BgpRouteObservation: 1, RpkiValidationObservation: 2}
|
||||
if isinstance(observation, PrefixOriginObservation):
|
||||
type_order = 0
|
||||
elif isinstance(observation, BgpRouteObservation):
|
||||
type_order = 1
|
||||
else:
|
||||
type_order = 2
|
||||
record = observation.to_record()
|
||||
return (
|
||||
observation.prefix,
|
||||
observation.origin_asn,
|
||||
observation.action,
|
||||
type_order[type(observation)],
|
||||
type_order,
|
||||
tuple((key, str(value)) for key, value in sorted(record.items())),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from collections.abc import Iterable
|
||||
from ipaddress import ip_address
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
||||
DEFAULT_DNS_RESOLVERS = ('1.1.1.1', '8.8.8.8', '9.9.9.9')
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ def _canonical_asns(values: Iterable[str | int]) -> tuple[tuple[str, ...], bool]
|
||||
continue
|
||||
try:
|
||||
normalized.add(normalize_asn(value))
|
||||
except (TypeError, ValueError):
|
||||
except TypeError, ValueError:
|
||||
continue
|
||||
return tuple(sorted(normalized, key=lambda value: int(value[2:]))), truncated
|
||||
|
||||
@@ -448,7 +448,7 @@ class _RouteViewsRuntime:
|
||||
self._record_error(error.error_type, error.stop_reason, override=error.terminal)
|
||||
if error.terminal:
|
||||
raise
|
||||
except (TypeError, ValueError):
|
||||
except TypeError, ValueError:
|
||||
self._record_error('ValueError', 'invalid-response')
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, StrEnum, auto
|
||||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
|
||||
class ActivityClass(StrEnum):
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from theHarvester.lib.takeover_evidence import TakeoverClassification, TakeoverRcode
|
||||
if TYPE_CHECKING:
|
||||
from theHarvester.lib.takeover_evidence import TakeoverClassification, TakeoverRcode
|
||||
|
||||
# This reviewed snapshot translates the provider-gated subset of
|
||||
# can-i-take-over-xyz@5bd4e128 and compound predicates from
|
||||
|
||||
@@ -391,7 +391,7 @@ class VirtualHostObservation:
|
||||
control_body_sha256=control_body_sha256,
|
||||
control_body_size=control_body_size,
|
||||
control_body_truncated=control_body_truncated,
|
||||
confirmation_body_sha256=cast('str | None', confirmation_body_sha256),
|
||||
confirmation_body_sha256=confirmation_body_sha256,
|
||||
)
|
||||
|
||||
def to_record(self) -> dict[str, object]:
|
||||
@@ -642,7 +642,7 @@ async def _probe_batch(
|
||||
|
||||
try:
|
||||
completed.update(enumerate(await asyncio.gather(*tasks)))
|
||||
except (asyncio.CancelledError, Exception):
|
||||
except asyncio.CancelledError, Exception:
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
@@ -762,7 +762,7 @@ async def discover_virtual_hosts(
|
||||
completed_candidates: dict[int, ProbeObservation] = {}
|
||||
try:
|
||||
await _probe_batch(session, request, candidate_names, completed_candidates)
|
||||
except (asyncio.CancelledError, Exception):
|
||||
except asyncio.CancelledError, Exception:
|
||||
record_request_errors(list(completed_candidates.values()))
|
||||
observations.extend(
|
||||
_classify_candidate(
|
||||
@@ -797,7 +797,7 @@ async def discover_virtual_hosts(
|
||||
completed_confirmations: dict[int, ProbeObservation] = {}
|
||||
try:
|
||||
await _probe_batch(session, request, confirmation_names, completed_confirmations)
|
||||
except (asyncio.CancelledError, Exception):
|
||||
except asyncio.CancelledError, Exception:
|
||||
record_request_errors(list(completed_confirmations.values()))
|
||||
for confirmation_index, confirmation in completed_confirmations.items():
|
||||
index = confirmation_indexes[confirmation_index]
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import re
|
||||
from collections.abc import Set
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from theHarvester.lib.hostnames import normalize_scoped_hostname
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Set
|
||||
|
||||
|
||||
class Parser:
|
||||
def __init__(self, results, word) -> None:
|
||||
|
||||
@@ -7,17 +7,20 @@ import logging
|
||||
import os
|
||||
import ssl
|
||||
import sys
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Collection
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from ipaddress import ip_address
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import aiohttp
|
||||
import certifi
|
||||
from aiohttp_socks import ProxyConnector
|
||||
from playwright.async_api import Browser, async_playwright
|
||||
from playwright.async_api import Browser, BrowserContext, Page, async_playwright
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Collection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -128,9 +131,9 @@ class ScreenShotter:
|
||||
return [reachable[index] for index in sorted(reachable)]
|
||||
|
||||
@staticmethod
|
||||
async def _close(resource: object, resource_name: str) -> BaseException | None:
|
||||
async def _close(resource: Browser | BrowserContext | Page, resource_name: str) -> BaseException | None:
|
||||
try:
|
||||
await resource.close() # type: ignore[attr-defined]
|
||||
await resource.close()
|
||||
except BaseException as error:
|
||||
logger.info(f'An exception occurred while closing screenshot {resource_name}: {error}')
|
||||
return error
|
||||
|
||||
Reference in New Issue
Block a user