diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index d5930544..26b56a2f 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -224,7 +224,9 @@ async def start(rest_args: argparse.Namespace | None = None): try: db = stash.StashManager() await db.do_init() - except Exception: + except (AttributeError, OSError, RuntimeError, ValueError) as init_error: + if not args.quiet: + print(f'Error initializing StashManager: {init_error}') raise ValueError('Failed to initialize StashManager') if len(filename) > 0: @@ -264,7 +266,7 @@ async def start(rest_args: argparse.Namespace | None = None): try: _ = netaddr.IPAddress(line) final_dns_resolver_list.append(line) - except Exception as e: + except (netaddr.core.AddrFormatError, ValueError, TypeError) as e: print(f'An exception has occurred while reading from: {dnsresolve}, {e}') print(f'Current line: {line}') else: @@ -277,7 +279,7 @@ async def start(rest_args: argparse.Namespace | None = None): # Verify user passed in an IP; this does not validate resolver behavior _ = netaddr.IPAddress(item) final_dns_resolver_list.append(item) - except Exception as e: + except (netaddr.core.AddrFormatError, ValueError, TypeError) as e: print(f'Passed DNS resolver is invalid, skipping: {item} ({e})') # if for some reason, there are duplicates @@ -1262,7 +1264,7 @@ async def start(rest_args: argparse.Namespace | None = None): elif rest_args is not None: try: rest_args.dns_brute - except Exception: + except AttributeError: print('\n[!] Invalid source.\n') sys.exit(1) else: @@ -1384,7 +1386,7 @@ async def start(rest_args: argparse.Namespace | None = None): ip_list.append(str(netaddr.IPNetwork(ip))) else: ip_list.append(str(netaddr.IPAddress(ip))) - except Exception as e: + except (netaddr.core.AddrFormatError, ValueError, TypeError) as e: print(f'An exception has occurred while adding: {ip} to ip_list: {e}') continue ip_list = list(sorted(ip_list)) @@ -1437,7 +1439,7 @@ async def start(rest_args: argparse.Namespace | None = None): if ':' in host: _, addr = host.split(':', 1) await db.store(word, addr, 'ip', 'DNS-resolver') - except Exception as e: + except (OSError, RuntimeError, ValueError, TypeError) as e: print(f'An exception has occurred while attempting to insert: {host} IP into DB: {e}') continue else: @@ -1635,7 +1637,7 @@ async def start(rest_args: argparse.Namespace | None = None): # TODO add Shodan output into XML report file.write('') print('[*] XML File saved.') - except Exception as error: + except (OSError, ValueError, TypeError, UnicodeEncodeError) as error: print(f'[!] An error occurred while saving the XML file: {error}') try: @@ -1693,7 +1695,7 @@ async def start(rest_args: argparse.Namespace | None = None): dumped_json = ujson.dumps(json_dict, sort_keys=True) fp.write(dumped_json) print('[*] JSON File saved.') - except Exception as er: + except (OSError, ValueError, TypeError, UnicodeEncodeError) as er: print(f'[!] An error occurred while saving the JSON file: {er} ') print('\n\n') diff --git a/theHarvester/discovery/api_endpoints.py b/theHarvester/discovery/api_endpoints.py index df54889e..76d32b06 100644 --- a/theHarvester/discovery/api_endpoints.py +++ b/theHarvester/discovery/api_endpoints.py @@ -13,6 +13,8 @@ from dataclasses import asdict, dataclass, field from typing import Any from urllib.parse import urlparse +import aiohttp + from theHarvester.lib.core import AsyncFetcher, Core # Configure logging @@ -443,7 +445,7 @@ class SearchApiEndpoints: return 'https' else: self.logger.info(f'[*] HTTPS request to {https_url} returned status: {getattr(response, "status", "No status")}') - except Exception as e: + except (aiohttp.ClientError, TimeoutError, OSError, TypeError, ValueError, AttributeError) as e: self.logger.error(f"Failed to fetch HTTPS URL '{https_url}': {e}") return 'http' # Fallback to HTTP if HTTPS fails @@ -473,7 +475,7 @@ class SearchApiEndpoints: return list(set(variations)) # Return unique endpoints - except Exception as e: + except OSError as e: self.logger.error(f'Error loading wordlist {self.wordlist}: {e}') return [] @@ -518,7 +520,7 @@ class SearchApiEndpoints: except TimeoutError: self.logger.debug(f'Timeout for {method} {url}') continue - except Exception as e: + except (aiohttp.ClientError, OSError, TypeError, ValueError, AttributeError) as e: self.logger.debug(f'Error checking {method} {url}: {e!s}') continue @@ -559,13 +561,13 @@ class SearchApiEndpoints: # Get response headers safely try: headers = dict(getattr(response, 'headers', {})) - except Exception as e: + except (TypeError, ValueError, AttributeError) as e: self.logger.error(f'Failed to get headers from response for URL {url}: {e}') headers = {} try: content = getattr(response, 'content', b'') - except Exception as e: + except (TypeError, AttributeError) as e: self.logger.error(f'Failed to get content from response for URL {url}: {e}') content = b'' @@ -580,7 +582,7 @@ class SearchApiEndpoints: if content: try: content_preview = content.decode('utf-8', errors='ignore')[:200] - except Exception as e: + except (AttributeError, UnicodeDecodeError) as e: self.logger.error(f'Failed to decode content for URL {url}: {e}') # Extract security headers @@ -650,7 +652,7 @@ class SearchApiEndpoints: except json.JSONDecodeError as e: self.logger.error(f'Failed to parse JSON from response content: {e}') - except Exception as e: + except (TypeError, UnicodeDecodeError) as e: self.logger.error(f'Unexpected error while extracting parameters from JSON: {e}') # Create result object @@ -701,7 +703,7 @@ class SearchApiEndpoints: self.logger.error(f'JSON at {url} is not a dictionary. Type: {type(schema).__name__}') except json.JSONDecodeError as e: self.logger.error(f'Failed to parse JSON from {url}: {e}') - except Exception as e: + except (TypeError, UnicodeDecodeError) as e: self.logger.error(f'Unexpected error while processing schema at {url}: {e}') return result diff --git a/theHarvester/discovery/bitbucket.py b/theHarvester/discovery/bitbucket.py index 13317591..ae9afb74 100644 --- a/theHarvester/discovery/bitbucket.py +++ b/theHarvester/discovery/bitbucket.py @@ -1,5 +1,6 @@ import asyncio import random +import re import urllib.parse as urlparse from typing import Any, NamedTuple @@ -56,7 +57,7 @@ class SearchBitBucket: for match in item.get('text_matches', []) if match.get('fragment') is not None ] - except Exception as e: + except (AttributeError, TypeError, ValueError) as e: print(f'Error extracting fragments: {e}') return [] @@ -68,7 +69,7 @@ class SearchBitBucket: if page_param := urlparse.parse_qs(parsed.query).get('page', [None])[0]: return int(page_param) return 0 - except Exception as e: + except (AttributeError, TypeError, ValueError) as e: print(f'Error parsing page response: {e}') return None @@ -84,7 +85,7 @@ class SearchBitBucket: if status in (429, 403): return RetryResult(60) return ErrorResult(status, json_data if isinstance(json_data, dict) else text) - except Exception as e: + except (TypeError, ValueError, KeyError, AttributeError) as e: print(f'Error handling response: {e}') return ErrorResult(500, str(e)) @@ -101,7 +102,7 @@ class SearchBitBucket: async with aiohttp.ClientSession(headers=self.headers) as sess: async with sess.get(url, proxy=random.choice(Core.proxy_list()) if self.proxy else None) as resp: return await resp.text(), await resp.json(), resp.status, resp.links - except Exception as e: + except (aiohttp.ClientError, TimeoutError, ValueError, OSError) as e: print(f'Error performing search: {e}') return '', {}, 500, {} @@ -141,17 +142,17 @@ class SearchBitBucket: print(f'\tException occurred: status_code: {result.status_code} reason: {result.body}') self.page = 0 break - except Exception as e: + except (aiohttp.ClientError, TimeoutError, ValueError, TypeError, AttributeError) as e: print(f'Error processing page: {e}') await asyncio.sleep(get_delay()) - except Exception as e: + except (aiohttp.ClientError, TimeoutError, ValueError, TypeError, AttributeError) as e: print(f'An exception has occurred in bitbucket process: {e}') async def get_emails(self): try: rawres = myparser.Parser(self.total_results, self.word) return await rawres.emails() - except Exception as e: + except (AttributeError, TypeError, re.error) as e: print(f'Error getting emails: {e}') return [] @@ -159,6 +160,6 @@ class SearchBitBucket: try: rawres = myparser.Parser(self.total_results, self.word) return await rawres.hostnames() - except Exception as e: + except (AttributeError, TypeError, re.error) as e: print(f'Error getting hostnames: {e}') return [] diff --git a/theHarvester/discovery/robtex.py b/theHarvester/discovery/robtex.py index 178bbc44..0fbbeee5 100644 --- a/theHarvester/discovery/robtex.py +++ b/theHarvester/discovery/robtex.py @@ -1,6 +1,8 @@ import json as _stdlib_json from types import ModuleType +import aiohttp + from theHarvester.lib.core import AsyncFetcher, Core json: ModuleType = _stdlib_json @@ -10,7 +12,7 @@ try: json = _ujson except ImportError as e: print(f"'ujson' not available. Falling back to standard 'json' module. Reason: {e}") -except Exception as e: +except (AttributeError, OSError, RuntimeError, SystemError, ValueError) as e: print(f"Unexpected error while importing 'ujson'. Falling back to standard 'json'. Reason: {e}") @@ -37,7 +39,7 @@ class SearchRobtex: if line.strip(): try: results.append(json.loads(line)) - except Exception: + except (TypeError, ValueError): continue return results @@ -55,7 +57,7 @@ class SearchRobtex: try: data = self._safe_parse_json_lines(response[0]) - except Exception as e: + except (TypeError, ValueError) as e: print(f'Failed to parse JSON lines from Robtex response: {e}') return @@ -98,10 +100,10 @@ class SearchRobtex: rrdata = record.get('rrdata', '') if rrdata and (rrdata.endswith(self.word) or f'.{self.word}' in rrdata): self.totalhosts.add(rrdata.rstrip('.')) - except Exception as e: + except (TypeError, ValueError) as e: print(f'Failed to parse reverse DNS data from Robtex: {e}') - except Exception as e: + except (aiohttp.ClientError, TimeoutError, OSError, TypeError, ValueError) as e: print(f'Robtex API error: {e}') async def get_hostnames(self) -> set: diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 1ca2c2bd..03eeaf56 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -432,7 +432,7 @@ class AsyncFetcher: async with session.post(url, data=data, ssl=sslcontext, params=params) as resp: await asyncio.sleep(3) return await resp.text() if json is False else await resp.json() - except Exception: + except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError): return '' @classmethod @@ -473,7 +473,7 @@ class AsyncFetcher: elif isinstance(proxy, bool) and proxy: try: proxy_url, proxy_type = cls._get_random_proxy(cls().proxy_list) - except Exception: + except (IndexError, TypeError, ValueError): proxy_url = None proxy_type = None @@ -515,7 +515,7 @@ class AsyncFetcher: finally: if owns_session: await session.close() - except Exception: + except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError): return '' @staticmethod @@ -546,7 +546,7 @@ class AsyncFetcher: async with session.get(url, ssl=False) as response: await asyncio.sleep(5) return url, await response.text() - except Exception as e: + except (aiohttp.ClientError, TimeoutError, OSError, ssl.SSLError, UnicodeDecodeError, ValueError) as e: print(f'Takeover check error: {e}') return url, ''