diff --git a/README.md b/README.md index 86040d68..ffd87303 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,6 @@ Passive: * google: Google search engine (Optional Google dorking.) - www.google.com -* google-certificates: Google Certificate Transparency report - * hunter: Hunter search engine (Requires API key, see below.) - www.hunter.io * intelx: Intelx search engine (Requires API key, see below.) - www.intelx.io diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 9932fef3..fb43e147 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -54,7 +54,7 @@ def start(): parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, censys, crtsh, dnsdumpster, dogpile, duckduckgo, github-code, google, - google-certificates, hunter, intelx, + hunter, intelx, linkedin, netcraft, securityTrails, threatcrowd, trello, twitter, vhost, virustotal, yahoo, all''') parser.add_argument('-x', '--exclude', help='exclude options when using all sources', type=str) @@ -223,15 +223,6 @@ def start(): db.store_all(word, all_hosts, 'host', 'google') db.store_all(word, all_emails, 'email', 'google') - elif engineitem == 'google-certificates': - print('\033[94m[*] Searching Google Certificate transparency report. \033[0m') - search = googlecertificates.SearchGoogleCertificates(word, limit, start) - search.process() - hosts = filter(search.get_domains()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google-certificates') - elif engineitem == 'hunter': print('\033[94m[*] Searching Hunter. \033[0m') from theHarvester.discovery import huntersearch @@ -488,14 +479,6 @@ def start(): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'google') - print('\033[94m[*] Searching Google Certificate transparency report. \033[0m') - search = googlecertificates.SearchGoogleCertificates(word, limit, start) - search.process() - domains = filter(search.get_domains()) - all_hosts.extend(domains) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google-certificates') - print('\033[94m[*] Searching Hunter. \033[0m') from theHarvester.discovery import huntersearch # Import locally. diff --git a/theHarvester/discovery/__init__.py b/theHarvester/discovery/__init__.py index ffe63379..5dd6f419 100644 --- a/theHarvester/discovery/__init__.py +++ b/theHarvester/discovery/__init__.py @@ -6,7 +6,6 @@ __all__ = ['baidusearch', 'dogpilesearch', 'duckduckgosearch', 'exaleadsearch', - 'googlecertificates', 'googlesearch', 'huntersearch', 'intelxsearch', diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py index 9caacc8e..d5dc5b2d 100644 --- a/theHarvester/discovery/crtsh.py +++ b/theHarvester/discovery/crtsh.py @@ -1,6 +1,6 @@ -#from theHarvester.lib.core import * +from theHarvester.lib.core import * import requests - +import urllib3 class SearchCrtsh: @@ -10,13 +10,11 @@ class SearchCrtsh: def do_search(self): url = f'https://crt.sh/?q=%25.{self.word}&output=json' - request = requests.get(url) + headers = {'User-Agent': Core.get_user_agent()} + request = requests.get(url, params=headers, timeout=30) if request.ok: - try: - content = request.json() - data = set([dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in content]) - except ValueError as error: - print(f'Error when requesting data from crt.sh: {error}') + content = request.json() + data = set([dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in content]) return data def process(self): diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 497b96fa..1038a560 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -74,7 +74,6 @@ class Core: 'duckduckgo', 'github-code', 'google', - 'google-certificates', 'hunter', 'intelx', 'linkedin',