remove google certs lookup and slight tweaks to crtsh

README.md

@@ -44,8 +44,6 @@ Passive:
 
 * google: Google search engine (Optional Google dorking.) - www.google.com
 
-* google-certificates: Google Certificate Transparency report 
-
 * hunter: Hunter search engine (Requires API key, see below.) - www.hunter.io
 
 * intelx: Intelx search engine (Requires API key, see below.) - www.intelx.io

theHarvester/__main__.py

@@ -54,7 +54,7 @@ def start():
     parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str)
     parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, censys, crtsh, dnsdumpster,
                         dogpile, duckduckgo, github-code, google, 
-                        google-certificates, hunter, intelx,
+                        hunter, intelx,
                         linkedin, netcraft, securityTrails, threatcrowd,
                         trello, twitter, vhost, virustotal, yahoo, all''')
     parser.add_argument('-x', '--exclude', help='exclude options when using all sources', type=str)
@@ -223,15 +223,6 @@ def start():
                     db.store_all(word, all_hosts, 'host', 'google')
                     db.store_all(word, all_emails, 'email', 'google')
 
-                elif engineitem == 'google-certificates':
-                    print('\033[94m[*] Searching Google Certificate transparency report. \033[0m')
-                    search = googlecertificates.SearchGoogleCertificates(word, limit, start)
-                    search.process()
-                    hosts = filter(search.get_domains())
-                    all_hosts.extend(hosts)
-                    db = stash.stash_manager()
-                    db.store_all(word, all_hosts, 'host', 'google-certificates')
-
                 elif engineitem == 'hunter':
                     print('\033[94m[*] Searching Hunter. \033[0m')
                     from theHarvester.discovery import huntersearch
@@ -488,14 +479,6 @@ def start():
                     db = stash.stash_manager()
                     db.store_all(word, all_hosts, 'host', 'google')
 
-                    print('\033[94m[*] Searching Google Certificate transparency report. \033[0m')
-                    search = googlecertificates.SearchGoogleCertificates(word, limit, start)
-                    search.process()
-                    domains = filter(search.get_domains())
-                    all_hosts.extend(domains)
-                    db = stash.stash_manager()
-                    db.store_all(word, all_hosts, 'host', 'google-certificates')
-
                     print('\033[94m[*] Searching Hunter. \033[0m')
                     from theHarvester.discovery import huntersearch
                     # Import locally.

theHarvester/discovery/__init__.py

@@ -6,7 +6,6 @@ __all__ = ['baidusearch',
            'dogpilesearch',
            'duckduckgosearch',
            'exaleadsearch',
-           'googlecertificates',
            'googlesearch',
            'huntersearch',
            'intelxsearch',

theHarvester/discovery/crtsh.py

@@ -1,6 +1,6 @@
-#from theHarvester.lib.core import *
+from theHarvester.lib.core import *
 import requests
-
+import urllib3
 
 class SearchCrtsh:
 
@@ -10,13 +10,11 @@ class SearchCrtsh:
         
     def do_search(self):
         url = f'https://crt.sh/?q=%25.{self.word}&output=json'
-        request = requests.get(url)
+        headers = {'User-Agent': Core.get_user_agent()}
+        request = requests.get(url, params=headers, timeout=30)
         if request.ok:
-            try:
-                content = request.json()
-                data = set([dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in content])
-            except ValueError as error:
-                print(f'Error when requesting data from crt.sh: {error}')
+            content = request.json()
+            data = set([dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in content])
         return data
 
     def process(self):

theHarvester/lib/core.py

@@ -74,7 +74,6 @@ class Core:
                             'duckduckgo',
                             'github-code',
                             'google',
-                            'google-certificates',
                             'hunter',
                             'intelx',
                             'linkedin',