Removed google-profiles and clean up.

.gitattributes

@@ -22,7 +22,6 @@
 *.pyd       binary
 *.pyo       binary
 
-# Note: .db, .p, and .pkl files are associated
+# Note: .db, .p, and .pkl files are associated with the python modules
-# with the python modules ``pickle``, ``dbm.*``,
+# ``pickle``, ``dbm.*``, # ``shelve``, ``marshal``, ``anydbm``, & ``bsddb``
-# ``shelve``, ``marshal``, ``anydbm``, & ``bsddb``
 # (among others).

.gitignore

@@ -4,7 +4,7 @@
 .html
 .vscode
 .xml
+api-keys.yaml
 debug_results.txt
 tests/myparser.py
 venv
-api-keys.yaml

theHarvester.py

@@ -5,16 +5,16 @@ from discovery.constants import *
 from lib import hostchecker
 from lib import htmlExport
 from lib import reportgraph
-from lib import statichtmlgenerator
 from lib import stash
+from lib import statichtmlgenerator
 from lib.core import *
 from platform import python_version
 import argparse
 import datetime
 import ipaddress
 import re
-import time
 import sys
+import time
 
 try:
     import bs4
@@ -32,11 +32,12 @@ Core.banner()
 
 
 def start():
-    parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon')
+    parser = argparse.ArgumentParser(description='theHarvester is used to gather open source intelligence (OSINT) on a\n'
+                                                 'company or domain.')
     parser.add_argument('-d', '--domain', help='company name or domain to search', required=True)
     parser.add_argument('-l', '--limit', help='limit the number of search results, default=500', default=500, type=int)
     parser.add_argument('-S', '--start', help='start with result number X, default=0', default=0, type=int)
-    parser.add_argument('-g', '--google-dork', help='use Google Dorks for google search', default=False, action='store_true')
+    parser.add_argument('-g', '--google-dork', help='use Google Dorks for Google search', default=False, action='store_true')
     parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True', default=False)
     parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False, action='store_true')
     parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts params=basic, default=False', default=False)
@@ -45,9 +46,9 @@ def start():
     parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default=False, params=True', default=False)
     parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true')
     parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str)
-    parser.add_argument('-b', '--source', help='''source: baidu, bing, bingapi, censys, crtsh, cymon,
+    parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, censys, crtsh, cymon,
                         dogpile, duckduckgo, google, googleCSE,
-                        google-certificates, google-profiles, hunter, intelx,
+                        google-certificates, hunter, intelx,
                         linkedin, netcraft, pgp, securityTrails, threatcrowd,
                         trello, twitter, vhost, virustotal, yahoo, all''')
     args = parser.parse_args()
@@ -78,6 +79,7 @@ def start():
     vhost = []
     virtual = args.virtual_host
     word = args.domain
+
     if args.source is not None:
         engines = set(args.source.split(','))
         if set(engines).issubset(Core.get_supportedengines()):
@@ -219,23 +221,6 @@ def start():
                     db = stash.stash_manager()
                     db.store_all(word, all_hosts, 'host', 'google-certificates')
 
-                elif engineitem == 'google-profiles':
-                    print('\033[94m[*] Searching Google profiles. \033[0m')
-                    search = googlesearch.search_google(word, limit, start)
-                    search.process_profiles()
-                    people = search.get_profiles()
-                    db = stash.stash_manager()
-                    db.store_all(word, people, 'name', 'google-profile')
-
-                    if len(people) == 0:
-                        print('\n[*] No users found.\n\n')
-                    else:
-                        print('\n[*] Users found: ' + str(len(people)))
-                        print('---------------------')
-                        for user in sorted(list(set(people))):
-                            print(user)
-                        sys.exit(0)
-
                 elif engineitem == 'hunter':
                     print('\033[94m[*] Searching Hunter. \033[0m')
                     from discovery import huntersearch
@@ -457,7 +442,6 @@ def start():
                     db = stash.stash_manager()
                     db.store_all(word, all_hosts, 'host', 'CRTsh')
 
-                    # cymon
                     print('\033[94m[*] Searching Cymon. \033[0m')
                     from discovery import cymon
                     # Import locally or won't work.
@@ -510,20 +494,6 @@ def start():
                     db = stash.stash_manager()
                     db.store_all(word, all_hosts, 'host', 'google-certificates')
 
-                    try:
-                        print('\033[94m[*] Searching Google profiles. \033[0m')
-                        search = googlesearch.search_google(word, limit, start)
-                        search.process_profiles()
-                        people = search.get_profiles()
-                        db = stash.stash_manager()
-                        db.store_all(word, people, 'name', 'google-profile')
-                        print('\nUsers from Google profiles:')
-                        print('---------------------------')
-                        for users in people:
-                            print(users)
-                    except Exception:
-                        pass
-
                     print('\033[94m[*] Searching Hunter. \033[0m')
                     from discovery import huntersearch
                     # Import locally.
@@ -907,7 +877,7 @@ def start():
     # Reporting
     if filename != "":
         try:
-            print('\nNEW REPORTING BEGINS.')
+            print('\n[*] Reporting started.')
             db = stash.stash_manager()
             scanboarddata = db.getscanboarddata()
             latestscanresults = db.getlatestscanresults(word)
@@ -933,7 +903,7 @@ def start():
             Html_file = open('report.html', 'w')
             Html_file.write(HTMLcode)
             Html_file.close()
-            print('NEW REPORTING FINISHED!')
+            print('[*] Reporting finished.')
             print('[*] Saving files.')
             html = htmlExport.htmlExport(
                 all_emails,
@@ -948,7 +918,8 @@ def start():
             save = html.writehtml()
         except Exception as e:
             print(e)
-            print('\n\033[93m[!] An error occurred creating the file.\033[0m')
+            print('\n\033[93m[!] An error occurred creating the output file.\n\n \033[0m')
+            sys.exit(1)
 
         try:
             filename = filename.split('.')[0] + '.xml'
@@ -1002,7 +973,7 @@ def start():
 
 if __name__ == '__main__':
     if python_version()[0:3] < '3.6':
-        print('\033[93m[!] Make sure you have Python 3.6+ installed, quitting. \033[0m')
+        print('\033[93m[!] Make sure you have Python 3.6+ installed, quitting.\n\n \033[0m')
         sys.exit(1)
     try:
         start()
@@ -1012,4 +983,4 @@ if __name__ == '__main__':
         import traceback
 
         print(traceback.print_exc())
-        sys.exit(1)
+        sys.exit(1)