From 052bd3980c3f084489c59c24a4fcf609a007e2da Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sun, 13 Jan 2019 00:01:57 +0000 Subject: [PATCH 01/16] Fix conflict and start work on new cli switch handling --- theHarvester.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/theHarvester.py b/theHarvester.py index c82b1760..fd62655f 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse from discovery import * from discovery.constants import * from lib.core import * @@ -28,19 +29,19 @@ except ImportError: Core.banner() -def start(argv): - if len(sys.argv) < 4: - Core.usage() - sys.exit(1) - try: - opts, args = getopt.getopt(argv, "l:d:b:s:u:vf:nhcgpte:") - except getopt.GetoptError: - Core.usage() - sys.exit(1) +def start(): + parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon') + parser.add_argument('-d', '--domain', help='Company name or domain to search', required=True) + parser.add_argument('-t', '--type', help='Perform a DNS TLD expansion discovery') + parser.add_argument('-b', + help='Name server to use for the lookup, default is Googles', default='8.8.8.8') + + args = parser.parse_args() + try: db = stash.stash_manager() db.do_init() - except Exception as e: + except Exception: pass start = 0 host_ip = [] From 4e2dde57af546432aa4c3ad8125604d26b94403a Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sun, 13 Jan 2019 17:59:41 +0000 Subject: [PATCH 02/16] Fix merge conflict again --- theHarvester.py | 1 - 1 file changed, 1 deletion(-) diff --git a/theHarvester.py b/theHarvester.py index 6660651d..060a9dd5 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -8,7 +8,6 @@ from lib import htmlExport from lib import reportgraph from lib import statichtmlgenerator import datetime -import getopt import ipaddress import re import stash From 8452216bec27a210ffd46c3c701c7ae9caddca32 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Mon, 14 Jan 2019 02:11:23 +0000 Subject: [PATCH 03/16] Got some of the cli switches refactor implemented and working --- lib/core.py | 30 ++++ theHarvester.py | 392 ++++++++++++++++++++++++------------------------ 2 files changed, 222 insertions(+), 200 deletions(-) diff --git a/lib/core.py b/lib/core.py index 305368f0..02ba5c25 100644 --- a/lib/core.py +++ b/lib/core.py @@ -54,8 +54,38 @@ class Core: print((' ' + comm + ' -d acme.com -l 200 -b googleCSE -s 300')) print((' ' + comm + ' -d acme.edu -l 300 -b bing -h \n')) + @staticmethod + def get_supportedengines(): + supportedengines = set(['baidu', + 'bing', + 'bingapi', + 'censys', + 'crtsh', + 'cymon', + 'dogpile', + 'duckduckgo', + 'google', + 'googleCSE', + 'google-certificates', + 'google-profiles', + 'hunter', + 'linkedin', + 'netcraft', + 'pgp', + 'securityTrails', + 'threatcrowd', + 'trello', + 'twitter', + 'vhost', + 'virustotal', + 'yahoo', + 'all' + ]) + return supportedengines + @staticmethod def get_user_agent(): + """User-Agents from https://github.com/tamimibrahim17/List-of-user-agents""" user_agents = [ 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205', diff --git a/theHarvester.py b/theHarvester.py index 060a9dd5..b3b1b10e 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse from discovery import * from discovery.constants import * from lib.core import * @@ -28,19 +29,22 @@ except ImportError: Core.banner() -def start(argv): - if len(sys.argv) < 4: - Core.usage() - sys.exit(1) - try: - opts, args = getopt.getopt(argv, 'l:d:b:s:u:vf:nhcgpte:') - except getopt.GetoptError: - Core.usage() - sys.exit(1) +def start(): + parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon') + parser.add_argument('-d', '--domain', help='Company name or domain to search', required=True) + parser.add_argument('-t', '--type', help='Perform a DNS TLD expansion discovery') + parser.add_argument('-l', '--limit', help='limit the number of search results', default=500) + parser.add_argument('-b', '--source', help='''source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, + google, googleCSE, google-certificates, google-profiles, + hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd, + trello, twitter, vhost, virustotal, yahoo, all''') + + args = parser.parse_args() + try: db = stash.stash_manager() db.do_init() - except Exception as e: + except Exception: pass all_emails = [] @@ -55,7 +59,7 @@ def start(argv): full = [] google_dorking = False host_ip = [] - limit = 500 + limit = args.limit ports_scanning = False shodan = False start = 0 @@ -63,195 +67,183 @@ def start(argv): trello_info = ([], False) vhost = [] virtual = False + word = args.domain - for value in enumerate(opts): - opt = value[1][0] - arg = value[1][1] - opt = str(opt) - arg = str(arg) - if opt == '-l': - limit = int(arg) - elif opt == '-d': - word = arg - elif opt == '-g': - google_dorking = True - elif opt == '-s': - start = int(arg) - elif opt == '-v': - virtual = 'basic' - elif opt == '-f': - filename = arg - elif opt == '-n': - dnslookup = True - elif opt == '-c': - dnsbrute = True - elif opt == '-h': - shodan = True - elif opt == '-e': - dnsserver = arg - elif opt == '-p': - ports_scanning = True - elif opt == '-t': - dnstld = True - elif opt == '-b': - engines = set(arg.split(',')) - supportedengines = set(['baidu', 'bing', 'bingapi', 'censys', 'crtsh', 'cymon', 'dogpile', 'duckduckgo', - 'google', 'googleCSE', 'google-certificates', 'google-profiles', 'hunter', - 'linkedin', 'netcraft', 'pgp', 'securityTrails', 'threatcrowd', 'trello', - 'twitter', 'vhost', 'virustotal', 'yahoo', 'all']) - if set(engines).issubset(supportedengines): - print(f'\033[94m[*] Target domain: {word} \n \033[0m') - for engineitem in engines: - if engineitem == 'baidu': - print('\033[94m[*] Searching Baidu. \033[0m') - try: - search = baidusearch.SearchBaidu(word, limit) - search.process() - all_emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'baidu') - db.store_all(word, all_emails, 'email', 'baidu') - except Exception: + + # elif opt == '-g': + # google_dorking = True + # elif opt == '-s': + # start = int(arg) + # elif opt == '-v': + # virtual = 'basic' + # elif opt == '-f': + # filename = arg + # elif opt == '-n': + # dnslookup = True + # elif opt == '-c': + # dnsbrute = True + # elif opt == '-h': + # shodan = True + # elif opt == '-e': + # dnsserver = arg + # elif opt == '-p': + # ports_scanning = True + # elif opt == '-t': + # dnstld = True + engines = set(args.source.split(',')) + if set(engines).issubset(Core.get_supportedengines()): + print(f'\033[94m[*] Target domain: {word} \n \033[0m') + for engineitem in engines: + if engineitem == 'baidu': + print('\033[94m[*] Searching Baidu. \033[0m') + try: + search = baidusearch.SearchBaidu(word, limit) + search.process() + all_emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'baidu') + db.store_all(word, all_emails, 'email', 'baidu') + except Exception: + pass + + elif engineitem == 'bing' or engineitem == 'bingapi': + print('\033[94m[*] Searching Bing. \033[0m') + try: + search = bingsearch.SearchBing(word, limit, start) + if engineitem == 'bingapi': + bingapi = 'yes' + else: + bingapi = 'no' + search.process(bingapi) + all_emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'bing') + db.store_all(word, all_hosts, 'host', 'bing') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: pass - elif engineitem == 'bing' or engineitem == 'bingapi': - print('\033[94m[*] Searching Bing. \033[0m') - try: - search = bingsearch.SearchBing(word, limit, start) - if engineitem == 'bingapi': - bingapi = 'yes' - else: - bingapi = 'no' - search.process(bingapi) - all_emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'bing') - db.store_all(word, all_hosts, 'host', 'bing') - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - pass + elif engineitem == 'censys': + print('\033[94m[*] Searching Censys. \033[0m') + from discovery import censys + # Import locally or won't work + search = censys.SearchCensys(word, limit) + search.process() + all_ip = search.get_ipaddresses() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'censys') + db.store_all(word, all_ip, 'ip', 'censys') - elif engineitem == 'censys': - print('\033[94m[*] Searching Censys. \033[0m') - from discovery import censys - # Import locally or won't work - search = censys.SearchCensys(word, limit) - search.process() - all_ip = search.get_ipaddresses() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'censys') - db.store_all(word, all_ip, 'ip', 'censys') + elif engineitem == 'crtsh': + print('\033[94m[*] Searching CRT.sh. \033[0m') + search = crtsh.search_crtsh(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'CRTsh') - elif engineitem == 'crtsh': - print('\033[94m[*] Searching CRT.sh. \033[0m') - search = crtsh.search_crtsh(word) - search.process() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'CRTsh') + elif engineitem == 'cymon': + print('\033[94m[*] Searching Cymon. \033[0m') + from discovery import cymon + # Import locally or won't work. + search = cymon.search_cymon(word) + search.process() + all_ip = search.get_ipaddresses() + db = stash.stash_manager() + db.store_all(word, all_ip, 'ip', 'cymon') - elif engineitem == 'cymon': - print('\033[94m[*] Searching Cymon. \033[0m') - from discovery import cymon - # Import locally or won't work. - search = cymon.search_cymon(word) - search.process() - all_ip = search.get_ipaddresses() - db = stash.stash_manager() - db.store_all(word, all_ip, 'ip', 'cymon') + elif engineitem == 'dogpile': + print('\033[94m[*] Searching Dogpile. \033[0m') + search = dogpilesearch.SearchDogpile(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'dogpile') + db.store_all(word, all_hosts, 'host', 'dogpile') - elif engineitem == 'dogpile': - print('\033[94m[*] Searching Dogpile. \033[0m') - search = dogpilesearch.SearchDogpile(word, limit) - search.process() - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - all_emails.extend(emails) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'dogpile') - db.store_all(word, all_hosts, 'host', 'dogpile') + elif engineitem == 'duckduckgo': + print('\033[94m[*] Searching DuckDuckGo. \033[0m') + from discovery import duckduckgosearch + search = duckduckgosearch.SearchDuckDuckGo(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'duckduckgo') + db.store_all(word, all_hosts, 'host', 'duckduckgo') - elif engineitem == 'duckduckgo': - print('\033[94m[*] Searching DuckDuckGo. \033[0m') - from discovery import duckduckgosearch - search = duckduckgosearch.SearchDuckDuckGo(word, limit) - search.process() - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - all_emails.extend(emails) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'duckduckgo') - db.store_all(word, all_hosts, 'host', 'duckduckgo') + elif engineitem == 'google': + print('\033[94m[*] Searching Google. \033[0m') + search = googlesearch.search_google(word, limit, start) + search.process(google_dorking) + emails = filter(search.get_emails()) + all_emails.extend(emails) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'google') + db.store_all(word, all_emails, 'email', 'google') - elif engineitem == 'google': - print('\033[94m[*] Searching Google. \033[0m') - search = googlesearch.search_google(word, limit, start) - search.process(google_dorking) - emails = filter(search.get_emails()) - all_emails.extend(emails) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google') - db.store_all(word, all_emails, 'email', 'google') + elif engineitem == 'googleCSE': + print('\033[94m[*] Searching Google Custom Search. \033[0m') + try: + search = googleCSE.SearchGoogleCSE(word, limit, start) + search.process() + search.store_results() + all_emails = filter(search.get_emails()) + db = stash.stash_manager() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db.store_all(word, all_hosts, 'email', 'googleCSE') + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'googleCSE') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + pass - elif engineitem == 'googleCSE': - print('\033[94m[*] Searching Google Custom Search. \033[0m') - try: - search = googleCSE.SearchGoogleCSE(word, limit, start) - search.process() - search.store_results() - all_emails = filter(search.get_emails()) - db = stash.stash_manager() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db.store_all(word, all_hosts, 'email', 'googleCSE') - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'googleCSE') - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - pass + elif engineitem == 'google-certificates': + print('\033[94m[*] Searching Google Certificate transparency report. \033[0m') + search = googlecertificates.SearchGoogleCertificates(word, limit, start) + search.process() + hosts = filter(search.get_domains()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'google-certificates') - elif engineitem == 'google-certificates': - print('\033[94m[*] Searching Google Certificate transparency report. \033[0m') - search = googlecertificates.SearchGoogleCertificates(word, limit, start) - search.process() - hosts = filter(search.get_domains()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google-certificates') + elif engineitem == 'google-profiles': + print('\033[94m[*] Searching Google profiles. \033[0m') + search = googlesearch.search_google(word, limit, start) + search.process_profiles() + people = search.get_profiles() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'google-profile') - elif engineitem == 'google-profiles': - print('\033[94m[*] Searching Google profiles. \033[0m') - search = googlesearch.search_google(word, limit, start) - search.process_profiles() - people = search.get_profiles() - db = stash.stash_manager() - db.store_all(word, people, 'name', 'google-profile') - - if len(people) == 0: - print('\n[*] No users found.\n\n') - else: - print('\n[*] Users found: ' + str(len(people))) - print('---------------------') - for user in sorted(list(set(people))): - print(user) + if len(people) == 0: + print('\n[*] No users found.\n\n') + else: + print('\n[*] Users found: ' + str(len(people))) + print('---------------------') + for user in sorted(list(set(people))): + print(user) sys.exit(0) - elif engineitem == 'hunter': + elif engineitem == 'hunter': print('\033[94m[*] Searching Hunter. \033[0m') from discovery import huntersearch # Import locally or won't work. @@ -271,7 +263,7 @@ def start(argv): else: pass - elif engineitem == 'linkedin': + elif engineitem == 'linkedin': print('\033[94m[*] Searching Linkedin. \033[0m') search = linkedinsearch.SearchLinkedin(word, limit) search.process() @@ -288,7 +280,7 @@ def start(argv): print(user) sys.exit(0) - elif engineitem == 'netcraft': + elif engineitem == 'netcraft': print('\033[94m[*] Searching Netcraft. \033[0m') search = netcraft.SearchNetcraft(word) search.process() @@ -297,7 +289,7 @@ def start(argv): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'netcraft') - elif engineitem == 'pgp': + elif engineitem == 'pgp': print('\033[94m[*] Searching PGP key server. \033[0m') try: search = pgpsearch.SearchPgp(word) @@ -311,7 +303,7 @@ def start(argv): except Exception: pass - elif engineitem == 'securityTrails': + elif engineitem == 'securityTrails': print('\033[94m[*] Searching SecurityTrails. \033[0m') from discovery import securitytrailssearch try: @@ -331,7 +323,7 @@ def start(argv): else: pass - elif engineitem == 'threatcrowd': + elif engineitem == 'threatcrowd': print('\033[94m[*] Searching Threatcrowd. \033[0m') try: search = threatcrowd.search_threatcrowd(word) @@ -343,7 +335,7 @@ def start(argv): except Exception: pass - elif engineitem == 'trello': + elif engineitem == 'trello': print('\033[94m[*] Searching Trello. \033[0m') from discovery import trello # Import locally or won't work. @@ -359,7 +351,7 @@ def start(argv): db.store_all(word, hosts, 'host', 'trello') db.store_all(word, emails, 'email', 'trello') - elif engineitem == 'twitter': + elif engineitem == 'twitter': print('\033[94m[*] Searching Twitter. \033[0m') search = twittersearch.search_twitter(word, limit) search.process() @@ -378,7 +370,7 @@ def start(argv): # vhost - elif engineitem == 'virustotal': + elif engineitem == 'virustotal': print('\033[94m[*] Searching VirusTotal. \033[0m') search = virustotal.search_virustotal(word) search.process() @@ -387,7 +379,7 @@ def start(argv): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'virustotal') - elif engineitem == 'yahoo': + elif engineitem == 'yahoo': print('\033[94m[*] Searching Yahoo. \033[0m') search = yahoosearch.search_yahoo(word, limit) search.process() @@ -399,7 +391,7 @@ def start(argv): db.store_all(word, all_hosts, 'host', 'yahoo') db.store_all(word, all_emails, 'email', 'yahoo') - elif engineitem == 'all': + elif engineitem == 'all': print(('Full harvest on ' + word)) all_emails = [] all_hosts = [] @@ -610,9 +602,9 @@ def start(argv): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'yahoo') db.store_all(word, all_emails, 'email', 'yahoo') - else: - print('\033[93m[!] Invalid source.\n\n \033[0m') - sys.exit(1) + else: + print('\033[93m[!] Invalid source.\n\n \033[0m') + sys.exit(1) # Sanity check to see if all_emails and all_hosts are defined. try: @@ -915,7 +907,7 @@ def start(argv): if __name__ == '__main__': try: - start(sys.argv[1:]) + start() except KeyboardInterrupt: print('\n\n\033[93m[!] ctrl+c detected from user, quitting.\n\n \033[0m') except Exception: From 59b5b79a97c5f4ce17cd9d216b9c66e1a62c146a Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Wed, 16 Jan 2019 23:47:43 +0000 Subject: [PATCH 04/16] More addtions to have more sitches working with argparse --- theHarvester.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/theHarvester.py b/theHarvester.py index a2269138..38631e7d 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -32,12 +32,15 @@ Core.banner() def start(): parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon') parser.add_argument('-d', '--domain', help='Company name or domain to search', required=True) - parser.add_argument('-t', '--type', help='Perform a DNS TLD expansion discovery') + parser.add_argument('-t', '--dnstld', help='Perform a DNS TLD expansion discovery', default=True) parser.add_argument('-l', '--limit', help='limit the number of search results', default=500) + parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=True) + parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file') + parser.add_argument('-p', '--portscan', help='port scan the detected hosts and check for Takeovers (21,22,80,443,8080)', default=True) parser.add_argument('-b', '--source', help='''source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, google, googleCSE, google-certificates, google-profiles, hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd, - trello, twitter, vhost, virustotal, yahoo, all''') + trello, twitter, vhost, virustotal, yahoo, all''', required=True) args = parser.parse_args() @@ -54,14 +57,14 @@ def start(): dnsbrute = False dnslookup = False dnsserver = "" - dnstld = False - filename = "" + dnstld = args.dnstld + filename = args.filename full = [] google_dorking = False host_ip = [] limit = args.limit - ports_scanning = False - shodan = False + ports_scanning = args.portscan + shodan = args.shodan start = 0 takeover_check = False trello_info = ([], False) From 1096dda46d41348210309ba3ccc443be711f76f5 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Thu, 17 Jan 2019 00:06:41 +0000 Subject: [PATCH 05/16] Add more engines to the -b all flag --- theHarvester.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/theHarvester.py b/theHarvester.py index 38631e7d..910ca94b 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -449,8 +449,25 @@ def start(): db.store_all(word, all_hosts, 'host', 'CRTsh') # cymon + print('\033[94m[*] Searching Cymon. \033[0m') + from discovery import cymon + # Import locally or won't work. + search = cymon.search_cymon(word) + search.process() + all_ip = search.get_ipaddresses() + db = stash.stash_manager() + db.store_all(word, all_ip, 'ip', 'cymon') - # dogpile + print('\033[94m[*] Searching Dogpile. \033[0m') + search = dogpilesearch.SearchDogpile(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'dogpile') + db.store_all(word, all_hosts, 'host', 'dogpile') print('[*] Searching DuckDuckGo.') from discovery import duckduckgosearch @@ -518,7 +535,20 @@ def start(): else: pass - # linkedin + print('\033[94m[*] Searching Linkedin. \033[0m') + search = linkedinsearch.SearchLinkedin(word, limit) + search.process() + people = search.get_people() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'linkedin') + + if len(people) == 0: + print('\n[*] No users found.\n\n') + else: + print('\n[*] Users found: ' + str(len(people))) + print('---------------------') + for user in sorted(list(set(people))): + print(user) print('[*] Searching Netcraft.') search = netcraft.SearchNetcraft(word) From 2920a8588d8ec3f5ae3a4c27a839eca05f7bccce Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Thu, 17 Jan 2019 22:05:10 +0000 Subject: [PATCH 06/16] More flags added and @NotoriousRebel's fix added --- theHarvester.py | 72 ++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/theHarvester.py b/theHarvester.py index 910ca94b..472d5a0a 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -32,11 +32,14 @@ Core.banner() def start(): parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon') parser.add_argument('-d', '--domain', help='Company name or domain to search', required=True) - parser.add_argument('-t', '--dnstld', help='Perform a DNS TLD expansion discovery', default=True) - parser.add_argument('-l', '--limit', help='limit the number of search results', default=500) - parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=True) + parser.add_argument('-t', '--dnstld', help='Perform a DNS TLD expansion discovery, default False', default=False) + parser.add_argument('-l', '--limit', help='limit the number of search results, default 500', default=500) + parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts, default False', default=False) + parser.add_argument('-S', '--start', help='start with result number X (default: 0)', default=0) parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file') - parser.add_argument('-p', '--portscan', help='port scan the detected hosts and check for Takeovers (21,22,80,443,8080)', default=True) + parser.add_argument('-g', '--googleDork', help='use googledorks for google search, default False', default=False) + parser.add_argument('-n', '--dns', help='specify DNS server') + parser.add_argument('-p', '--portscan', help='port scan the detected hosts and check for Takeovers (21,22,80,443,8080) default False', default=False) parser.add_argument('-b', '--source', help='''source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, google, googleCSE, google-certificates, google-profiles, hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd, @@ -56,16 +59,16 @@ def start(): bingapi = 'yes' dnsbrute = False dnslookup = False - dnsserver = "" + dnsserver = args.dns dnstld = args.dnstld filename = args.filename full = [] - google_dorking = False + google_dorking = args.googleDork host_ip = [] limit = args.limit ports_scanning = args.portscan shodan = args.shodan - start = 0 + start = args.start takeover_check = False trello_info = ([], False) vhost = [] @@ -115,10 +118,11 @@ def start(): print('\033[94m[*] Searching Bing. \033[0m') try: search = bingsearch.SearchBing(word, limit, start) + bingapi = '' if engineitem == 'bingapi': - bingapi = 'yes' + bingapi += 'yes' else: - bingapi = 'no' + bingapi += 'no' search.process(bingapi) all_emails = filter(search.get_emails()) hosts = filter(search.get_hostnames()) @@ -202,23 +206,23 @@ def start(): db.store_all(word, all_emails, 'email', 'google') elif engineitem == 'googleCSE': - print('\033[94m[*] Searching Google Custom Search. \033[0m') - try: - search = googleCSE.SearchGoogleCSE(word, limit, start) - search.process() - search.store_results() - all_emails = filter(search.get_emails()) - db = stash.stash_manager() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db.store_all(word, all_hosts, 'email', 'googleCSE') - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'googleCSE') - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - pass + print('\033[94m[*] Searching Google Custom Search. \033[0m') + try: + search = googleCSE.SearchGoogleCSE(word, limit, start) + search.process() + search.store_results() + all_emails = filter(search.get_emails()) + db = stash.stash_manager() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db.store_all(word, all_hosts, 'email', 'googleCSE') + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'googleCSE') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + pass elif engineitem == 'google-certificates': print('\033[94m[*] Searching Google Certificate transparency report. \033[0m') @@ -275,9 +279,9 @@ def start(): db.store_all(word, people, 'name', 'linkedin') if len(people) == 0: - print('\n[*] No users found.\n\n') + print('\n[*] No users found Linkedin.\n\n') else: - print('\n[*] Users found: ' + str(len(people))) + print(f'\n[*] Users found: {len(people)}') print('---------------------') for user in sorted(list(set(people))): print(user) @@ -363,7 +367,7 @@ def start(): db.store_all(word, people, 'name', 'twitter') if len(people) == 0: - print('\n[*] No users found.\n\n') + print('\n[*] No users found on Twitter.\n\n') else: print('\n[*] Users found: ' + str(len(people))) print('---------------------') @@ -635,9 +639,9 @@ def start(): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'yahoo') db.store_all(word, all_emails, 'email', 'yahoo') - else: - print('\033[93m[!] Invalid source.\n\n \033[0m') - sys.exit(1) + else: + print('\033[93m[!] Invalid source.\n\n \033[0m') + sys.exit(1) # Sanity check to see if all_emails and all_hosts are defined. try: @@ -846,7 +850,7 @@ def start(): # Reporting if filename != "": try: - print('NEW REPORTING BEGINS.') + print('\n NEW REPORTING BEGINS.') db = stash.stash_manager() scanboarddata = db.getscanboarddata() latestscanresults = db.getlatestscanresults(word) @@ -946,4 +950,4 @@ if __name__ == '__main__': except Exception: import traceback print(traceback.print_exc()) - sys.exit(1) \ No newline at end of file + sys.exit(1) From 27ec4c1caea67ce0e873908fedf5ab856efde795 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sun, 20 Jan 2019 01:59:59 +0000 Subject: [PATCH 07/16] All cli switches have been impplemented and missing engines in the -b all, plus prep work for future rework --- discovery/crtsh.py | 2 +- lib/core.py | 294 ++++++++++++++++++++++++++++++++++----- stash.py => lib/stash.py | 0 theHarvester.py | 55 ++++---- 4 files changed, 288 insertions(+), 63 deletions(-) rename stash.py => lib/stash.py (100%) diff --git a/discovery/crtsh.py b/discovery/crtsh.py index 9e331280..7f78de02 100644 --- a/discovery/crtsh.py +++ b/discovery/crtsh.py @@ -22,7 +22,7 @@ class search_crtsh: print(e) try: params = {'User-Agent': Core.get_user_agent()} - r=requests.get(urly, headers=params) + r = requests.get(urly, headers=params) except Exception as e: print(e) links = self.get_info(r.text) diff --git a/lib/core.py b/lib/core.py index 02ba5c25..6aab9c34 100644 --- a/lib/core.py +++ b/lib/core.py @@ -1,8 +1,11 @@ # coding=utf-8 +#from discovery import * import os import random import sys +# from lib import stash +# import re class Core: @@ -22,38 +25,6 @@ class Core: print('* *') print('******************************************************************* \n\n \033[0m') - @staticmethod - def usage(): - comm = os.path.basename(sys.argv[0]) - - if os.path.dirname(sys.argv[0]) == os.getcwd(): - comm = './' + comm - - print('\033[94m Usage: theHarvester.py \n \033[0m') - print(' -d: company name or domain to search') - print(""" -b: source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, - google, googleCSE, google-certificates, google-profiles, - hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd, - trello, twitter, vhost, virustotal, yahoo, all""") - print(' -l: limit the number of search results') - print(' -s: start with result number X (default: 0)') - print(' -g: use Google Dorking instead of normal Google search') - print(' -h: use Shodan to query discovered hosts') - print(' -e: specify DNS server') - print(' -v: verify host name via DNS resolution and search for virtual hosts') - print(' -n: perform a DNS reverse query on all ranges discovered') - print(' -c: perform a DNS brute force on the domain') - print(' -t: perform a DNS TLD expansion discovery') - print(' -p: port scan the detected hosts and check for Takeovers (21,22,80,443,8080)') - print(' -f: save the results to an HTML and/or XML file') - print('\n\033[94m Examples: \033[0m') - print((' ' + comm + ' -d acme -l 200 -b linkedin')) - print((' ' + comm + ' -d acme.com -l 500 -b google -f myresults.html')) - print((' ' + comm + ' -d acme.com -b pgp, virustotal')) - print((' ' + comm + ' -d acme.com -l 100 -g -b google')) - print((' ' + comm + ' -d acme.com -l 200 -b googleCSE -s 300')) - print((' ' + comm + ' -d acme.edu -l 300 -b bing -h \n')) - @staticmethod def get_supportedengines(): supportedengines = set(['baidu', @@ -317,3 +288,262 @@ class Core: 'Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00' ] return random.choice(user_agents) + + # TODO use this method when -b all is called to replace lines 383-635 in theHarvester.py + # TODO and to find the best approch of getting the + # word, limit and start etc vars from the arguments and importing libs that are needed + # @staticmethod + # def engine_all_search(): + # print(('Full harvest on ' + word)) + # all_emails = [] + # all_hosts = [] + # try: + # print('[*] Searching Baidu.') + # search = baidusearch.SearchBaidu(word, limit) + # search.process() + # all_emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'baidu') + # db.store_all(word, all_emails, 'email', 'baidu') + # except Exception: + # pass + # + # print('[*] Searching Bing.') + # bingapi = 'no' + # search = bingsearch.SearchBing(word, limit, start) + # search.process(bingapi) + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'bing') + # all_emails.extend(emails) + # all_emails = sorted(set(all_emails)) + # db.store_all(word, all_emails, 'email', 'bing') + # + # print('[*] Searching Censys.') + # from discovery import censys + # search = censys.SearchCensys(word, limit) + # search.process() + # ips = search.get_ipaddresses() + # setips = set(ips) + # uniqueips = list(setips) # Remove duplicates. + # all_ip.extend(uniqueips) + # hosts = filter(search.get_hostnames()) + # sethosts = set(hosts) + # uniquehosts = list(sethosts) # Remove duplicates. + # all_hosts.extend(uniquehosts) + # db = stash.stash_manager() + # db.store_all(word, uniquehosts, 'host', 'censys') + # db.store_all(word, uniqueips, 'ip', 'censys') + # + # print('[*] Searching CRT.sh.') + # search = crtsh.search_crtsh(word) + # search.process() + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'CRTsh') + # + # # cymon + # print('\033[94m[*] Searching Cymon. \033[0m') + # from discovery import cymon + # # Import locally or won't work. + # search = cymon.search_cymon(word) + # search.process() + # all_ip = search.get_ipaddresses() + # db = stash.stash_manager() + # db.store_all(word, all_ip, 'ip', 'cymon') + # + # print('\033[94m[*] Searching Dogpile. \033[0m') + # search = dogpilesearch.SearchDogpile(word, limit) + # search.process() + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # all_emails.extend(emails) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'email', 'dogpile') + # db.store_all(word, all_hosts, 'host', 'dogpile') + # + # print('[*] Searching DuckDuckGo.') + # from discovery import duckduckgosearch + # search = duckduckgosearch.SearchDuckDuckGo(word, limit) + # search.process() + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # all_emails.extend(emails) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'email', 'duckduckgo') + # db.store_all(word, all_hosts, 'host', 'duckduckgo') + # + # print('[*] Searching Google.') + # search = googlesearch.search_google(word, limit, start) + # search.process(google_dorking) + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_emails.extend(emails) + # db = stash.stash_manager() + # db.store_all(word, all_emails, 'email', 'google') + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'google') + # + # print('[*] Searching Google Certificate transparency report.') + # search = googlecertificates.SearchGoogleCertificates(word, limit, start) + # search.process() + # domains = filter(search.get_domains()) + # all_hosts.extend(domains) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'google-certificates') + # + # try: + # print('[*] Searching Google profiles.') + # search = googlesearch.search_google(word, limit, start) + # search.process_profiles() + # people = search.get_profiles() + # db = stash.stash_manager() + # db.store_all(word, people, 'name', 'google-profile') + # print('\nUsers from Google profiles:') + # print('---------------------------') + # for users in people: + # print(users) + # except Exception: + # pass + # + # print('[*] Searching Hunter.') + # from discovery import huntersearch + # # Import locally. + # try: + # search = huntersearch.SearchHunter(word, limit, start) + # search.process() + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, hosts, 'host', 'hunter') + # all_emails.extend(emails) + # all_emails = sorted(set(all_emails)) + # db.store_all(word, all_emails, 'email', 'hunter') + # except Exception as e: + # if isinstance(e, MissingKey): + # print(e) + # else: + # pass + # + # print('\033[94m[*] Searching Linkedin. \033[0m') + # search = linkedinsearch.SearchLinkedin(word, limit) + # search.process() + # people = search.get_people() + # db = stash.stash_manager() + # db.store_all(word, people, 'name', 'linkedin') + # + # if len(people) == 0: + # print('\n[*] No users found.\n\n') + # else: + # print('\n[*] Users found: ' + str(len(people))) + # print('---------------------') + # for user in sorted(list(set(people))): + # print(user) + # + # print('[*] Searching Netcraft.') + # search = netcraft.SearchNetcraft(word) + # search.process() + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'netcraft') + # + # print('[*] Searching PGP key server.') + # try: + # search = pgpsearch.SearchPgp(word) + # search.process() + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # sethosts = set(hosts) + # uniquehosts = list(sethosts) # Remove duplicates. + # all_hosts.extend(uniquehosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'PGP') + # all_emails.extend(emails) + # db = stash.stash_manager() + # db.store_all(word, all_emails, 'email', 'PGP') + # except Exception: + # pass + # + # print('[*] Searching Threatcrowd.') + # try: + # search = threatcrowd.search_threatcrowd(word) + # search.process() + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'threatcrowd') + # except Exception: + # pass + # + # print('[*] Searching Trello.') + # from discovery import trello + # # Import locally or won't work. + # search = trello.search_trello(word, limit) + # search.process() + # emails = filter(search.get_emails()) + # all_emails.extend(emails) + # info = search.get_urls() + # hosts = filter(info[0]) + # trello_info = (info[1], True) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, hosts, 'host', 'trello') + # db.store_all(word, emails, 'email', 'trello') + # + # try: + # print('[*] Searching Twitter.') + # search = twittersearch.search_twitter(word, limit) + # search.process() + # people = search.get_people() + # db = stash.stash_manager() + # db.store_all(word, people, 'name', 'twitter') + # print('\nUsers from Twitter:') + # print('-------------------') + # for user in people: + # print(user) + # except Exception: + # pass + # + # print('\n[*] Virtual hosts:') + # print('------------------') + # for l in host_ip: + # search = bingsearch.SearchBing(l, limit, start) + # search.process_vhost() + # res = search.get_allhostnames() + # for x in res: + # x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x) + # x = re.sub('<', '', x) + # x = re.sub('>', '', x) + # print((l + '\t' + x)) + # vhost.append(l + ':' + x) + # full.append(l + ':' + x) + # vhost = sorted(set(vhost)) + # + # print('[*] Searching VirusTotal.') + # search = virustotal.search_virustotal(word) + # search.process() + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'virustotal') + # + # print('[*] Searching Yahoo.') + # search = yahoosearch.search_yahoo(word, limit) + # search.process() + # hosts = search.get_hostnames() + # emails = search.get_emails() + # all_hosts.extend(filter(hosts)) + # all_emails.extend(filter(emails)) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'yahoo') + # db.store_all(word, all_emails, 'email', 'yahoo') diff --git a/stash.py b/lib/stash.py similarity index 100% rename from stash.py rename to lib/stash.py diff --git a/theHarvester.py b/theHarvester.py index 472d5a0a..4c5ed936 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -8,10 +8,10 @@ from lib import hostchecker from lib import htmlExport from lib import reportgraph from lib import statichtmlgenerator +from lib import stash import datetime import ipaddress import re -import stash import time try: @@ -31,15 +31,18 @@ Core.banner() def start(): parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon') + parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain, default=False, params=True', default=False) parser.add_argument('-d', '--domain', help='Company name or domain to search', required=True) parser.add_argument('-t', '--dnstld', help='Perform a DNS TLD expansion discovery, default False', default=False) parser.add_argument('-l', '--limit', help='limit the number of search results, default 500', default=500) - parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts, default False', default=False) + parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts, default=False, params=True', default=False) parser.add_argument('-S', '--start', help='start with result number X (default: 0)', default=0) parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file') parser.add_argument('-g', '--googleDork', help='use googledorks for google search, default False', default=False) - parser.add_argument('-n', '--dns', help='specify DNS server') - parser.add_argument('-p', '--portscan', help='port scan the detected hosts and check for Takeovers (21,22,80,443,8080) default False', default=False) + parser.add_argument('-n', '--dns-lookup', help='Enable DNS server lookup, default=False, params=True', default=False) + parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup') + parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts params=basic, default=False', default=False) + parser.add_argument('-p', '--portscan', help='port scan the detected hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True', default=False) parser.add_argument('-b', '--source', help='''source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, google, googleCSE, google-certificates, google-profiles, hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd, @@ -57,9 +60,9 @@ def start(): all_hosts = [] all_ip = [] bingapi = 'yes' - dnsbrute = False - dnslookup = False - dnsserver = args.dns + dnsbrute = args.dns_brute + dnslookup = args.dns_lookup + dnsserver = args.dns_server dnstld = args.dnstld filename = args.filename full = [] @@ -72,30 +75,9 @@ def start(): takeover_check = False trello_info = ([], False) vhost = [] - virtual = False + virtual = args.virtual_host word = args.domain - - # elif opt == '-g': - # google_dorking = True - # elif opt == '-s': - # start = int(arg) - # elif opt == '-v': - # virtual = 'basic' - # elif opt == '-f': - # filename = arg - # elif opt == '-n': - # dnslookup = True - # elif opt == '-c': - # dnsbrute = True - # elif opt == '-h': - # shodan = True - # elif opt == '-e': - # dnsserver = arg - # elif opt == '-p': - # ports_scanning = True - # elif opt == '-t': - # dnstld = True engines = set(args.source.split(',')) if set(engines).issubset(Core.get_supportedengines()): print(f'\033[94m[*] Target domain: {word} \n \033[0m') @@ -619,7 +601,20 @@ def start(): except Exception: pass - # vhost + print('\n[*] Virtual hosts:') + print('------------------') + for l in host_ip: + search = bingsearch.SearchBing(l, limit, start) + search.process_vhost() + res = search.get_allhostnames() + for x in res: + x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x) + x = re.sub('<', '', x) + x = re.sub('>', '', x) + print((l + '\t' + x)) + vhost.append(l + ':' + x) + full.append(l + ':' + x) + vhost = sorted(set(vhost)) print('[*] Searching VirusTotal.') search = virustotal.search_virustotal(word) From d95097f6dded4604b224d7827a4794c9da96442e Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sun, 20 Jan 2019 03:02:29 +0000 Subject: [PATCH 08/16] Add python version check and fix bug due to moving the stash lib into lib --- README.md | 2 +- lib/core.py | 52 +++++++++++++++++++++++----------------------- lib/reportgraph.py | 40 +++++++++++++++++------------------ theHarvester.py | 4 ++++ 4 files changed, 51 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 7edf62ff..01d175a3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ * | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | * * \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| * * * -* theHarvester 3.0.6 v183 * +* theHarvester 3.0.6 v184 * * Coded by Christian Martorella * * Edge-Security Research * * cmartorella@edge-security.com * diff --git a/lib/core.py b/lib/core.py index 6aab9c34..189f766f 100644 --- a/lib/core.py +++ b/lib/core.py @@ -18,7 +18,7 @@ class Core: print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *") print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *") print('* *') - print('* theHarvester 3.0.6 v183 *') + print('* theHarvester 3.0.6 v184 *') print('* Coded by Christian Martorella *') print('* Edge-Security Research *') print('* cmartorella@edge-security.com *') @@ -27,31 +27,31 @@ class Core: @staticmethod def get_supportedengines(): - supportedengines = set(['baidu', - 'bing', - 'bingapi', - 'censys', - 'crtsh', - 'cymon', - 'dogpile', - 'duckduckgo', - 'google', - 'googleCSE', - 'google-certificates', - 'google-profiles', - 'hunter', - 'linkedin', - 'netcraft', - 'pgp', - 'securityTrails', - 'threatcrowd', - 'trello', - 'twitter', - 'vhost', - 'virustotal', - 'yahoo', - 'all' - ]) + supportedengines = {'baidu', + 'bing', + 'bingapi', + 'censys', + 'crtsh', + 'cymon', + 'dogpile', + 'duckduckgo', + 'google', + 'googleCSE', + 'google-certificates', + 'google-profiles', + 'hunter', + 'linkedin', + 'netcraft', + 'pgp', + 'securityTrails', + 'threatcrowd', + 'trello', + 'twitter', + 'vhost', + 'virustotal', + 'yahoo', + 'all' + } return supportedengines @staticmethod diff --git a/lib/reportgraph.py b/lib/reportgraph.py index 16641912..46e19d23 100644 --- a/lib/reportgraph.py +++ b/lib/reportgraph.py @@ -3,7 +3,7 @@ try: import plotly.graph_objs as go import plotly.plotly as py import plotly - import stash + from lib import stash try: db = stash.stash_manager() db.do_init() @@ -23,26 +23,26 @@ try: self.scattercountshodans = [] self.scattercountvhosts = [] - def drawlatestscangraph(self,domain,latestscandata): + def drawlatestscangraph(self, domain, latestscandata): try: - self.barcolumns= ['email', 'host', 'ip', 'shodan', 'vhost'] + self.barcolumns = ['email', 'host', 'ip', 'shodan', 'vhost'] self.bardata.append(latestscandata['email']) self.bardata.append(latestscandata['host']) self.bardata.append(latestscandata['ip']) self.bardata.append(latestscandata['shodan']) self.bardata.append(latestscandata['vhost']) - layout = dict(title = 'Latest scan - number of targets identified for ' + domain, - xaxis = dict(title = 'Targets'), - yaxis = dict(title = 'Hits'),) + layout = dict(title='Latest scan - number of targets identified for ' + domain, + xaxis = dict(title='Targets'), + yaxis = dict(title='Hits'),) barchartcode = plotly.offline.plot({ 'data': [go.Bar(x=self.barcolumns, y=self.bardata)], 'layout': layout, }, auto_open=False, include_plotlyjs=False, filename='report.html', output_type='div') return barchartcode except Exception as e: - print('Error generating HTML bar graph code for domain: ' + str(e)) + print(f'Error generating HTML bar graph code for domain: {e}') - def drawscattergraphscanhistory(self,domain,scanhistorydomain): + def drawscattergraphscanhistory(self, domain, scanhistorydomain): try: scandata = scanhistorydomain for i in scandata: @@ -56,20 +56,20 @@ try: trace0 = go.Scatter( x=self.scatterxdata, y=self.scattercounthosts, - mode = 'lines+markers', - name = 'hosts') + mode='lines+markers', + name='hosts') trace1 = go.Scatter( x=self.scatterxdata, y=self.scattercountips, - mode = 'lines+markers', - name = 'IP address') + mode='lines+markers', + name='IP address') trace2 = go.Scatter( x=self.scatterxdata, y=self.scattercountvhosts, - mode = 'lines+markers', - name = 'vhost') + mode='lines+markers', + name='vhost') trace3 = go.Scatter( x=self.scatterxdata, @@ -84,16 +84,16 @@ try: name='email') data = [trace0, trace1, trace2, trace3, trace4] - layout = dict(title = 'Scanning history for ' + domain, - xaxis = dict(title = 'Date'), - yaxis = dict(title = 'Results'), - ) + layout = dict(title='Scanning history for ' + domain, + xaxis=dict(title='Date'), + yaxis=dict(title='Results'), + ) scatterchartcode = plotly.offline.plot({ 'data': data, 'layout': layout}, auto_open=False, include_plotlyjs=False, filename='report.html', output_type='div') return scatterchartcode except Exception as e: - print('Error generating HTML for the historical graph for domain: ' + str(e)) + print(f'Error generating HTML for the historical graph for domain: {e}') except Exception as e: - print('Error in the reportgraph module: ' + str(e)) + print(f'Error in the reportgraph module: {e}') diff --git a/theHarvester.py b/theHarvester.py index 4c5ed936..52313d32 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -12,6 +12,7 @@ from lib import stash import datetime import ipaddress import re +from platform import python_version import time try: @@ -938,6 +939,9 @@ def start(): if __name__ == '__main__': + if python_version()[0:3] < '3.6': + print('\033[93m[!] Please make sure you have python 3.6+ installed, quitting.\033[0m') + sys.exit(1) try: start() except KeyboardInterrupt: From 9c9c849db4f088da52637b824927c863b69a17d5 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sun, 20 Jan 2019 16:13:34 +0000 Subject: [PATCH 09/16] Update readme with new instructions for install --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 01d175a3..26aa50af 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Add your keys to discovery/constants.py Dependencies: ------------- * Python 3.6 -* pip3 install -r requirements.txt +* python3 -m pip install -r requirements.txt Changelog in 3.0: ----------------- From 80fe3c879b92a62951abc2a353d90818c9767ca8 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sun, 20 Jan 2019 21:13:18 +0000 Subject: [PATCH 10/16] Implemented Matts fixes for the reporting, nice one Matt :) --- theHarvester.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/theHarvester.py b/theHarvester.py index 52313d32..bcdf4e35 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -35,10 +35,10 @@ def start(): parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain, default=False, params=True', default=False) parser.add_argument('-d', '--domain', help='Company name or domain to search', required=True) parser.add_argument('-t', '--dnstld', help='Perform a DNS TLD expansion discovery, default False', default=False) - parser.add_argument('-l', '--limit', help='limit the number of search results, default 500', default=500) + parser.add_argument('-l', '--limit', help='limit the number of search results, default 500', default=500, type=int) parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts, default=False, params=True', default=False) parser.add_argument('-S', '--start', help='start with result number X (default: 0)', default=0) - parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file') + parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) parser.add_argument('-g', '--googleDork', help='use googledorks for google search, default False', default=False) parser.add_argument('-n', '--dns-lookup', help='Enable DNS server lookup, default=False, params=True', default=False) parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup') @@ -78,7 +78,6 @@ def start(): vhost = [] virtual = args.virtual_host word = args.domain - engines = set(args.source.split(',')) if set(engines).issubset(Core.get_supportedengines()): print(f'\033[94m[*] Target domain: {word} \n \033[0m') @@ -106,13 +105,13 @@ def start(): bingapi += 'yes' else: bingapi += 'no' - search.process(bingapi) - all_emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'bing') - db.store_all(word, all_hosts, 'host', 'bing') + search.process(bingapi) + all_emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'bing') + db.store_all(word, all_hosts, 'host', 'bing') except Exception as e: if isinstance(e, MissingKey): print(e) @@ -801,16 +800,16 @@ def start(): # Shodan shodanres = [] - import texttable - tab = texttable.Texttable() - header = ['IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'] - tab.header(header) - tab.set_cols_align(['c', 'c', 'c', 'c', 'c']) - tab.set_cols_valign(['m', 'm', 'm', 'm', 'm']) - tab.set_chars(['-', '|', '+', '#']) - tab.set_cols_width([15, 20, 15, 15, 18]) - host_ip = list(set(host_ip)) if shodan is True: + import texttable + tab = texttable.Texttable() + header = ['IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'] + tab.header(header) + tab.set_cols_align(['c', 'c', 'c', 'c', 'c']) + tab.set_cols_valign(['m', 'm', 'm', 'm', 'm']) + tab.set_chars(['-', '|', '+', '#']) + tab.set_cols_width([15, 20, 15, 15, 18]) + host_ip = list(set(host_ip)) print('\n\n[*] Shodan DB search (passive):\n') try: for ip in host_ip: @@ -846,7 +845,7 @@ def start(): # Reporting if filename != "": try: - print('\n NEW REPORTING BEGINS.') + print('\nNEW REPORTING BEGINS.') db = stash.stash_manager() scanboarddata = db.getscanboarddata() latestscanresults = db.getlatestscanresults(word) From dabfa5e221d16fdd5268eb5261b67d3bc7e9cc04 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sun, 20 Jan 2019 21:42:18 +0000 Subject: [PATCH 11/16] Update requirements.txt --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index a8f05c29..a1b77d06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -beautifulsoup4>=4.7.0 -plotly>=3.4.2 -pytest>=4.0.2 +beautifulsoup4>=4.7.1 +plotly>=3.5.0 +pytest>=4.1.1 requests>=2.21.0 -texttable>=1.4.0 +texttable>=1.6.0 shodan>=1.10.0 \ No newline at end of file From 82eb13762d330f66aaaba0347a99b068248692a5 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Mon, 21 Jan 2019 00:32:58 +0000 Subject: [PATCH 12/16] Implemented new way of managing api keys --- discovery/bingsearch.py | 4 ++-- discovery/constants.py | 19 ------------------- discovery/googleCSE.py | 9 +++++---- discovery/huntersearch.py | 3 ++- discovery/securitytrailssearch.py | 3 ++- discovery/shodansearch.py | 3 ++- discovery/twittersearch.py | 2 +- lib/core.py | 31 +++++++++++++++++++++++++++++++ requirements.txt | 1 + 9 files changed, 46 insertions(+), 29 deletions(-) diff --git a/discovery/bingsearch.py b/discovery/bingsearch.py index 9c076ba9..cfb17147 100644 --- a/discovery/bingsearch.py +++ b/discovery/bingsearch.py @@ -16,7 +16,7 @@ class SearchBing: self.hostname = 'www.bing.com' self.quantity = '50' self.limit = int(limit) - self.bingApi = bingAPI_key + self.bingApi = Core.bing_key() self.counter = start def do_search(self): @@ -70,7 +70,7 @@ class SearchBing: def process(self, api): if api == 'yes': - if self.bingApi == "": + if self.bingApi is None: raise MissingKey(True) while self.counter < self.limit: if api == 'yes': diff --git a/discovery/constants.py b/discovery/constants.py index d59affdb..1e9fa8d5 100644 --- a/discovery/constants.py +++ b/discovery/constants.py @@ -1,27 +1,8 @@ -""" -Module that contains constants used across plugins. -Contains list of API keys, user agents, and a function to get random delay and user agent. -As well as a defined User Agent for Google Search. -User-Agents from: https://github.com/tamimibrahim17/List-of-user-agents -""" - import random googleUA = "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36" -bingAPI_key = '' - -googleCSEAPI_key = '' - -googleCSE_id = '' - -hunterAPI_key = '' - -securityTrailsAPI_key = '' - -shodanAPI_key = 'oCiMsgM6rQWqiTvPxFHYcExlZgg7wvTt' # This is the default key. - def filter(lst): """ diff --git a/discovery/googleCSE.py b/discovery/googleCSE.py index 32c5067f..68ccbc38 100644 --- a/discovery/googleCSE.py +++ b/discovery/googleCSE.py @@ -1,5 +1,6 @@ from discovery.constants import * from parsers import myparser +from lib.core import * import re import requests import sys @@ -19,11 +20,11 @@ class SearchGoogleCSE: self.quantity = "10" self.limit = limit self.counter = 1 - self.api_key = googleCSEAPI_key - if self.api_key == "": + self.api_key = Core.google_cse_key()['key'] + if self.api_key is None: raise MissingKey(True) - self.cse_id = googleCSE_id - if self.cse_id == "": + self.cse_id = Core.google_cse_key()['id'] + if self.cse_id is None: raise MissingKey(False) self.lowRange = start self.highRange = start + 100 diff --git a/discovery/huntersearch.py b/discovery/huntersearch.py index e4f6ab3f..c5923ae0 100644 --- a/discovery/huntersearch.py +++ b/discovery/huntersearch.py @@ -1,5 +1,6 @@ from discovery.constants import * from parsers import myparser +from lib.core import * import requests @@ -9,7 +10,7 @@ class SearchHunter: self.word = word self.limit = 100 self.start = start - self.key = hunterAPI_key + self.key = Core.hunter_key() if self.key == "": raise MissingKey(True) self.results = "" diff --git a/discovery/securitytrailssearch.py b/discovery/securitytrailssearch.py index 30cc57ec..dd7aff01 100644 --- a/discovery/securitytrailssearch.py +++ b/discovery/securitytrailssearch.py @@ -1,5 +1,6 @@ from discovery.constants import * from parsers import securitytrailsparser +from lib.core import * import requests import sys import time @@ -9,7 +10,7 @@ class search_securitytrail: def __init__(self, word): self.word = word - self.key = securityTrailsAPI_key + self.key = Core.security_trails_key() if self.key == "": raise MissingKey(True) self.results = "" diff --git a/discovery/shodansearch.py b/discovery/shodansearch.py index 7b1480e9..4f8713a7 100644 --- a/discovery/shodansearch.py +++ b/discovery/shodansearch.py @@ -1,4 +1,5 @@ from discovery.constants import * +from lib.core import * from shodan import Shodan from shodan import exception @@ -6,7 +7,7 @@ from shodan import exception class search_shodan: def __init__(self): - self.key = shodanAPI_key + self.key = Core.shodan_key() if self.key == '': raise MissingKey(True) self.api = Shodan(self.key) diff --git a/discovery/twittersearch.py b/discovery/twittersearch.py index 486c76b4..035a6966 100644 --- a/discovery/twittersearch.py +++ b/discovery/twittersearch.py @@ -24,7 +24,7 @@ class search_twitter: print(e) headers = {'User-Agent': Core.get_user_agent()} try: - r=requests.get(urly, headers=headers) + r = requests.get(urly, headers=headers) except Exception as e: print(e) self.results = r.text diff --git a/lib/core.py b/lib/core.py index 189f766f..4376d858 100644 --- a/lib/core.py +++ b/lib/core.py @@ -3,12 +3,43 @@ #from discovery import * import os import random +import yaml import sys # from lib import stash # import re class Core: + @staticmethod + def bing_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['bing']['key'] + + @staticmethod + def google_cse_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['googleCSE'] + + @staticmethod + def hunter_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['hunter']['key'] + + @staticmethod + def security_trails_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['securityTrails']['key'] + + @staticmethod + def shodan_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['shodan']['key'] + @staticmethod def banner(): print('\n\033[93m*******************************************************************') diff --git a/requirements.txt b/requirements.txt index a1b77d06..0d7cd374 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ beautifulsoup4>=4.7.1 +PyYaml==3.13 plotly>=3.5.0 pytest>=4.1.1 requests>=2.21.0 From 846e22efe22aeece596c106797c5db125c0b197c Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Mon, 21 Jan 2019 00:33:57 +0000 Subject: [PATCH 13/16] Add missing file from previous commit --- api-keys.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 api-keys.yaml diff --git a/api-keys.yaml b/api-keys.yaml new file mode 100644 index 00000000..866de4a1 --- /dev/null +++ b/api-keys.yaml @@ -0,0 +1,16 @@ +apikeys: + bing: + key: + + googleCSE: + key: + id: + + hunter: + key: + + securityTrails: + key: + + shodan: + key: oCiMsgM6rQWqiTvPxFHYcExlZgg7wvTt \ No newline at end of file From b6f250baebff3863202f2c6f5199b035ec48ea9f Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Mon, 21 Jan 2019 20:46:10 +0000 Subject: [PATCH 14/16] Done the changes that @leebaird mentioned --- README.md | 4 +-- discovery/googleCSE.py | 2 +- discovery/huntersearch.py | 2 +- discovery/securitytrailssearch.py | 2 +- discovery/shodansearch.py | 2 +- lib/core.py | 10 ++++---- lib/reportgraph.py | 33 +++++++++++------------- requirements.txt | 6 ++--- theHarvester.py | 42 +++++++++++++++---------------- 9 files changed, 50 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 26aa50af..c7d856ce 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ * | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | * * \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| * * * -* theHarvester 3.0.6 v184 * +* theHarvester 3.0.6 v206 * * Coded by Christian Martorella * * Edge-Security Research * * cmartorella@edge-security.com * @@ -82,7 +82,7 @@ Active: Modules that require an API key: -------------------------------- -Add your keys to discovery/constants.py +Add your keys to api-keys.yaml * googleCSE: API key and CSE ID * hunter: API key diff --git a/discovery/googleCSE.py b/discovery/googleCSE.py index 68ccbc38..a9404c1f 100644 --- a/discovery/googleCSE.py +++ b/discovery/googleCSE.py @@ -1,6 +1,6 @@ from discovery.constants import * -from parsers import myparser from lib.core import * +from parsers import myparser import re import requests import sys diff --git a/discovery/huntersearch.py b/discovery/huntersearch.py index c5923ae0..a3f3a98a 100644 --- a/discovery/huntersearch.py +++ b/discovery/huntersearch.py @@ -1,6 +1,6 @@ from discovery.constants import * -from parsers import myparser from lib.core import * +from parsers import myparser import requests diff --git a/discovery/securitytrailssearch.py b/discovery/securitytrailssearch.py index dd7aff01..5b4b53ed 100644 --- a/discovery/securitytrailssearch.py +++ b/discovery/securitytrailssearch.py @@ -1,6 +1,6 @@ from discovery.constants import * -from parsers import securitytrailsparser from lib.core import * +from parsers import securitytrailsparser import requests import sys import time diff --git a/discovery/shodansearch.py b/discovery/shodansearch.py index 4f8713a7..210ebde6 100644 --- a/discovery/shodansearch.py +++ b/discovery/shodansearch.py @@ -1,7 +1,7 @@ from discovery.constants import * from lib.core import * -from shodan import Shodan from shodan import exception +from shodan import Shodan class search_shodan: diff --git a/lib/core.py b/lib/core.py index 4376d858..ab2a8a54 100644 --- a/lib/core.py +++ b/lib/core.py @@ -1,12 +1,12 @@ # coding=utf-8 #from discovery import * +# from lib import stash import os import random -import yaml -import sys -# from lib import stash # import re +import sys +import yaml class Core: @@ -49,7 +49,7 @@ class Core: print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *") print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *") print('* *') - print('* theHarvester 3.0.6 v184 *') + print('* theHarvester 3.0.6 v206 *') print('* Coded by Christian Martorella *') print('* Edge-Security Research *') print('* cmartorella@edge-security.com *') @@ -87,7 +87,7 @@ class Core: @staticmethod def get_user_agent(): - """User-Agents from https://github.com/tamimibrahim17/List-of-user-agents""" + # User-Agents from https://github.com/tamimibrahim17/List-of-user-agents user_agents = [ 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205', diff --git a/lib/reportgraph.py b/lib/reportgraph.py index 46e19d23..a615d513 100644 --- a/lib/reportgraph.py +++ b/lib/reportgraph.py @@ -1,16 +1,16 @@ -try: - from datetime import datetime - import plotly.graph_objs as go - import plotly.plotly as py - import plotly - from lib import stash - try: - db = stash.stash_manager() - db.do_init() - except Exception as e: - pass +from datetime import datetime +from lib import stash +import plotly +import plotly.graph_objs as go +import plotly.plotly as py - class graphgenerator: +try: + db = stash.stash_manager() + db.do_init() +except Exception: + pass + + class GraphGenerator: def __init__(self, domain): self.domain = domain @@ -32,8 +32,8 @@ try: self.bardata.append(latestscandata['shodan']) self.bardata.append(latestscandata['vhost']) layout = dict(title='Latest scan - number of targets identified for ' + domain, - xaxis = dict(title='Targets'), - yaxis = dict(title='Hits'),) + xaxis=dict(title='Targets'), + yaxis=dict(title='Hits'),) barchartcode = plotly.offline.plot({ 'data': [go.Bar(x=self.barcolumns, y=self.bardata)], 'layout': layout, @@ -84,10 +84,7 @@ try: name='email') data = [trace0, trace1, trace2, trace3, trace4] - layout = dict(title='Scanning history for ' + domain, - xaxis=dict(title='Date'), - yaxis=dict(title='Results'), - ) + layout = dict(title='Scanning history for ' + domain, xaxis=dict(title='Date'), yaxis=dict(title='Results')) scatterchartcode = plotly.offline.plot({ 'data': data, 'layout': layout}, auto_open=False, include_plotlyjs=False, filename='report.html', output_type='div') diff --git a/requirements.txt b/requirements.txt index 0d7cd374..244b3b73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ beautifulsoup4>=4.7.1 -PyYaml==3.13 plotly>=3.5.0 +PyYaml==3.13 pytest>=4.1.1 requests>=2.21.0 -texttable>=1.6.0 -shodan>=1.10.0 \ No newline at end of file +shodan>=1.10.0 +texttable>=1.6.0 \ No newline at end of file diff --git a/theHarvester.py b/theHarvester.py index bcdf4e35..24946a0c 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -1,18 +1,18 @@ #!/usr/bin/env python3 -import argparse from discovery import * from discovery.constants import * -from lib.core import * from lib import hostchecker from lib import htmlExport from lib import reportgraph from lib import statichtmlgenerator from lib import stash +from lib.core import * +from platform import python_version +import argparse import datetime import ipaddress import re -from platform import python_version import time try: @@ -32,22 +32,22 @@ Core.banner() def start(): parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon') - parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain, default=False, params=True', default=False) - parser.add_argument('-d', '--domain', help='Company name or domain to search', required=True) - parser.add_argument('-t', '--dnstld', help='Perform a DNS TLD expansion discovery, default False', default=False) - parser.add_argument('-l', '--limit', help='limit the number of search results, default 500', default=500, type=int) + parser.add_argument('-d', '--domain', help='company name or domain to search', required=True) + parser.add_argument('-l', '--limit', help='limit the number of search results, default=500', default=500, type=int) + parser.add_argument('-S', '--start', help='start with result number X, default=0', default=0, type=int) + parser.add_argument('-g', '--google-dork', help='use Google Dorks for google search, default=False, params=True', default=False) + parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True', default=False) parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts, default=False, params=True', default=False) - parser.add_argument('-S', '--start', help='start with result number X (default: 0)', default=0) - parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) - parser.add_argument('-g', '--googleDork', help='use googledorks for google search, default False', default=False) - parser.add_argument('-n', '--dns-lookup', help='Enable DNS server lookup, default=False, params=True', default=False) - parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup') parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts params=basic, default=False', default=False) - parser.add_argument('-p', '--portscan', help='port scan the detected hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True', default=False) + parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup') + parser.add_argument('-t', '--dns-tld', help='perform a DNS TLD expansion discovery, default False', default=False) + parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default=False, params=True', default=False) + parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain, default=False, params=True', default=False) + parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) parser.add_argument('-b', '--source', help='''source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, - google, googleCSE, google-certificates, google-profiles, - hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd, - trello, twitter, vhost, virustotal, yahoo, all''', required=True) + google, googleCSE, google-certificates, google-profiles, + hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd, + trello, twitter, vhost, virustotal, yahoo, all''', required=True) args = parser.parse_args() @@ -64,13 +64,13 @@ def start(): dnsbrute = args.dns_brute dnslookup = args.dns_lookup dnsserver = args.dns_server - dnstld = args.dnstld + dnstld = args.dns_tld filename = args.filename full = [] - google_dorking = args.googleDork + google_dorking = args.google_dork host_ip = [] limit = args.limit - ports_scanning = args.portscan + ports_scanning = args.port_scan shodan = args.shodan start = args.start takeover_check = False @@ -349,7 +349,7 @@ def start(): db.store_all(word, people, 'name', 'twitter') if len(people) == 0: - print('\n[*] No users found on Twitter.\n\n') + print('\n[*] No users found.\n\n') else: print('\n[*] Users found: ' + str(len(people))) print('---------------------') @@ -857,7 +857,7 @@ def start(): HTMLcode = generator.beginhtml() HTMLcode += generator.generatelatestscanresults(latestscanresults) HTMLcode += generator.generatepreviousscanresults(previousscanresults) - graph = reportgraph.graphgenerator(word) + graph = reportgraph.GraphGenerator(word) HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata) HTMLcode += graph.drawscattergraphscanhistory(word, scanhistorydomain) HTMLcode += generator.generatepluginscanstatistics(pluginscanstatistics) From 995568177f3b44f574c97d9c4fca7bf986d4aba3 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Mon, 21 Jan 2019 23:59:54 +0000 Subject: [PATCH 15/16] Fix dogpile from crashing theharvester.py --- discovery/dogpilesearch.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/discovery/dogpilesearch.py b/discovery/dogpilesearch.py index 1fa7649b..63dcf385 100644 --- a/discovery/dogpilesearch.py +++ b/discovery/dogpilesearch.py @@ -17,14 +17,17 @@ class SearchDogpile: def do_search(self): # Dogpile is hardcoded to return 10 results. - url = 'http://' + self.server + "/search/web?qsi=" + str(self.counter) \ + url = 'https://' + self.server + "/search/web?qsi=" + str(self.counter) \ + "&q=\"%40" + self.word + "\"" headers = { 'Host': self.hostname, 'User-agent': Core.get_user_agent() } - h = requests.get(url=url, headers=headers) - self.total_results += h.text + try: + h = requests.get(url=url, headers=headers) + self.total_results += h.text + except requests.exceptions.ConnectionError: + pass def process(self): while self.counter <= self.limit and self.counter <= 1000: From 7aaf05859f13056b4aa59cf7306b9b089cbc0b74 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Tue, 22 Jan 2019 00:26:03 +0000 Subject: [PATCH 16/16] Update readme with mine and Lee's twitter --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c7d856ce..ef002669 100644 --- a/README.md +++ b/README.md @@ -110,8 +110,8 @@ Main contributors: ------- * Matthew Brown @NotoriousRebel * Janos Zold @Jzold -* Lee Baird @discoverscripts -* Jay Townsend @L1ghtn1ng +* Lee Baird @discoverscripts [![Twitter Follow](https://img.shields.io/twitter/follow/discoverscripts.svg?style=social&label=Follow)](https://twitter.com/discoverscripts) +* Jay Townsend @L1ghtn1ng [![Twitter Follow](https://img.shields.io/twitter/follow/jay_townsend1.svg?style=social&label=Follow)](https://twitter.com/jay_townsend1) Thanks: -------