diff --git a/README.md b/README.md index 7edf62ff..ef002669 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ * | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | * * \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| * * * -* theHarvester 3.0.6 v183 * +* theHarvester 3.0.6 v206 * * Coded by Christian Martorella * * Edge-Security Research * * cmartorella@edge-security.com * @@ -82,7 +82,7 @@ Active: Modules that require an API key: -------------------------------- -Add your keys to discovery/constants.py +Add your keys to api-keys.yaml * googleCSE: API key and CSE ID * hunter: API key @@ -92,7 +92,7 @@ Add your keys to discovery/constants.py Dependencies: ------------- * Python 3.6 -* pip3 install -r requirements.txt +* python3 -m pip install -r requirements.txt Changelog in 3.0: ----------------- @@ -110,8 +110,8 @@ Main contributors: ------- * Matthew Brown @NotoriousRebel * Janos Zold @Jzold -* Lee Baird @discoverscripts -* Jay Townsend @L1ghtn1ng +* Lee Baird @discoverscripts [![Twitter Follow](https://img.shields.io/twitter/follow/discoverscripts.svg?style=social&label=Follow)](https://twitter.com/discoverscripts) +* Jay Townsend @L1ghtn1ng [![Twitter Follow](https://img.shields.io/twitter/follow/jay_townsend1.svg?style=social&label=Follow)](https://twitter.com/jay_townsend1) Thanks: ------- diff --git a/api-keys.yaml b/api-keys.yaml new file mode 100644 index 00000000..866de4a1 --- /dev/null +++ b/api-keys.yaml @@ -0,0 +1,16 @@ +apikeys: + bing: + key: + + googleCSE: + key: + id: + + hunter: + key: + + securityTrails: + key: + + shodan: + key: oCiMsgM6rQWqiTvPxFHYcExlZgg7wvTt \ No newline at end of file diff --git a/discovery/bingsearch.py b/discovery/bingsearch.py index 9c076ba9..cfb17147 100644 --- a/discovery/bingsearch.py +++ b/discovery/bingsearch.py @@ -16,7 +16,7 @@ class SearchBing: self.hostname = 'www.bing.com' self.quantity = '50' self.limit = int(limit) - self.bingApi = bingAPI_key + self.bingApi = Core.bing_key() self.counter = start def do_search(self): @@ -70,7 +70,7 @@ class SearchBing: def process(self, api): if api == 'yes': - if self.bingApi == "": + if self.bingApi is None: raise MissingKey(True) while self.counter < self.limit: if api == 'yes': diff --git a/discovery/constants.py b/discovery/constants.py index d59affdb..1e9fa8d5 100644 --- a/discovery/constants.py +++ b/discovery/constants.py @@ -1,27 +1,8 @@ -""" -Module that contains constants used across plugins. -Contains list of API keys, user agents, and a function to get random delay and user agent. -As well as a defined User Agent for Google Search. -User-Agents from: https://github.com/tamimibrahim17/List-of-user-agents -""" - import random googleUA = "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36" -bingAPI_key = '' - -googleCSEAPI_key = '' - -googleCSE_id = '' - -hunterAPI_key = '' - -securityTrailsAPI_key = '' - -shodanAPI_key = 'oCiMsgM6rQWqiTvPxFHYcExlZgg7wvTt' # This is the default key. - def filter(lst): """ diff --git a/discovery/crtsh.py b/discovery/crtsh.py index 9e331280..7f78de02 100644 --- a/discovery/crtsh.py +++ b/discovery/crtsh.py @@ -22,7 +22,7 @@ class search_crtsh: print(e) try: params = {'User-Agent': Core.get_user_agent()} - r=requests.get(urly, headers=params) + r = requests.get(urly, headers=params) except Exception as e: print(e) links = self.get_info(r.text) diff --git a/discovery/dogpilesearch.py b/discovery/dogpilesearch.py index 1fa7649b..63dcf385 100644 --- a/discovery/dogpilesearch.py +++ b/discovery/dogpilesearch.py @@ -17,14 +17,17 @@ class SearchDogpile: def do_search(self): # Dogpile is hardcoded to return 10 results. - url = 'http://' + self.server + "/search/web?qsi=" + str(self.counter) \ + url = 'https://' + self.server + "/search/web?qsi=" + str(self.counter) \ + "&q=\"%40" + self.word + "\"" headers = { 'Host': self.hostname, 'User-agent': Core.get_user_agent() } - h = requests.get(url=url, headers=headers) - self.total_results += h.text + try: + h = requests.get(url=url, headers=headers) + self.total_results += h.text + except requests.exceptions.ConnectionError: + pass def process(self): while self.counter <= self.limit and self.counter <= 1000: diff --git a/discovery/googleCSE.py b/discovery/googleCSE.py index 32c5067f..a9404c1f 100644 --- a/discovery/googleCSE.py +++ b/discovery/googleCSE.py @@ -1,4 +1,5 @@ from discovery.constants import * +from lib.core import * from parsers import myparser import re import requests @@ -19,11 +20,11 @@ class SearchGoogleCSE: self.quantity = "10" self.limit = limit self.counter = 1 - self.api_key = googleCSEAPI_key - if self.api_key == "": + self.api_key = Core.google_cse_key()['key'] + if self.api_key is None: raise MissingKey(True) - self.cse_id = googleCSE_id - if self.cse_id == "": + self.cse_id = Core.google_cse_key()['id'] + if self.cse_id is None: raise MissingKey(False) self.lowRange = start self.highRange = start + 100 diff --git a/discovery/huntersearch.py b/discovery/huntersearch.py index e4f6ab3f..a3f3a98a 100644 --- a/discovery/huntersearch.py +++ b/discovery/huntersearch.py @@ -1,4 +1,5 @@ from discovery.constants import * +from lib.core import * from parsers import myparser import requests @@ -9,7 +10,7 @@ class SearchHunter: self.word = word self.limit = 100 self.start = start - self.key = hunterAPI_key + self.key = Core.hunter_key() if self.key == "": raise MissingKey(True) self.results = "" diff --git a/discovery/securitytrailssearch.py b/discovery/securitytrailssearch.py index 30cc57ec..5b4b53ed 100644 --- a/discovery/securitytrailssearch.py +++ b/discovery/securitytrailssearch.py @@ -1,4 +1,5 @@ from discovery.constants import * +from lib.core import * from parsers import securitytrailsparser import requests import sys @@ -9,7 +10,7 @@ class search_securitytrail: def __init__(self, word): self.word = word - self.key = securityTrailsAPI_key + self.key = Core.security_trails_key() if self.key == "": raise MissingKey(True) self.results = "" diff --git a/discovery/shodansearch.py b/discovery/shodansearch.py index 7b1480e9..210ebde6 100644 --- a/discovery/shodansearch.py +++ b/discovery/shodansearch.py @@ -1,12 +1,13 @@ from discovery.constants import * -from shodan import Shodan +from lib.core import * from shodan import exception +from shodan import Shodan class search_shodan: def __init__(self): - self.key = shodanAPI_key + self.key = Core.shodan_key() if self.key == '': raise MissingKey(True) self.api = Shodan(self.key) diff --git a/discovery/twittersearch.py b/discovery/twittersearch.py index 486c76b4..035a6966 100644 --- a/discovery/twittersearch.py +++ b/discovery/twittersearch.py @@ -24,7 +24,7 @@ class search_twitter: print(e) headers = {'User-Agent': Core.get_user_agent()} try: - r=requests.get(urly, headers=headers) + r = requests.get(urly, headers=headers) except Exception as e: print(e) self.results = r.text diff --git a/lib/core.py b/lib/core.py index 305368f0..ab2a8a54 100644 --- a/lib/core.py +++ b/lib/core.py @@ -1,11 +1,45 @@ # coding=utf-8 +#from discovery import * +# from lib import stash import os import random +# import re import sys +import yaml class Core: + @staticmethod + def bing_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['bing']['key'] + + @staticmethod + def google_cse_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['googleCSE'] + + @staticmethod + def hunter_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['hunter']['key'] + + @staticmethod + def security_trails_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['securityTrails']['key'] + + @staticmethod + def shodan_key(): + with open('api-keys.yaml', 'r') as api_keys: + keys = yaml.safe_load(api_keys) + return keys['apikeys']['shodan']['key'] + @staticmethod def banner(): print('\n\033[93m*******************************************************************') @@ -15,7 +49,7 @@ class Core: print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *") print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *") print('* *') - print('* theHarvester 3.0.6 v183 *') + print('* theHarvester 3.0.6 v206 *') print('* Coded by Christian Martorella *') print('* Edge-Security Research *') print('* cmartorella@edge-security.com *') @@ -23,39 +57,37 @@ class Core: print('******************************************************************* \n\n \033[0m') @staticmethod - def usage(): - comm = os.path.basename(sys.argv[0]) - - if os.path.dirname(sys.argv[0]) == os.getcwd(): - comm = './' + comm - - print('\033[94m Usage: theHarvester.py \n \033[0m') - print(' -d: company name or domain to search') - print(""" -b: source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, - google, googleCSE, google-certificates, google-profiles, - hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd, - trello, twitter, vhost, virustotal, yahoo, all""") - print(' -l: limit the number of search results') - print(' -s: start with result number X (default: 0)') - print(' -g: use Google Dorking instead of normal Google search') - print(' -h: use Shodan to query discovered hosts') - print(' -e: specify DNS server') - print(' -v: verify host name via DNS resolution and search for virtual hosts') - print(' -n: perform a DNS reverse query on all ranges discovered') - print(' -c: perform a DNS brute force on the domain') - print(' -t: perform a DNS TLD expansion discovery') - print(' -p: port scan the detected hosts and check for Takeovers (21,22,80,443,8080)') - print(' -f: save the results to an HTML and/or XML file') - print('\n\033[94m Examples: \033[0m') - print((' ' + comm + ' -d acme -l 200 -b linkedin')) - print((' ' + comm + ' -d acme.com -l 500 -b google -f myresults.html')) - print((' ' + comm + ' -d acme.com -b pgp, virustotal')) - print((' ' + comm + ' -d acme.com -l 100 -g -b google')) - print((' ' + comm + ' -d acme.com -l 200 -b googleCSE -s 300')) - print((' ' + comm + ' -d acme.edu -l 300 -b bing -h \n')) + def get_supportedengines(): + supportedengines = {'baidu', + 'bing', + 'bingapi', + 'censys', + 'crtsh', + 'cymon', + 'dogpile', + 'duckduckgo', + 'google', + 'googleCSE', + 'google-certificates', + 'google-profiles', + 'hunter', + 'linkedin', + 'netcraft', + 'pgp', + 'securityTrails', + 'threatcrowd', + 'trello', + 'twitter', + 'vhost', + 'virustotal', + 'yahoo', + 'all' + } + return supportedengines @staticmethod def get_user_agent(): + # User-Agents from https://github.com/tamimibrahim17/List-of-user-agents user_agents = [ 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205', @@ -287,3 +319,262 @@ class Core: 'Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00' ] return random.choice(user_agents) + + # TODO use this method when -b all is called to replace lines 383-635 in theHarvester.py + # TODO and to find the best approch of getting the + # word, limit and start etc vars from the arguments and importing libs that are needed + # @staticmethod + # def engine_all_search(): + # print(('Full harvest on ' + word)) + # all_emails = [] + # all_hosts = [] + # try: + # print('[*] Searching Baidu.') + # search = baidusearch.SearchBaidu(word, limit) + # search.process() + # all_emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'baidu') + # db.store_all(word, all_emails, 'email', 'baidu') + # except Exception: + # pass + # + # print('[*] Searching Bing.') + # bingapi = 'no' + # search = bingsearch.SearchBing(word, limit, start) + # search.process(bingapi) + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'bing') + # all_emails.extend(emails) + # all_emails = sorted(set(all_emails)) + # db.store_all(word, all_emails, 'email', 'bing') + # + # print('[*] Searching Censys.') + # from discovery import censys + # search = censys.SearchCensys(word, limit) + # search.process() + # ips = search.get_ipaddresses() + # setips = set(ips) + # uniqueips = list(setips) # Remove duplicates. + # all_ip.extend(uniqueips) + # hosts = filter(search.get_hostnames()) + # sethosts = set(hosts) + # uniquehosts = list(sethosts) # Remove duplicates. + # all_hosts.extend(uniquehosts) + # db = stash.stash_manager() + # db.store_all(word, uniquehosts, 'host', 'censys') + # db.store_all(word, uniqueips, 'ip', 'censys') + # + # print('[*] Searching CRT.sh.') + # search = crtsh.search_crtsh(word) + # search.process() + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'CRTsh') + # + # # cymon + # print('\033[94m[*] Searching Cymon. \033[0m') + # from discovery import cymon + # # Import locally or won't work. + # search = cymon.search_cymon(word) + # search.process() + # all_ip = search.get_ipaddresses() + # db = stash.stash_manager() + # db.store_all(word, all_ip, 'ip', 'cymon') + # + # print('\033[94m[*] Searching Dogpile. \033[0m') + # search = dogpilesearch.SearchDogpile(word, limit) + # search.process() + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # all_emails.extend(emails) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'email', 'dogpile') + # db.store_all(word, all_hosts, 'host', 'dogpile') + # + # print('[*] Searching DuckDuckGo.') + # from discovery import duckduckgosearch + # search = duckduckgosearch.SearchDuckDuckGo(word, limit) + # search.process() + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # all_emails.extend(emails) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'email', 'duckduckgo') + # db.store_all(word, all_hosts, 'host', 'duckduckgo') + # + # print('[*] Searching Google.') + # search = googlesearch.search_google(word, limit, start) + # search.process(google_dorking) + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_emails.extend(emails) + # db = stash.stash_manager() + # db.store_all(word, all_emails, 'email', 'google') + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'google') + # + # print('[*] Searching Google Certificate transparency report.') + # search = googlecertificates.SearchGoogleCertificates(word, limit, start) + # search.process() + # domains = filter(search.get_domains()) + # all_hosts.extend(domains) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'google-certificates') + # + # try: + # print('[*] Searching Google profiles.') + # search = googlesearch.search_google(word, limit, start) + # search.process_profiles() + # people = search.get_profiles() + # db = stash.stash_manager() + # db.store_all(word, people, 'name', 'google-profile') + # print('\nUsers from Google profiles:') + # print('---------------------------') + # for users in people: + # print(users) + # except Exception: + # pass + # + # print('[*] Searching Hunter.') + # from discovery import huntersearch + # # Import locally. + # try: + # search = huntersearch.SearchHunter(word, limit, start) + # search.process() + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, hosts, 'host', 'hunter') + # all_emails.extend(emails) + # all_emails = sorted(set(all_emails)) + # db.store_all(word, all_emails, 'email', 'hunter') + # except Exception as e: + # if isinstance(e, MissingKey): + # print(e) + # else: + # pass + # + # print('\033[94m[*] Searching Linkedin. \033[0m') + # search = linkedinsearch.SearchLinkedin(word, limit) + # search.process() + # people = search.get_people() + # db = stash.stash_manager() + # db.store_all(word, people, 'name', 'linkedin') + # + # if len(people) == 0: + # print('\n[*] No users found.\n\n') + # else: + # print('\n[*] Users found: ' + str(len(people))) + # print('---------------------') + # for user in sorted(list(set(people))): + # print(user) + # + # print('[*] Searching Netcraft.') + # search = netcraft.SearchNetcraft(word) + # search.process() + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'netcraft') + # + # print('[*] Searching PGP key server.') + # try: + # search = pgpsearch.SearchPgp(word) + # search.process() + # emails = filter(search.get_emails()) + # hosts = filter(search.get_hostnames()) + # sethosts = set(hosts) + # uniquehosts = list(sethosts) # Remove duplicates. + # all_hosts.extend(uniquehosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'PGP') + # all_emails.extend(emails) + # db = stash.stash_manager() + # db.store_all(word, all_emails, 'email', 'PGP') + # except Exception: + # pass + # + # print('[*] Searching Threatcrowd.') + # try: + # search = threatcrowd.search_threatcrowd(word) + # search.process() + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'threatcrowd') + # except Exception: + # pass + # + # print('[*] Searching Trello.') + # from discovery import trello + # # Import locally or won't work. + # search = trello.search_trello(word, limit) + # search.process() + # emails = filter(search.get_emails()) + # all_emails.extend(emails) + # info = search.get_urls() + # hosts = filter(info[0]) + # trello_info = (info[1], True) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, hosts, 'host', 'trello') + # db.store_all(word, emails, 'email', 'trello') + # + # try: + # print('[*] Searching Twitter.') + # search = twittersearch.search_twitter(word, limit) + # search.process() + # people = search.get_people() + # db = stash.stash_manager() + # db.store_all(word, people, 'name', 'twitter') + # print('\nUsers from Twitter:') + # print('-------------------') + # for user in people: + # print(user) + # except Exception: + # pass + # + # print('\n[*] Virtual hosts:') + # print('------------------') + # for l in host_ip: + # search = bingsearch.SearchBing(l, limit, start) + # search.process_vhost() + # res = search.get_allhostnames() + # for x in res: + # x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x) + # x = re.sub('<', '', x) + # x = re.sub('>', '', x) + # print((l + '\t' + x)) + # vhost.append(l + ':' + x) + # full.append(l + ':' + x) + # vhost = sorted(set(vhost)) + # + # print('[*] Searching VirusTotal.') + # search = virustotal.search_virustotal(word) + # search.process() + # hosts = filter(search.get_hostnames()) + # all_hosts.extend(hosts) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'virustotal') + # + # print('[*] Searching Yahoo.') + # search = yahoosearch.search_yahoo(word, limit) + # search.process() + # hosts = search.get_hostnames() + # emails = search.get_emails() + # all_hosts.extend(filter(hosts)) + # all_emails.extend(filter(emails)) + # db = stash.stash_manager() + # db.store_all(word, all_hosts, 'host', 'yahoo') + # db.store_all(word, all_emails, 'email', 'yahoo') diff --git a/lib/reportgraph.py b/lib/reportgraph.py index 16641912..a615d513 100644 --- a/lib/reportgraph.py +++ b/lib/reportgraph.py @@ -1,16 +1,16 @@ -try: - from datetime import datetime - import plotly.graph_objs as go - import plotly.plotly as py - import plotly - import stash - try: - db = stash.stash_manager() - db.do_init() - except Exception as e: - pass +from datetime import datetime +from lib import stash +import plotly +import plotly.graph_objs as go +import plotly.plotly as py - class graphgenerator: +try: + db = stash.stash_manager() + db.do_init() +except Exception: + pass + + class GraphGenerator: def __init__(self, domain): self.domain = domain @@ -23,26 +23,26 @@ try: self.scattercountshodans = [] self.scattercountvhosts = [] - def drawlatestscangraph(self,domain,latestscandata): + def drawlatestscangraph(self, domain, latestscandata): try: - self.barcolumns= ['email', 'host', 'ip', 'shodan', 'vhost'] + self.barcolumns = ['email', 'host', 'ip', 'shodan', 'vhost'] self.bardata.append(latestscandata['email']) self.bardata.append(latestscandata['host']) self.bardata.append(latestscandata['ip']) self.bardata.append(latestscandata['shodan']) self.bardata.append(latestscandata['vhost']) - layout = dict(title = 'Latest scan - number of targets identified for ' + domain, - xaxis = dict(title = 'Targets'), - yaxis = dict(title = 'Hits'),) + layout = dict(title='Latest scan - number of targets identified for ' + domain, + xaxis=dict(title='Targets'), + yaxis=dict(title='Hits'),) barchartcode = plotly.offline.plot({ 'data': [go.Bar(x=self.barcolumns, y=self.bardata)], 'layout': layout, }, auto_open=False, include_plotlyjs=False, filename='report.html', output_type='div') return barchartcode except Exception as e: - print('Error generating HTML bar graph code for domain: ' + str(e)) + print(f'Error generating HTML bar graph code for domain: {e}') - def drawscattergraphscanhistory(self,domain,scanhistorydomain): + def drawscattergraphscanhistory(self, domain, scanhistorydomain): try: scandata = scanhistorydomain for i in scandata: @@ -56,20 +56,20 @@ try: trace0 = go.Scatter( x=self.scatterxdata, y=self.scattercounthosts, - mode = 'lines+markers', - name = 'hosts') + mode='lines+markers', + name='hosts') trace1 = go.Scatter( x=self.scatterxdata, y=self.scattercountips, - mode = 'lines+markers', - name = 'IP address') + mode='lines+markers', + name='IP address') trace2 = go.Scatter( x=self.scatterxdata, y=self.scattercountvhosts, - mode = 'lines+markers', - name = 'vhost') + mode='lines+markers', + name='vhost') trace3 = go.Scatter( x=self.scatterxdata, @@ -84,16 +84,13 @@ try: name='email') data = [trace0, trace1, trace2, trace3, trace4] - layout = dict(title = 'Scanning history for ' + domain, - xaxis = dict(title = 'Date'), - yaxis = dict(title = 'Results'), - ) + layout = dict(title='Scanning history for ' + domain, xaxis=dict(title='Date'), yaxis=dict(title='Results')) scatterchartcode = plotly.offline.plot({ 'data': data, 'layout': layout}, auto_open=False, include_plotlyjs=False, filename='report.html', output_type='div') return scatterchartcode except Exception as e: - print('Error generating HTML for the historical graph for domain: ' + str(e)) + print(f'Error generating HTML for the historical graph for domain: {e}') except Exception as e: - print('Error in the reportgraph module: ' + str(e)) + print(f'Error in the reportgraph module: {e}') diff --git a/stash.py b/lib/stash.py similarity index 100% rename from stash.py rename to lib/stash.py diff --git a/requirements.txt b/requirements.txt index a8f05c29..244b3b73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ -beautifulsoup4>=4.7.0 -plotly>=3.4.2 -pytest>=4.0.2 +beautifulsoup4>=4.7.1 +plotly>=3.5.0 +PyYaml==3.13 +pytest>=4.1.1 requests>=2.21.0 -texttable>=1.4.0 -shodan>=1.10.0 \ No newline at end of file +shodan>=1.10.0 +texttable>=1.6.0 \ No newline at end of file diff --git a/theHarvester.py b/theHarvester.py index 91281552..24946a0c 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -2,16 +2,17 @@ from discovery import * from discovery.constants import * -from lib.core import * from lib import hostchecker from lib import htmlExport from lib import reportgraph from lib import statichtmlgenerator +from lib import stash +from lib.core import * +from platform import python_version +import argparse import datetime -import getopt import ipaddress import re -import stash import time try: @@ -29,230 +30,209 @@ except ImportError: Core.banner() -def start(argv): - if len(sys.argv) < 4: - Core.usage() - sys.exit(1) - try: - opts, args = getopt.getopt(argv, 'l:d:b:s:u:vf:nhcgpte:') - except getopt.GetoptError: - Core.usage() - sys.exit(1) +def start(): + parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon') + parser.add_argument('-d', '--domain', help='company name or domain to search', required=True) + parser.add_argument('-l', '--limit', help='limit the number of search results, default=500', default=500, type=int) + parser.add_argument('-S', '--start', help='start with result number X, default=0', default=0, type=int) + parser.add_argument('-g', '--google-dork', help='use Google Dorks for google search, default=False, params=True', default=False) + parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True', default=False) + parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts, default=False, params=True', default=False) + parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts params=basic, default=False', default=False) + parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup') + parser.add_argument('-t', '--dns-tld', help='perform a DNS TLD expansion discovery, default False', default=False) + parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default=False, params=True', default=False) + parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain, default=False, params=True', default=False) + parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) + parser.add_argument('-b', '--source', help='''source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile, + google, googleCSE, google-certificates, google-profiles, + hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd, + trello, twitter, vhost, virustotal, yahoo, all''', required=True) + + args = parser.parse_args() + try: db = stash.stash_manager() db.do_init() - except Exception as e: + except Exception: pass all_emails = [] all_hosts = [] all_ip = [] bingapi = 'yes' - dnsbrute = False - dnslookup = False - dnsserver = "" - dnstld = False - filename = "" + dnsbrute = args.dns_brute + dnslookup = args.dns_lookup + dnsserver = args.dns_server + dnstld = args.dns_tld + filename = args.filename full = [] - google_dorking = False + google_dorking = args.google_dork host_ip = [] - limit = 500 - ports_scanning = False - shodan = False - start = 0 + limit = args.limit + ports_scanning = args.port_scan + shodan = args.shodan + start = args.start takeover_check = False trello_info = ([], False) vhost = [] - virtual = False + virtual = args.virtual_host + word = args.domain + engines = set(args.source.split(',')) + if set(engines).issubset(Core.get_supportedengines()): + print(f'\033[94m[*] Target domain: {word} \n \033[0m') + for engineitem in engines: + if engineitem == 'baidu': + print('\033[94m[*] Searching Baidu. \033[0m') + try: + search = baidusearch.SearchBaidu(word, limit) + search.process() + all_emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'baidu') + db.store_all(word, all_emails, 'email', 'baidu') + except Exception: + pass - for value in enumerate(opts): - opt = value[1][0] - arg = value[1][1] - opt = str(opt) - arg = str(arg) - if opt == '-l': - limit = int(arg) - elif opt == '-d': - word = arg - elif opt == '-g': - google_dorking = True - elif opt == '-s': - start = int(arg) - elif opt == '-v': - virtual = 'basic' - elif opt == '-f': - filename = arg - elif opt == '-n': - dnslookup = True - elif opt == '-c': - dnsbrute = True - elif opt == '-h': - shodan = True - elif opt == '-e': - dnsserver = arg - elif opt == '-p': - ports_scanning = True - elif opt == '-t': - dnstld = True - elif opt == '-b': - engines = set(arg.split(',')) - supportedengines = set(['baidu', 'bing', 'bingapi', 'censys', 'crtsh', 'cymon', 'dogpile', 'duckduckgo', - 'google', 'googleCSE', 'google-certificates', 'google-profiles', 'hunter', - 'linkedin', 'netcraft', 'pgp', 'securityTrails', 'threatcrowd', 'trello', - 'twitter', 'vhost', 'virustotal', 'yahoo', 'all']) - if set(engines).issubset(supportedengines): - print(f'\033[94m[*] Target domain: {word} \n \033[0m') - for engineitem in engines: - if engineitem == 'baidu': - print('\033[94m[*] Searching Baidu. \033[0m') - try: - search = baidusearch.SearchBaidu(word, limit) - search.process() - all_emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'baidu') - db.store_all(word, all_emails, 'email', 'baidu') - except Exception: + elif engineitem == 'bing' or engineitem == 'bingapi': + print('\033[94m[*] Searching Bing. \033[0m') + try: + search = bingsearch.SearchBing(word, limit, start) + bingapi = '' + if engineitem == 'bingapi': + bingapi += 'yes' + else: + bingapi += 'no' + search.process(bingapi) + all_emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'bing') + db.store_all(word, all_hosts, 'host', 'bing') + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: pass - elif engineitem == 'bing' or engineitem == 'bingapi': - print('\033[94m[*] Searching Bing. \033[0m') - try: - search = bingsearch.SearchBing(word, limit, start) - if engineitem == 'bingapi': - bingapi = 'yes' - else: - bingapi = 'no' - search.process(bingapi) - all_emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'bing') - db.store_all(word, all_hosts, 'host', 'bing') - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - pass + elif engineitem == 'censys': + print('\033[94m[*] Searching Censys. \033[0m') + from discovery import censys + # Import locally or won't work + search = censys.SearchCensys(word, limit) + search.process() + all_ip = search.get_ipaddresses() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'censys') + db.store_all(word, all_ip, 'ip', 'censys') - elif engineitem == 'censys': - print('\033[94m[*] Searching Censys. \033[0m') - from discovery import censys - # Import locally or won't work - search = censys.SearchCensys(word, limit) + elif engineitem == 'crtsh': + print('\033[94m[*] Searching CRT.sh. \033[0m') + search = crtsh.search_crtsh(word) + search.process() + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'CRTsh') + + elif engineitem == 'cymon': + print('\033[94m[*] Searching Cymon. \033[0m') + from discovery import cymon + # Import locally or won't work. + search = cymon.search_cymon(word) + search.process() + all_ip = search.get_ipaddresses() + db = stash.stash_manager() + db.store_all(word, all_ip, 'ip', 'cymon') + + elif engineitem == 'dogpile': + print('\033[94m[*] Searching Dogpile. \033[0m') + search = dogpilesearch.SearchDogpile(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'dogpile') + db.store_all(word, all_hosts, 'host', 'dogpile') + + elif engineitem == 'duckduckgo': + print('\033[94m[*] Searching DuckDuckGo. \033[0m') + from discovery import duckduckgosearch + search = duckduckgosearch.SearchDuckDuckGo(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'duckduckgo') + db.store_all(word, all_hosts, 'host', 'duckduckgo') + + elif engineitem == 'google': + print('\033[94m[*] Searching Google. \033[0m') + search = googlesearch.search_google(word, limit, start) + search.process(google_dorking) + emails = filter(search.get_emails()) + all_emails.extend(emails) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'google') + db.store_all(word, all_emails, 'email', 'google') + + elif engineitem == 'googleCSE': + print('\033[94m[*] Searching Google Custom Search. \033[0m') + try: + search = googleCSE.SearchGoogleCSE(word, limit, start) search.process() - all_ip = search.get_ipaddresses() + search.store_results() + all_emails = filter(search.get_emails()) + db = stash.stash_manager() hosts = filter(search.get_hostnames()) all_hosts.extend(hosts) + db.store_all(word, all_hosts, 'email', 'googleCSE') db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'censys') - db.store_all(word, all_ip, 'ip', 'censys') - - elif engineitem == 'crtsh': - print('\033[94m[*] Searching CRT.sh. \033[0m') - search = crtsh.search_crtsh(word) - search.process() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'CRTsh') - - elif engineitem == 'cymon': - print('\033[94m[*] Searching Cymon. \033[0m') - from discovery import cymon - # Import locally or won't work. - search = cymon.search_cymon(word) - search.process() - all_ip = search.get_ipaddresses() - db = stash.stash_manager() - db.store_all(word, all_ip, 'ip', 'cymon') - - elif engineitem == 'dogpile': - print('\033[94m[*] Searching Dogpile. \033[0m') - search = dogpilesearch.SearchDogpile(word, limit) - search.process() - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - all_emails.extend(emails) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'dogpile') - db.store_all(word, all_hosts, 'host', 'dogpile') - - elif engineitem == 'duckduckgo': - print('\033[94m[*] Searching DuckDuckGo. \033[0m') - from discovery import duckduckgosearch - search = duckduckgosearch.SearchDuckDuckGo(word, limit) - search.process() - emails = filter(search.get_emails()) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - all_emails.extend(emails) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'email', 'duckduckgo') - db.store_all(word, all_hosts, 'host', 'duckduckgo') - - elif engineitem == 'google': - print('\033[94m[*] Searching Google. \033[0m') - search = googlesearch.search_google(word, limit, start) - search.process(google_dorking) - emails = filter(search.get_emails()) - all_emails.extend(emails) - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google') - db.store_all(word, all_emails, 'email', 'google') - - elif engineitem == 'googleCSE': - print('\033[94m[*] Searching Google Custom Search. \033[0m') - try: - search = googleCSE.SearchGoogleCSE(word, limit, start) - search.process() - search.store_results() - all_emails = filter(search.get_emails()) - db = stash.stash_manager() - hosts = filter(search.get_hostnames()) - all_hosts.extend(hosts) - db.store_all(word, all_hosts, 'email', 'googleCSE') - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'googleCSE') - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - pass - - elif engineitem == 'google-certificates': - print('\033[94m[*] Searching Google Certificate transparency report. \033[0m') - search = googlecertificates.SearchGoogleCertificates(word, limit, start) - search.process() - hosts = filter(search.get_domains()) - all_hosts.extend(hosts) - db = stash.stash_manager() - db.store_all(word, all_hosts, 'host', 'google-certificates') - - elif engineitem == 'google-profiles': - print('\033[94m[*] Searching Google profiles. \033[0m') - search = googlesearch.search_google(word, limit, start) - search.process_profiles() - people = search.get_profiles() - db = stash.stash_manager() - db.store_all(word, people, 'name', 'google-profile') - - if len(people) == 0: - print('\n[*] No users found.\n\n') + db.store_all(word, all_hosts, 'host', 'googleCSE') + except Exception as e: + if isinstance(e, MissingKey): + print(e) else: - print('\n[*] Users found: ' + str(len(people))) - print('---------------------') - for user in sorted(list(set(people))): - print(user) + pass + + elif engineitem == 'google-certificates': + print('\033[94m[*] Searching Google Certificate transparency report. \033[0m') + search = googlecertificates.SearchGoogleCertificates(word, limit, start) + search.process() + hosts = filter(search.get_domains()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'google-certificates') + + elif engineitem == 'google-profiles': + print('\033[94m[*] Searching Google profiles. \033[0m') + search = googlesearch.search_google(word, limit, start) + search.process_profiles() + people = search.get_profiles() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'google-profile') + + if len(people) == 0: + print('\n[*] No users found.\n\n') + else: + print('\n[*] Users found: ' + str(len(people))) + print('---------------------') + for user in sorted(list(set(people))): + print(user) sys.exit(0) - elif engineitem == 'hunter': + elif engineitem == 'hunter': print('\033[94m[*] Searching Hunter. \033[0m') from discovery import huntersearch # Import locally or won't work. @@ -272,7 +252,7 @@ def start(argv): else: pass - elif engineitem == 'linkedin': + elif engineitem == 'linkedin': print('\033[94m[*] Searching Linkedin. \033[0m') search = linkedinsearch.SearchLinkedin(word, limit) search.process() @@ -281,15 +261,15 @@ def start(argv): db.store_all(word, people, 'name', 'linkedin') if len(people) == 0: - print('\n[*] No users found.\n\n') + print('\n[*] No users found Linkedin.\n\n') else: - print('\n[*] Users found: ' + str(len(people))) + print(f'\n[*] Users found: {len(people)}') print('---------------------') for user in sorted(list(set(people))): print(user) sys.exit(0) - elif engineitem == 'netcraft': + elif engineitem == 'netcraft': print('\033[94m[*] Searching Netcraft. \033[0m') search = netcraft.SearchNetcraft(word) search.process() @@ -298,7 +278,7 @@ def start(argv): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'netcraft') - elif engineitem == 'pgp': + elif engineitem == 'pgp': print('\033[94m[*] Searching PGP key server. \033[0m') try: search = pgpsearch.SearchPgp(word) @@ -312,7 +292,7 @@ def start(argv): except Exception: pass - elif engineitem == 'securityTrails': + elif engineitem == 'securityTrails': print('\033[94m[*] Searching SecurityTrails. \033[0m') from discovery import securitytrailssearch try: @@ -332,7 +312,7 @@ def start(argv): else: pass - elif engineitem == 'threatcrowd': + elif engineitem == 'threatcrowd': print('\033[94m[*] Searching Threatcrowd. \033[0m') try: search = threatcrowd.search_threatcrowd(word) @@ -344,7 +324,7 @@ def start(argv): except Exception: pass - elif engineitem == 'trello': + elif engineitem == 'trello': print('\033[94m[*] Searching Trello. \033[0m') from discovery import trello # Import locally or won't work. @@ -360,7 +340,7 @@ def start(argv): db.store_all(word, hosts, 'host', 'trello') db.store_all(word, emails, 'email', 'trello') - elif engineitem == 'twitter': + elif engineitem == 'twitter': print('\033[94m[*] Searching Twitter. \033[0m') search = twittersearch.search_twitter(word, limit) search.process() @@ -379,7 +359,7 @@ def start(argv): # vhost - elif engineitem == 'virustotal': + elif engineitem == 'virustotal': print('\033[94m[*] Searching VirusTotal. \033[0m') search = virustotal.search_virustotal(word) search.process() @@ -388,7 +368,7 @@ def start(argv): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'virustotal') - elif engineitem == 'yahoo': + elif engineitem == 'yahoo': print('\033[94m[*] Searching Yahoo. \033[0m') search = yahoosearch.search_yahoo(word, limit) search.process() @@ -400,7 +380,7 @@ def start(argv): db.store_all(word, all_hosts, 'host', 'yahoo') db.store_all(word, all_emails, 'email', 'yahoo') - elif engineitem == 'all': + elif engineitem == 'all': print(('Full harvest on ' + word)) all_emails = [] all_hosts = [] @@ -455,8 +435,25 @@ def start(argv): db.store_all(word, all_hosts, 'host', 'CRTsh') # cymon + print('\033[94m[*] Searching Cymon. \033[0m') + from discovery import cymon + # Import locally or won't work. + search = cymon.search_cymon(word) + search.process() + all_ip = search.get_ipaddresses() + db = stash.stash_manager() + db.store_all(word, all_ip, 'ip', 'cymon') - # dogpile + print('\033[94m[*] Searching Dogpile. \033[0m') + search = dogpilesearch.SearchDogpile(word, limit) + search.process() + emails = filter(search.get_emails()) + hosts = filter(search.get_hostnames()) + all_hosts.extend(hosts) + all_emails.extend(emails) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'email', 'dogpile') + db.store_all(word, all_hosts, 'host', 'dogpile') print('[*] Searching DuckDuckGo.') from discovery import duckduckgosearch @@ -524,7 +521,20 @@ def start(argv): else: pass - # linkedin + print('\033[94m[*] Searching Linkedin. \033[0m') + search = linkedinsearch.SearchLinkedin(word, limit) + search.process() + people = search.get_people() + db = stash.stash_manager() + db.store_all(word, people, 'name', 'linkedin') + + if len(people) == 0: + print('\n[*] No users found.\n\n') + else: + print('\n[*] Users found: ' + str(len(people))) + print('---------------------') + for user in sorted(list(set(people))): + print(user) print('[*] Searching Netcraft.') search = netcraft.SearchNetcraft(word) @@ -591,7 +601,20 @@ def start(argv): except Exception: pass - # vhost + print('\n[*] Virtual hosts:') + print('------------------') + for l in host_ip: + search = bingsearch.SearchBing(l, limit, start) + search.process_vhost() + res = search.get_allhostnames() + for x in res: + x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x) + x = re.sub('<', '', x) + x = re.sub('>', '', x) + print((l + '\t' + x)) + vhost.append(l + ':' + x) + full.append(l + ':' + x) + vhost = sorted(set(vhost)) print('[*] Searching VirusTotal.') search = virustotal.search_virustotal(word) @@ -611,9 +634,9 @@ def start(argv): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'yahoo') db.store_all(word, all_emails, 'email', 'yahoo') - else: - print('\033[93m[!] Invalid source.\n\n \033[0m') - sys.exit(1) + else: + print('\033[93m[!] Invalid source.\n\n \033[0m') + sys.exit(1) # Sanity check to see if all_emails and all_hosts are defined. try: @@ -777,16 +800,16 @@ def start(argv): # Shodan shodanres = [] - import texttable - tab = texttable.Texttable() - header = ['IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'] - tab.header(header) - tab.set_cols_align(['c', 'c', 'c', 'c', 'c']) - tab.set_cols_valign(['m', 'm', 'm', 'm', 'm']) - tab.set_chars(['-', '|', '+', '#']) - tab.set_cols_width([15, 20, 15, 15, 18]) - host_ip = list(set(host_ip)) if shodan is True: + import texttable + tab = texttable.Texttable() + header = ['IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'] + tab.header(header) + tab.set_cols_align(['c', 'c', 'c', 'c', 'c']) + tab.set_cols_valign(['m', 'm', 'm', 'm', 'm']) + tab.set_chars(['-', '|', '+', '#']) + tab.set_cols_width([15, 20, 15, 15, 18]) + host_ip = list(set(host_ip)) print('\n\n[*] Shodan DB search (passive):\n') try: for ip in host_ip: @@ -822,7 +845,7 @@ def start(argv): # Reporting if filename != "": try: - print('NEW REPORTING BEGINS.') + print('\nNEW REPORTING BEGINS.') db = stash.stash_manager() scanboarddata = db.getscanboarddata() latestscanresults = db.getlatestscanresults(word) @@ -834,7 +857,7 @@ def start(argv): HTMLcode = generator.beginhtml() HTMLcode += generator.generatelatestscanresults(latestscanresults) HTMLcode += generator.generatepreviousscanresults(previousscanresults) - graph = reportgraph.graphgenerator(word) + graph = reportgraph.GraphGenerator(word) HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata) HTMLcode += graph.drawscattergraphscanhistory(word, scanhistorydomain) HTMLcode += generator.generatepluginscanstatistics(pluginscanstatistics) @@ -915,11 +938,14 @@ def start(argv): if __name__ == '__main__': + if python_version()[0:3] < '3.6': + print('\033[93m[!] Please make sure you have python 3.6+ installed, quitting.\033[0m') + sys.exit(1) try: - start(sys.argv[1:]) + start() except KeyboardInterrupt: print('\n\n\033[93m[!] ctrl+c detected from user, quitting.\n\n \033[0m') except Exception: import traceback print(traceback.print_exc()) - sys.exit(1) \ No newline at end of file + sys.exit(1)