WEBDIENSTE/theHarvester
1
0
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2026-08-17 19:35:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #162 from L1ghtn1ng/dev

Browse Source 
Move from getopt to argparse, plus many other changes and fixes
This commit is contained in:
Christian Martorella
2019-01-22 23:20:20 +01:00
committed by GitHub
parent cae56fbfd7 7aaf05859f
commit ac847532ee
16 changed files with 658 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+5 -5
View File
@@ -6,7 +6,7 @@
* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *
* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *
* *
* theHarvester 3.0.6 v183 *
* theHarvester 3.0.6 v206 *
* Coded by Christian Martorella *
* Edge-Security Research *
* cmartorella@edge-security.com *
@@ -82,7 +82,7 @@ Active:
Modules that require an API key:
--------------------------------
Add your keys to discovery/constants.py
Add your keys to api-keys.yaml
* googleCSE: API key and CSE ID
* hunter: API key
@@ -92,7 +92,7 @@ Add your keys to discovery/constants.py
Dependencies:
-------------
* Python 3.6
* pip3 install -r requirements.txt
* python3 -m pip install -r requirements.txt
Changelog in 3.0:
-----------------
@@ -110,8 +110,8 @@ Main contributors:
-------
* Matthew Brown @NotoriousRebel
* Janos Zold @Jzold
* Lee Baird @discoverscripts
* Jay Townsend @L1ghtn1ng
* Lee Baird @discoverscripts [![Twitter Follow](https://img.shields.io/twitter/follow/discoverscripts.svg?style=social&label=Follow)](https://twitter.com/discoverscripts)
* Jay Townsend @L1ghtn1ng [![Twitter Follow](https://img.shields.io/twitter/follow/jay_townsend1.svg?style=social&label=Follow)](https://twitter.com/jay_townsend1)
Thanks:
-------
api-keys.yaml
+16
View File
@@ -0,0 +1,16 @@
apikeys:
bing:
key:
googleCSE:
key:
id:
hunter:
key:
securityTrails:
key:
shodan:
key: oCiMsgM6rQWqiTvPxFHYcExlZgg7wvTt
discovery/bingsearch.py
+2 -2
View File
@@ -16,7 +16,7 @@ class SearchBing:
self.hostname = 'www.bing.com'
self.quantity = '50'
self.limit = int(limit)
self.bingApi = bingAPI_key
self.bingApi = Core.bing_key()
self.counter = start
def do_search(self):
@@ -70,7 +70,7 @@ class SearchBing:
def process(self, api):
if api == 'yes':
if self.bingApi == "":
if self.bingApi is None:
raise MissingKey(True)
while self.counter < self.limit:
if api == 'yes':
discovery/constants.py
-19
View File
@@ -1,27 +1,8 @@
"""
Module that contains constants used across plugins.
Contains list of API keys, user agents, and a function to get random delay and user agent.
As well as a defined User Agent for Google Search.
User-Agents from: https://github.com/tamimibrahim17/List-of-user-agents
"""
import random
googleUA = "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36"
bingAPI_key = ''
googleCSEAPI_key = ''
googleCSE_id = ''
hunterAPI_key = ''
securityTrailsAPI_key = ''
shodanAPI_key = 'oCiMsgM6rQWqiTvPxFHYcExlZgg7wvTt' # This is the default key.
def filter(lst):
"""
discovery/crtsh.py
+1 -1
View File
@@ -22,7 +22,7 @@ class search_crtsh:
print(e)
try:
params = {'User-Agent': Core.get_user_agent()}
r=requests.get(urly, headers=params)
r = requests.get(urly, headers=params)
except Exception as e:
print(e)
links = self.get_info(r.text)
discovery/dogpilesearch.py
+6 -3
View File
@@ -17,14 +17,17 @@ class SearchDogpile:
def do_search(self):
# Dogpile is hardcoded to return 10 results.
url = 'http://' + self.server + "/search/web?qsi=" + str(self.counter) \
url = 'https://' + self.server + "/search/web?qsi=" + str(self.counter) \
+ "&q=\"%40" + self.word + "\""
headers = {
'Host': self.hostname,
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.total_results += h.text
try:
h = requests.get(url=url, headers=headers)
self.total_results += h.text
except requests.exceptions.ConnectionError:
pass
def process(self):
while self.counter <= self.limit and self.counter <= 1000:
discovery/googleCSE.py
+5 -4
View File
@@ -1,4 +1,5 @@
from discovery.constants import *
from lib.core import *
from parsers import myparser
import re
import requests
@@ -19,11 +20,11 @@ class SearchGoogleCSE:
self.quantity = "10"
self.limit = limit
self.counter = 1
self.api_key = googleCSEAPI_key
if self.api_key == "":
self.api_key = Core.google_cse_key()['key']
if self.api_key is None:
raise MissingKey(True)
self.cse_id = googleCSE_id
if self.cse_id == "":
self.cse_id = Core.google_cse_key()['id']
if self.cse_id is None:
raise MissingKey(False)
self.lowRange = start
self.highRange = start + 100
discovery/huntersearch.py
+2 -1
View File
@@ -1,4 +1,5 @@
from discovery.constants import *
from lib.core import *
from parsers import myparser
import requests
@@ -9,7 +10,7 @@ class SearchHunter:
self.word = word
self.limit = 100
self.start = start
self.key = hunterAPI_key
self.key = Core.hunter_key()
if self.key == "":
raise MissingKey(True)
self.results = ""
discovery/securitytrailssearch.py
+2 -1
View File
@@ -1,4 +1,5 @@
from discovery.constants import *
from lib.core import *
from parsers import securitytrailsparser
import requests
import sys
@@ -9,7 +10,7 @@ class search_securitytrail:
def __init__(self, word):
self.word = word
self.key = securityTrailsAPI_key
self.key = Core.security_trails_key()
if self.key == "":
raise MissingKey(True)
self.results = ""
discovery/shodansearch.py
+3 -2
View File
@@ -1,12 +1,13 @@
from discovery.constants import *
from shodan import Shodan
from lib.core import *
from shodan import exception
from shodan import Shodan
class search_shodan:
def __init__(self):
self.key = shodanAPI_key
self.key = Core.shodan_key()
if self.key == '':
raise MissingKey(True)
self.api = Shodan(self.key)
discovery/twittersearch.py
+1 -1
View File
@@ -24,7 +24,7 @@ class search_twitter:
print(e)
headers = {'User-Agent': Core.get_user_agent()}
try:
r=requests.get(urly, headers=headers)
r = requests.get(urly, headers=headers)
except Exception as e:
print(e)
self.results = r.text
lib/core.py
+322 -31
View File
@@ -1,11 +1,45 @@
# coding=utf-8
#from discovery import *
# from lib import stash
import os
import random
# import re
import sys
import yaml
class Core:
@staticmethod
def bing_key():
with open('api-keys.yaml', 'r') as api_keys:
keys = yaml.safe_load(api_keys)
return keys['apikeys']['bing']['key']
@staticmethod
def google_cse_key():
with open('api-keys.yaml', 'r') as api_keys:
keys = yaml.safe_load(api_keys)
return keys['apikeys']['googleCSE']
@staticmethod
def hunter_key():
with open('api-keys.yaml', 'r') as api_keys:
keys = yaml.safe_load(api_keys)
return keys['apikeys']['hunter']['key']
@staticmethod
def security_trails_key():
with open('api-keys.yaml', 'r') as api_keys:
keys = yaml.safe_load(api_keys)
return keys['apikeys']['securityTrails']['key']
@staticmethod
def shodan_key():
with open('api-keys.yaml', 'r') as api_keys:
keys = yaml.safe_load(api_keys)
return keys['apikeys']['shodan']['key']
@staticmethod
def banner():
print('\n\033[93m*******************************************************************')
@@ -15,7 +49,7 @@ class Core:
print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *")
print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *")
print('* *')
print('* theHarvester 3.0.6 v183 *')
print('* theHarvester 3.0.6 v206 *')
print('* Coded by Christian Martorella *')
print('* Edge-Security Research *')
print('* cmartorella@edge-security.com *')
@@ -23,39 +57,37 @@ class Core:
print('******************************************************************* \n\n \033[0m')
@staticmethod
def usage():
comm = os.path.basename(sys.argv[0])
if os.path.dirname(sys.argv[0]) == os.getcwd():
comm = './' + comm
print('\033[94m Usage: theHarvester.py <options> \n \033[0m')
print(' -d: company name or domain to search')
print(""" -b: source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile,
google, googleCSE, google-certificates, google-profiles,
hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd,
trello, twitter, vhost, virustotal, yahoo, all""")
print(' -l: limit the number of search results')
print(' -s: start with result number X (default: 0)')
print(' -g: use Google Dorking instead of normal Google search')
print(' -h: use Shodan to query discovered hosts')
print(' -e: specify DNS server')
print(' -v: verify host name via DNS resolution and search for virtual hosts')
print(' -n: perform a DNS reverse query on all ranges discovered')
print(' -c: perform a DNS brute force on the domain')
print(' -t: perform a DNS TLD expansion discovery')
print(' -p: port scan the detected hosts and check for Takeovers (21,22,80,443,8080)')
print(' -f: save the results to an HTML and/or XML file')
print('\n\033[94m Examples: \033[0m')
print((' ' + comm + ' -d acme -l 200 -b linkedin'))
print((' ' + comm + ' -d acme.com -l 500 -b google -f myresults.html'))
print((' ' + comm + ' -d acme.com -b pgp, virustotal'))
print((' ' + comm + ' -d acme.com -l 100 -g -b google'))
print((' ' + comm + ' -d acme.com -l 200 -b googleCSE -s 300'))
print((' ' + comm + ' -d acme.edu -l 300 -b bing -h \n'))
def get_supportedengines():
supportedengines = {'baidu',
'bing',
'bingapi',
'censys',
'crtsh',
'cymon',
'dogpile',
'duckduckgo',
'google',
'googleCSE',
'google-certificates',
'google-profiles',
'hunter',
'linkedin',
'netcraft',
'pgp',
'securityTrails',
'threatcrowd',
'trello',
'twitter',
'vhost',
'virustotal',
'yahoo',
'all'
}
return supportedengines
@staticmethod
def get_user_agent():
# User-Agents from https://github.com/tamimibrahim17/List-of-user-agents
user_agents = [
'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205',
@@ -287,3 +319,262 @@ class Core:
'Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00'
]
return random.choice(user_agents)
# TODO use this method when -b all is called to replace lines 383-635 in theHarvester.py
# TODO and to find the best approch of getting the
# word, limit and start etc vars from the arguments and importing libs that are needed
# @staticmethod
# def engine_all_search():
# print(('Full harvest on ' + word))
# all_emails = []
# all_hosts = []
# try:
# print('[*] Searching Baidu.')
# search = baidusearch.SearchBaidu(word, limit)
# search.process()
# all_emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'baidu')
# db.store_all(word, all_emails, 'email', 'baidu')
# except Exception:
# pass
#
# print('[*] Searching Bing.')
# bingapi = 'no'
# search = bingsearch.SearchBing(word, limit, start)
# search.process(bingapi)
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'bing')
# all_emails.extend(emails)
# all_emails = sorted(set(all_emails))
# db.store_all(word, all_emails, 'email', 'bing')
#
# print('[*] Searching Censys.')
# from discovery import censys
# search = censys.SearchCensys(word, limit)
# search.process()
# ips = search.get_ipaddresses()
# setips = set(ips)
# uniqueips = list(setips) # Remove duplicates.
# all_ip.extend(uniqueips)
# hosts = filter(search.get_hostnames())
# sethosts = set(hosts)
# uniquehosts = list(sethosts) # Remove duplicates.
# all_hosts.extend(uniquehosts)
# db = stash.stash_manager()
# db.store_all(word, uniquehosts, 'host', 'censys')
# db.store_all(word, uniqueips, 'ip', 'censys')
#
# print('[*] Searching CRT.sh.')
# search = crtsh.search_crtsh(word)
# search.process()
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'CRTsh')
#
# # cymon
# print('\033[94m[*] Searching Cymon. \033[0m')
# from discovery import cymon
# # Import locally or won't work.
# search = cymon.search_cymon(word)
# search.process()
# all_ip = search.get_ipaddresses()
# db = stash.stash_manager()
# db.store_all(word, all_ip, 'ip', 'cymon')
#
# print('\033[94m[*] Searching Dogpile. \033[0m')
# search = dogpilesearch.SearchDogpile(word, limit)
# search.process()
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# all_emails.extend(emails)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'email', 'dogpile')
# db.store_all(word, all_hosts, 'host', 'dogpile')
#
# print('[*] Searching DuckDuckGo.')
# from discovery import duckduckgosearch
# search = duckduckgosearch.SearchDuckDuckGo(word, limit)
# search.process()
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# all_emails.extend(emails)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'email', 'duckduckgo')
# db.store_all(word, all_hosts, 'host', 'duckduckgo')
#
# print('[*] Searching Google.')
# search = googlesearch.search_google(word, limit, start)
# search.process(google_dorking)
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_emails.extend(emails)
# db = stash.stash_manager()
# db.store_all(word, all_emails, 'email', 'google')
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'google')
#
# print('[*] Searching Google Certificate transparency report.')
# search = googlecertificates.SearchGoogleCertificates(word, limit, start)
# search.process()
# domains = filter(search.get_domains())
# all_hosts.extend(domains)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'google-certificates')
#
# try:
# print('[*] Searching Google profiles.')
# search = googlesearch.search_google(word, limit, start)
# search.process_profiles()
# people = search.get_profiles()
# db = stash.stash_manager()
# db.store_all(word, people, 'name', 'google-profile')
# print('\nUsers from Google profiles:')
# print('---------------------------')
# for users in people:
# print(users)
# except Exception:
# pass
#
# print('[*] Searching Hunter.')
# from discovery import huntersearch
# # Import locally.
# try:
# search = huntersearch.SearchHunter(word, limit, start)
# search.process()
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, hosts, 'host', 'hunter')
# all_emails.extend(emails)
# all_emails = sorted(set(all_emails))
# db.store_all(word, all_emails, 'email', 'hunter')
# except Exception as e:
# if isinstance(e, MissingKey):
# print(e)
# else:
# pass
#
# print('\033[94m[*] Searching Linkedin. \033[0m')
# search = linkedinsearch.SearchLinkedin(word, limit)
# search.process()
# people = search.get_people()
# db = stash.stash_manager()
# db.store_all(word, people, 'name', 'linkedin')
#
# if len(people) == 0:
# print('\n[*] No users found.\n\n')
# else:
# print('\n[*] Users found: ' + str(len(people)))
# print('---------------------')
# for user in sorted(list(set(people))):
# print(user)
#
# print('[*] Searching Netcraft.')
# search = netcraft.SearchNetcraft(word)
# search.process()
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'netcraft')
#
# print('[*] Searching PGP key server.')
# try:
# search = pgpsearch.SearchPgp(word)
# search.process()
# emails = filter(search.get_emails())
# hosts = filter(search.get_hostnames())
# sethosts = set(hosts)
# uniquehosts = list(sethosts) # Remove duplicates.
# all_hosts.extend(uniquehosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'PGP')
# all_emails.extend(emails)
# db = stash.stash_manager()
# db.store_all(word, all_emails, 'email', 'PGP')
# except Exception:
# pass
#
# print('[*] Searching Threatcrowd.')
# try:
# search = threatcrowd.search_threatcrowd(word)
# search.process()
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'threatcrowd')
# except Exception:
# pass
#
# print('[*] Searching Trello.')
# from discovery import trello
# # Import locally or won't work.
# search = trello.search_trello(word, limit)
# search.process()
# emails = filter(search.get_emails())
# all_emails.extend(emails)
# info = search.get_urls()
# hosts = filter(info[0])
# trello_info = (info[1], True)
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, hosts, 'host', 'trello')
# db.store_all(word, emails, 'email', 'trello')
#
# try:
# print('[*] Searching Twitter.')
# search = twittersearch.search_twitter(word, limit)
# search.process()
# people = search.get_people()
# db = stash.stash_manager()
# db.store_all(word, people, 'name', 'twitter')
# print('\nUsers from Twitter:')
# print('-------------------')
# for user in people:
# print(user)
# except Exception:
# pass
#
# print('\n[*] Virtual hosts:')
# print('------------------')
# for l in host_ip:
# search = bingsearch.SearchBing(l, limit, start)
# search.process_vhost()
# res = search.get_allhostnames()
# for x in res:
# x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
# x = re.sub('<', '', x)
# x = re.sub('>', '', x)
# print((l + '\t' + x))
# vhost.append(l + ':' + x)
# full.append(l + ':' + x)
# vhost = sorted(set(vhost))
#
# print('[*] Searching VirusTotal.')
# search = virustotal.search_virustotal(word)
# search.process()
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'virustotal')
#
# print('[*] Searching Yahoo.')
# search = yahoosearch.search_yahoo(word, limit)
# search.process()
# hosts = search.get_hostnames()
# emails = search.get_emails()
# all_hosts.extend(filter(hosts))
# all_emails.extend(filter(emails))
# db = stash.stash_manager()
# db.store_all(word, all_hosts, 'host', 'yahoo')
# db.store_all(word, all_emails, 'email', 'yahoo')
lib/reportgraph.py
+28 -31
View File
@@ -1,16 +1,16 @@
try:
from datetime import datetime
import plotly.graph_objs as go
import plotly.plotly as py
import plotly
import stash
try:
db = stash.stash_manager()
db.do_init()
except Exception as e:
pass
from datetime import datetime
from lib import stash
import plotly
import plotly.graph_objs as go
import plotly.plotly as py
class graphgenerator:
try:
db = stash.stash_manager()
db.do_init()
except Exception:
pass
class GraphGenerator:
def __init__(self, domain):
self.domain = domain
@@ -23,26 +23,26 @@ try:
self.scattercountshodans = []
self.scattercountvhosts = []
def drawlatestscangraph(self,domain,latestscandata):
def drawlatestscangraph(self, domain, latestscandata):
try:
self.barcolumns= ['email', 'host', 'ip', 'shodan', 'vhost']
self.barcolumns = ['email', 'host', 'ip', 'shodan', 'vhost']
self.bardata.append(latestscandata['email'])
self.bardata.append(latestscandata['host'])
self.bardata.append(latestscandata['ip'])
self.bardata.append(latestscandata['shodan'])
self.bardata.append(latestscandata['vhost'])
layout = dict(title = 'Latest scan - number of targets identified for ' + domain,
xaxis = dict(title = 'Targets'),
yaxis = dict(title = 'Hits'),)
layout = dict(title='Latest scan - number of targets identified for ' + domain,
xaxis=dict(title='Targets'),
yaxis=dict(title='Hits'),)
barchartcode = plotly.offline.plot({
'data': [go.Bar(x=self.barcolumns, y=self.bardata)],
'layout': layout,
}, auto_open=False, include_plotlyjs=False, filename='report.html', output_type='div')
return barchartcode
except Exception as e:
print('Error generating HTML bar graph code for domain: ' + str(e))
print(f'Error generating HTML bar graph code for domain: {e}')
def drawscattergraphscanhistory(self,domain,scanhistorydomain):
def drawscattergraphscanhistory(self, domain, scanhistorydomain):
try:
scandata = scanhistorydomain
for i in scandata:
@@ -56,20 +56,20 @@ try:
trace0 = go.Scatter(
x=self.scatterxdata,
y=self.scattercounthosts,
mode = 'lines+markers',
name = 'hosts')
mode='lines+markers',
name='hosts')
trace1 = go.Scatter(
x=self.scatterxdata,
y=self.scattercountips,
mode = 'lines+markers',
name = 'IP address')
mode='lines+markers',
name='IP address')
trace2 = go.Scatter(
x=self.scatterxdata,
y=self.scattercountvhosts,
mode = 'lines+markers',
name = 'vhost')
mode='lines+markers',
name='vhost')
trace3 = go.Scatter(
x=self.scatterxdata,
@@ -84,16 +84,13 @@ try:
name='email')
data = [trace0, trace1, trace2, trace3, trace4]
layout = dict(title = 'Scanning history for ' + domain,
xaxis = dict(title = 'Date'),
yaxis = dict(title = 'Results'),
)
layout = dict(title='Scanning history for ' + domain, xaxis=dict(title='Date'), yaxis=dict(title='Results'))
scatterchartcode = plotly.offline.plot({
'data': data,
'layout': layout}, auto_open=False, include_plotlyjs=False, filename='report.html', output_type='div')
return scatterchartcode
except Exception as e:
print('Error generating HTML for the historical graph for domain: ' + str(e))
print(f'Error generating HTML for the historical graph for domain: {e}')
except Exception as e:
print('Error in the reportgraph module: ' + str(e))
print(f'Error in the reportgraph module: {e}')
stash.py → lib/stash.py
View File
requirements.txt
+6 -5
View File
@@ -1,6 +1,7 @@
beautifulsoup4>=4.7.0
plotly>=3.4.2
pytest>=4.0.2
beautifulsoup4>=4.7.1
plotly>=3.5.0
PyYaml==3.13
pytest>=4.1.1
requests>=2.21.0
texttable>=1.4.0
shodan>=1.10.0
shodan>=1.10.0
texttable>=1.6.0
theHarvester.py
+259 -233
View File
@@ -2,16 +2,17 @@
from discovery import *
from discovery.constants import *
from lib.core import *
from lib import hostchecker
from lib import htmlExport
from lib import reportgraph
from lib import statichtmlgenerator
from lib import stash
from lib.core import *
from platform import python_version
import argparse
import datetime
import getopt
import ipaddress
import re
import stash
import time
try:
@@ -29,230 +30,209 @@ except ImportError:
Core.banner()
def start(argv):
if len(sys.argv) < 4:
Core.usage()
sys.exit(1)
try:
opts, args = getopt.getopt(argv, 'l:d:b:s:u:vf:nhcgpte:')
except getopt.GetoptError:
Core.usage()
sys.exit(1)
def start():
parser = argparse.ArgumentParser(description='theHarvester is a open source intelligence gathering tool(OSINT) that is used for recon')
parser.add_argument('-d', '--domain', help='company name or domain to search', required=True)
parser.add_argument('-l', '--limit', help='limit the number of search results, default=500', default=500, type=int)
parser.add_argument('-S', '--start', help='start with result number X, default=0', default=0, type=int)
parser.add_argument('-g', '--google-dork', help='use Google Dorks for google search, default=False, params=True', default=False)
parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True', default=False)
parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts, default=False, params=True', default=False)
parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts params=basic, default=False', default=False)
parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup')
parser.add_argument('-t', '--dns-tld', help='perform a DNS TLD expansion discovery, default False', default=False)
parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default=False, params=True', default=False)
parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain, default=False, params=True', default=False)
parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str)
parser.add_argument('-b', '--source', help='''source: baidu, bing, bingapi, censys, crtsh, cymon, dogpile,
google, googleCSE, google-certificates, google-profiles,
hunter, linkedin, netcraft, pgp, securityTrails, threatcrowd,
trello, twitter, vhost, virustotal, yahoo, all''', required=True)
args = parser.parse_args()
try:
db = stash.stash_manager()
db.do_init()
except Exception as e:
except Exception:
pass
all_emails = []
all_hosts = []
all_ip = []
bingapi = 'yes'
dnsbrute = False
dnslookup = False
dnsserver = ""
dnstld = False
filename = ""
dnsbrute = args.dns_brute
dnslookup = args.dns_lookup
dnsserver = args.dns_server
dnstld = args.dns_tld
filename = args.filename
full = []
google_dorking = False
google_dorking = args.google_dork
host_ip = []
limit = 500
ports_scanning = False
shodan = False
start = 0
limit = args.limit
ports_scanning = args.port_scan
shodan = args.shodan
start = args.start
takeover_check = False
trello_info = ([], False)
vhost = []
virtual = False
virtual = args.virtual_host
word = args.domain
engines = set(args.source.split(','))
if set(engines).issubset(Core.get_supportedengines()):
print(f'\033[94m[*] Target domain: {word} \n \033[0m')
for engineitem in engines:
if engineitem == 'baidu':
print('\033[94m[*] Searching Baidu. \033[0m')
try:
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
for value in enumerate(opts):
opt = value[1][0]
arg = value[1][1]
opt = str(opt)
arg = str(arg)
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-g':
google_dorking = True
elif opt == '-s':
start = int(arg)
elif opt == '-v':
virtual = 'basic'
elif opt == '-f':
filename = arg
elif opt == '-n':
dnslookup = True
elif opt == '-c':
dnsbrute = True
elif opt == '-h':
shodan = True
elif opt == '-e':
dnsserver = arg
elif opt == '-p':
ports_scanning = True
elif opt == '-t':
dnstld = True
elif opt == '-b':
engines = set(arg.split(','))
supportedengines = set(['baidu', 'bing', 'bingapi', 'censys', 'crtsh', 'cymon', 'dogpile', 'duckduckgo',
'google', 'googleCSE', 'google-certificates', 'google-profiles', 'hunter',
'linkedin', 'netcraft', 'pgp', 'securityTrails', 'threatcrowd', 'trello',
'twitter', 'vhost', 'virustotal', 'yahoo', 'all'])
if set(engines).issubset(supportedengines):
print(f'\033[94m[*] Target domain: {word} \n \033[0m')
for engineitem in engines:
if engineitem == 'baidu':
print('\033[94m[*] Searching Baidu. \033[0m')
try:
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
elif engineitem == 'bing' or engineitem == 'bingapi':
print('\033[94m[*] Searching Bing. \033[0m')
try:
search = bingsearch.SearchBing(word, limit, start)
bingapi = ''
if engineitem == 'bingapi':
bingapi += 'yes'
else:
bingapi += 'no'
search.process(bingapi)
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'bing')
db.store_all(word, all_hosts, 'host', 'bing')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'bing' or engineitem == 'bingapi':
print('\033[94m[*] Searching Bing. \033[0m')
try:
search = bingsearch.SearchBing(word, limit, start)
if engineitem == 'bingapi':
bingapi = 'yes'
else:
bingapi = 'no'
search.process(bingapi)
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'bing')
db.store_all(word, all_hosts, 'host', 'bing')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'censys':
print('\033[94m[*] Searching Censys. \033[0m')
from discovery import censys
# Import locally or won't work
search = censys.SearchCensys(word, limit)
search.process()
all_ip = search.get_ipaddresses()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == 'censys':
print('\033[94m[*] Searching Censys. \033[0m')
from discovery import censys
# Import locally or won't work
search = censys.SearchCensys(word, limit)
elif engineitem == 'crtsh':
print('\033[94m[*] Searching CRT.sh. \033[0m')
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
elif engineitem == 'cymon':
print('\033[94m[*] Searching Cymon. \033[0m')
from discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == 'dogpile':
print('\033[94m[*] Searching Dogpile. \033[0m')
search = dogpilesearch.SearchDogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
elif engineitem == 'duckduckgo':
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
from discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
elif engineitem == 'google':
print('\033[94m[*] Searching Google. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
db.store_all(word, all_emails, 'email', 'google')
elif engineitem == 'googleCSE':
print('\033[94m[*] Searching Google Custom Search. \033[0m')
try:
search = googleCSE.SearchGoogleCSE(word, limit, start)
search.process()
all_ip = search.get_ipaddresses()
search.store_results()
all_emails = filter(search.get_emails())
db = stash.stash_manager()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db.store_all(word, all_hosts, 'email', 'googleCSE')
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == 'crtsh':
print('\033[94m[*] Searching CRT.sh. \033[0m')
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
elif engineitem == 'cymon':
print('\033[94m[*] Searching Cymon. \033[0m')
from discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == 'dogpile':
print('\033[94m[*] Searching Dogpile. \033[0m')
search = dogpilesearch.SearchDogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
elif engineitem == 'duckduckgo':
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
from discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
elif engineitem == 'google':
print('\033[94m[*] Searching Google. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
db.store_all(word, all_emails, 'email', 'google')
elif engineitem == 'googleCSE':
print('\033[94m[*] Searching Google Custom Search. \033[0m')
try:
search = googleCSE.SearchGoogleCSE(word, limit, start)
search.process()
search.store_results()
all_emails = filter(search.get_emails())
db = stash.stash_manager()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db.store_all(word, all_hosts, 'email', 'googleCSE')
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'googleCSE')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'google-certificates':
print('\033[94m[*] Searching Google Certificate transparency report. \033[0m')
search = googlecertificates.SearchGoogleCertificates(word, limit, start)
search.process()
hosts = filter(search.get_domains())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google-certificates')
elif engineitem == 'google-profiles':
print('\033[94m[*] Searching Google profiles. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'google-profile')
if len(people) == 0:
print('\n[*] No users found.\n\n')
db.store_all(word, all_hosts, 'host', 'googleCSE')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
pass
elif engineitem == 'google-certificates':
print('\033[94m[*] Searching Google Certificate transparency report. \033[0m')
search = googlecertificates.SearchGoogleCertificates(word, limit, start)
search.process()
hosts = filter(search.get_domains())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google-certificates')
elif engineitem == 'google-profiles':
print('\033[94m[*] Searching Google profiles. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'google-profile')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
sys.exit(0)
elif engineitem == 'hunter':
elif engineitem == 'hunter':
print('\033[94m[*] Searching Hunter. \033[0m')
from discovery import huntersearch
# Import locally or won't work.
@@ -272,7 +252,7 @@ def start(argv):
else:
pass
elif engineitem == 'linkedin':
elif engineitem == 'linkedin':
print('\033[94m[*] Searching Linkedin. \033[0m')
search = linkedinsearch.SearchLinkedin(word, limit)
search.process()
@@ -281,15 +261,15 @@ def start(argv):
db.store_all(word, people, 'name', 'linkedin')
if len(people) == 0:
print('\n[*] No users found.\n\n')
print('\n[*] No users found Linkedin.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print(f'\n[*] Users found: {len(people)}')
print('---------------------')
for user in sorted(list(set(people))):
print(user)
sys.exit(0)
elif engineitem == 'netcraft':
elif engineitem == 'netcraft':
print('\033[94m[*] Searching Netcraft. \033[0m')
search = netcraft.SearchNetcraft(word)
search.process()
@@ -298,7 +278,7 @@ def start(argv):
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
elif engineitem == 'pgp':
elif engineitem == 'pgp':
print('\033[94m[*] Searching PGP key server. \033[0m')
try:
search = pgpsearch.SearchPgp(word)
@@ -312,7 +292,7 @@ def start(argv):
except Exception:
pass
elif engineitem == 'securityTrails':
elif engineitem == 'securityTrails':
print('\033[94m[*] Searching SecurityTrails. \033[0m')
from discovery import securitytrailssearch
try:
@@ -332,7 +312,7 @@ def start(argv):
else:
pass
elif engineitem == 'threatcrowd':
elif engineitem == 'threatcrowd':
print('\033[94m[*] Searching Threatcrowd. \033[0m')
try:
search = threatcrowd.search_threatcrowd(word)
@@ -344,7 +324,7 @@ def start(argv):
except Exception:
pass
elif engineitem == 'trello':
elif engineitem == 'trello':
print('\033[94m[*] Searching Trello. \033[0m')
from discovery import trello
# Import locally or won't work.
@@ -360,7 +340,7 @@ def start(argv):
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
elif engineitem == 'twitter':
elif engineitem == 'twitter':
print('\033[94m[*] Searching Twitter. \033[0m')
search = twittersearch.search_twitter(word, limit)
search.process()
@@ -379,7 +359,7 @@ def start(argv):
# vhost
elif engineitem == 'virustotal':
elif engineitem == 'virustotal':
print('\033[94m[*] Searching VirusTotal. \033[0m')
search = virustotal.search_virustotal(word)
search.process()
@@ -388,7 +368,7 @@ def start(argv):
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
elif engineitem == 'yahoo':
elif engineitem == 'yahoo':
print('\033[94m[*] Searching Yahoo. \033[0m')
search = yahoosearch.search_yahoo(word, limit)
search.process()
@@ -400,7 +380,7 @@ def start(argv):
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
elif engineitem == 'all':
elif engineitem == 'all':
print(('Full harvest on ' + word))
all_emails = []
all_hosts = []
@@ -455,8 +435,25 @@ def start(argv):
db.store_all(word, all_hosts, 'host', 'CRTsh')
# cymon
print('\033[94m[*] Searching Cymon. \033[0m')
from discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
# dogpile
print('\033[94m[*] Searching Dogpile. \033[0m')
search = dogpilesearch.SearchDogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
print('[*] Searching DuckDuckGo.')
from discovery import duckduckgosearch
@@ -524,7 +521,20 @@ def start(argv):
else:
pass
# linkedin
print('\033[94m[*] Searching Linkedin. \033[0m')
search = linkedinsearch.SearchLinkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
print('[*] Searching Netcraft.')
search = netcraft.SearchNetcraft(word)
@@ -591,7 +601,20 @@ def start(argv):
except Exception:
pass
# vhost
print('\n[*] Virtual hosts:')
print('------------------')
for l in host_ip:
search = bingsearch.SearchBing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + '\t' + x))
vhost.append(l + ':' + x)
full.append(l + ':' + x)
vhost = sorted(set(vhost))
print('[*] Searching VirusTotal.')
search = virustotal.search_virustotal(word)
@@ -611,9 +634,9 @@ def start(argv):
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
else:
print('\033[93m[!] Invalid source.\n\n \033[0m')
sys.exit(1)
else:
print('\033[93m[!] Invalid source.\n\n \033[0m')
sys.exit(1)
# Sanity check to see if all_emails and all_hosts are defined.
try:
@@ -777,16 +800,16 @@ def start(argv):
# Shodan
shodanres = []
import texttable
tab = texttable.Texttable()
header = ['IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies']
tab.header(header)
tab.set_cols_align(['c', 'c', 'c', 'c', 'c'])
tab.set_cols_valign(['m', 'm', 'm', 'm', 'm'])
tab.set_chars(['-', '|', '+', '#'])
tab.set_cols_width([15, 20, 15, 15, 18])
host_ip = list(set(host_ip))
if shodan is True:
import texttable
tab = texttable.Texttable()
header = ['IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies']
tab.header(header)
tab.set_cols_align(['c', 'c', 'c', 'c', 'c'])
tab.set_cols_valign(['m', 'm', 'm', 'm', 'm'])
tab.set_chars(['-', '|', '+', '#'])
tab.set_cols_width([15, 20, 15, 15, 18])
host_ip = list(set(host_ip))
print('\n\n[*] Shodan DB search (passive):\n')
try:
for ip in host_ip:
@@ -822,7 +845,7 @@ def start(argv):
# Reporting
if filename != "":
try:
print('NEW REPORTING BEGINS.')
print('\nNEW REPORTING BEGINS.')
db = stash.stash_manager()
scanboarddata = db.getscanboarddata()
latestscanresults = db.getlatestscanresults(word)
@@ -834,7 +857,7 @@ def start(argv):
HTMLcode = generator.beginhtml()
HTMLcode += generator.generatelatestscanresults(latestscanresults)
HTMLcode += generator.generatepreviousscanresults(previousscanresults)
graph = reportgraph.graphgenerator(word)
graph = reportgraph.GraphGenerator(word)
HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata)
HTMLcode += graph.drawscattergraphscanhistory(word, scanhistorydomain)
HTMLcode += generator.generatepluginscanstatistics(pluginscanstatistics)
@@ -915,11 +938,14 @@ def start(argv):
if __name__ == '__main__':
if python_version()[0:3] < '3.6':
print('\033[93m[!] Please make sure you have python 3.6+ installed, quitting.\033[0m')
sys.exit(1)
try:
start(sys.argv[1:])
start()
except KeyboardInterrupt:
print('\n\n\033[93m[!] ctrl+c detected from user, quitting.\n\n \033[0m')
except Exception:
import traceback
print(traceback.print_exc())
sys.exit(1)
sys.exit(1)