WEBDIENSTE/theHarvester
1
0
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2026-09-27 20:14:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into github-code

Browse Source 
# Conflicts:
#	theHarvester/__main__.py
This commit is contained in:
Cameron Ruatta
2019-06-29 11:31:57 -07:00
parent 195802bb09 ddf64c39ee
commit 7ff3a36ca2
8 changed files with 32 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
-2
View File
@@ -34,8 +34,6 @@ Passive:
* crtsh: Comodo Certificate search - www.crt.sh
* cymon: Cymon.io search engine - www.cymon.io
* dnsdumpster: DNSdumpster search engine - dnsdumpster.com
* dogpile: Dogpile search engine - www.dogpile.com
requirements.txt
+5 -5
View File
@@ -1,8 +1,8 @@
beautifulsoup4>=4.7.1
censys==0.0.8
plotly>=3.8.1
pytest>=4.4.1
PyYaml>=5.1
requests>=2.21.0
shodan>=1.12.1
plotly>=3.10.0
pytest>=4.6.3
PyYaml>=5.1.1
requests>=2.22.0
shodan>=1.13.0
texttable>=1.6.1
theHarvester/__main__.py
+18 -37
View File
@@ -53,7 +53,7 @@ def start():
parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False, action='store_true')
parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true')
parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str)
parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, censys, crtsh, cymon, dnsdumpster,
parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, censys, crtsh, dnsdumpster,
dogpile, duckduckgo, github-code, google,
google-certificates, hunter, intelx,
linkedin, netcraft, securityTrails, threatcrowd,
@@ -146,23 +146,13 @@ def start():
elif engineitem == 'crtsh':
print('\033[94m[*] Searching CRT.sh. \033[0m')
search = crtsh.search_crtsh(word)
search = crtsh.SearchCrtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
elif engineitem == 'cymon':
print('\033[94m[*] Searching Cymon. \033[0m')
from theHarvester.discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == 'dnsdumpster':
try:
print('\033[94m[*] Searching DNSdumpster. \033[0m')
@@ -332,7 +322,7 @@ def start():
elif engineitem == 'threatcrowd':
print('\033[94m[*] Searching Threatcrowd. \033[0m')
try:
search = threatcrowd.search_threatcrowd(word)
search = threatcrowd.SearchThreatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
@@ -443,22 +433,13 @@ def start():
db.store_all(word, uniqueips, 'ip', 'censys')
print('\033[94m[*] Searching CRT.sh. \033[0m')
search = crtsh.search_crtsh(word)
search = crtsh.SearchCrtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
print('\033[94m[*] Searching Cymon. \033[0m')
from theHarvester.discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
try:
print('\033[94m[*] Searching DNSdumpster. \033[0m')
from theHarvester.discovery import dnsdumpster
@@ -535,7 +516,7 @@ def start():
if isinstance(e, MissingKey):
print(e)
else:
pass
print(f'Error occured in hunter: {e}')
print('\033[94m[*] Searching Intelx. \033[0m')
from theHarvester.discovery import intelxsearch
@@ -600,14 +581,14 @@ def start():
print('\033[94m[*] Searching Threatcrowd. \033[0m')
try:
search = threatcrowd.search_threatcrowd(word)
search = threatcrowd.SearchThreatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'threatcrowd')
except Exception:
pass
except Exception as error:
print(error)
print('\033[94m[*] Searching Trello. \033[0m')
from theHarvester.discovery import trello
@@ -635,8 +616,8 @@ def start():
print('-------------------')
for user in people:
print(user)
except Exception:
pass
except Exception as error:
print(error)
print('\n[*] Virtual hosts:')
print('------------------')
@@ -728,7 +709,7 @@ def start():
if trello_info[1] is True:
trello_urls = trello_info[0]
if trello_urls == []:
if trello_urls is []:
print('\n[*] No URLs found.')
else:
total = len(trello_urls)
@@ -768,10 +749,9 @@ def start():
if len(openports) > 1:
print(('\t[*] Detected open ports: ' + ','.join(str(e) for e in openports)))
takeover_check = 'True'
if takeover_check == 'True':
if len(openports) > 0:
search_take = takeover.take_over(domain)
search_take.process()
if takeover_check == 'True' and len(openports) > 0:
search_take = takeover.take_over(domain)
search_take.process()
except Exception as e:
print(e)
@@ -869,9 +849,9 @@ def start():
# We have to take out the TLDs to do this.
recursion = None
if recursion:
start = 0
counter = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search = googlesearch.search_google(word, limit, counter)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
@@ -921,7 +901,7 @@ def start():
word,
shodanres,
dnstldres)
save = html.writehtml()
html.writehtml()
except Exception as e:
print(e)
print('\n\033[93m[!] An error occurred while creating the output file.\n\n \033[0m')
@@ -991,5 +971,6 @@ def entry_point():
print(traceback.print_exc())
sys.exit(1)
if __name__ == '__main__':
entry_point()
theHarvester/discovery/__init__.py
-1
View File
@@ -2,7 +2,6 @@ __all__ = ['baidusearch',
'bingsearch',
'censys',
'crtsh',
'cymon',
'dnssearch',
'dogpilesearch',
'duckduckgosearch',
theHarvester/discovery/crtsh.py
+1 -1
View File
@@ -5,7 +5,7 @@ import requests
import time
class search_crtsh:
class SearchCrtsh:
def __init__(self, word):
self.word = word.replace(' ', '%20')
theHarvester/discovery/cymon.py
-38
View File
@@ -1,38 +0,0 @@
from theHarvester.discovery.constants import *
from theHarvester.lib.core import *
from theHarvester.parsers import cymonparser
import requests
import time
class search_cymon:
def __init__(self, word):
self.word = word
self.url = ""
self.results = ""
self.server = 'cymon.io'
def do_search(self):
try:
headers = {'user-agent': Core.get_user_agent(), 'Accept': '*/*', 'Referer': self.url}
response = requests.get(self.url, headers=headers)
time.sleep(getDelay())
self.results = response.content
except Exception as e:
print(e)
def process(self):
try:
self.url = 'https://' + self.server + '/domain/' + str(self.word)
print('\tSearching results.')
self.do_search()
except Exception as e:
print(f'Error occurred: {e}')
def get_ipaddresses(self):
try:
ips = cymonparser.Parser(self)
return ips.search_ipaddresses()
except Exception as e:
print(f'Error occurred: {e}')
theHarvester/discovery/threatcrowd.py
+5 -9
View File
@@ -3,7 +3,7 @@ from theHarvester.parsers import myparser
import requests
class search_threatcrowd:
class SearchThreatcrowd:
def __init__(self, word):
self.word = word.replace(' ', '%20')
@@ -15,21 +15,17 @@ class search_threatcrowd:
self.counter = 0
def do_search(self):
try:
urly = 'https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=' + self.word
except Exception as e:
print(e)
url = f'https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={self.word}'
headers = {'User-Agent': Core.get_user_agent()}
try:
r = requests.get(urly, headers=headers)
request = requests.get(url, headers=headers)
self.results = request.text
except Exception as e:
print(e)
self.results = r.text
self.totalresults += self.results
def get_hostnames(self):
rawres = myparser.Parser(self.results, self.word)
return rawres.hostnames()
return myparser.Parser(self.results, self.word).hostnames()
def process(self):
self.do_search()
theHarvester/lib/core.py
+3 -3
View File
@@ -7,7 +7,7 @@ import yaml
class Core:
@staticmethod
def version():
return '3.1.0.dev0'
return '3.1.0.dev1'
@staticmethod
def bing_key():
@@ -377,7 +377,7 @@ class Core:
# db.store_all(word, uniqueips, 'ip', 'censys')
#
# print('[*] Searching CRT.sh.')
# search = crtsh.search_crtsh(word)
# search = crtsh.SearchCrtsh(word)
# search.process()
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)
@@ -499,7 +499,7 @@ class Core:
#
# print('[*] Searching Threatcrowd.')
# try:
# search = threatcrowd.search_threatcrowd(word)
# search = threatcrowd.SearchThreatcrowd(word)
# search.process()
# hosts = filter(search.get_hostnames())
# all_hosts.extend(hosts)