WEBDIENSTE/theHarvester
1
0
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2026-09-18 15:48:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into master

Browse Source
This commit is contained in:
Matt
2019-09-22 14:55:10 -04:00
committed by GitHub
parent 37d1c3ac11 6de05139bf
commit c5dca64de1
9 changed files with 111 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/theHarvester.yml
+22
View File
@@ -26,63 +26,85 @@ jobs:
- name: Install dependencies
run: |
pip install -r requirements.txt
- name: Run theHarvester module baidu
run: |
python theHarvester.py -d metasploit.com -b baidu
- name: Run theHarvester module bing
run: |
python theHarvester.py -d metasploit.com -b bing
- name: Run theHarvester module censys
run: |
python theHarvester.py -d metasploit.com -b censys
- name: Run theHarvester module crtsh
run: |
python theHarvester.py -d metasploit.com -b crtsh
- name: Run theHarvester module dnsdumpster
run: |
python theHarvester.py -d metasploit.com -b dnsdumpster
- name: Run theHarvester module dogplie
run: |
python theHarvester.py -d metasploit.com -b dogpile
- name: Run theHarvester module duckduckgo
run: |
python theHarvester.py -d metasploit.com -b duckduckgo
- name: Run theHarvester module exalead
run: |
python theHarvester.py -d metasploit.com -b exalead
- name: Run theHarvester module google
run: |
python theHarvester.py -d metasploit.com -b google
- name: Run theHarvester module linkedin
run: |
python theHarvester.py -d metasploit.com -b linkedin
- name: Run theHarvester module linkedin_links
run: |
python theHarvester.py -d metasploit.com -b linkedin_links
- name: Run theHarvester module netcraft
run: |
python theHarvester.py -d metasploit.com -b netcraft
- name: Run theHarvester module Otx
run: |
python theHarvester.py -d metasploit.com -b otx
- name: Run theHarvester module threatcrowd
run: |
python theHarvester.py -d metasploit.com -b threatcrowd
- name: Run theHarvester module trello
run: |
python theHarvester.py -d metasploit.com -b trello
- name: Run theHarvester module twitter
run: |
python theHarvester.py -d metasploit.com -b twitter
- name: Run theHarvester module virustotal
run: |
python theHarvester.py -d metasploit.com -b virustotal
- name: Run theHarvester module yahoo
run: |
python theHarvester.py -d metasploit.com -b yahoo
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-line-length=127 --statistics
- name: Test with pytest
run: |
pytest
requirements.txt
+2 -2
View File
@@ -5,8 +5,8 @@ flake8==3.7.8
grequests==0.4.0
mypy==0.720
plotly==4.1.1
pytest==5.1.2
pytest==5.1.3
PyYaml==5.1.2
requests==2.22.0
shodan==1.15.0
shodan==1.17.0
texttable==1.6.2
tests/discovery/test_otx.py
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
# coding=utf-8
from theHarvester.lib.core import *
from theHarvester.discovery import otxsearch
import requests
import pytest
class TestOtx(object):
@staticmethod
def domain() -> str:
return 'metasploit.com'
def test_api(self):
base_url = f'https://otx.alienvault.com/api/v1/indicators/domain/{TestOtx.domain()}/passive_dns'
headers = {'User-Agent': Core.get_user_agent()}
request = requests.get(base_url, headers=headers)
assert request.status_code == 200
def test_search(self):
search = otxsearch.SearchOtx(TestOtx.domain())
search.process()
assert type(search.get_hostnames()) == list
def test_search_no_results(self):
search = otxsearch.SearchOtx('radiant.eu')
search.process()
assert len(search.get_hostnames()) == 0
if __name__ == '__main__':
pytest.main()
theHarvester/__main__.py
+14 -1
View File
@@ -35,7 +35,7 @@ def start():
parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, censys, crtsh, dnsdumpster,
dogpile, duckduckgo, github-code, google,
hunter, intelx,
linkedin, linkedin_links, netcraft, securityTrails, threatcrowd,
linkedin, linkedin_links, netcraft, otx, securityTrails, threatcrowd,
trello, twitter, vhost, virustotal, yahoo''')
args = parser.parse_args()
@@ -308,6 +308,19 @@ def start():
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
elif engineitem == 'otx':
print('\033[94m[*] Searching AlienVault OTX. \033[0m')
from theHarvester.discovery import otxsearch
try:
otxsearch_search = otxsearch.SearchOtx(word)
otxsearch_search.process()
hosts = filter(otxsearch_search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'otx')
except Exception as e:
print(e)
elif engineitem == 'securityTrails':
print('\033[94m[*] Searching SecurityTrails. \033[0m')
from theHarvester.discovery import securitytrailssearch
theHarvester/discovery/__init__.py
+2 -1
View File
@@ -12,6 +12,7 @@ __all__ = ['baidusearch',
'intelxsearch',
'linkedinsearch',
'netcraft',
'otxsearch',
'port_scanner',
'securitytrailssearch',
'shodansearch',
@@ -21,4 +22,4 @@ __all__ = ['baidusearch',
'twittersearch',
'virustotal',
'yahoosearch',
'yandexsearch']
]
theHarvester/discovery/crtsh.py
+8 -11
View File
@@ -9,17 +9,14 @@ class SearchCrtsh:
self.data = set()
def do_search(self) -> Set:
try:
data = set() # type: Set
url = f'https://crt.sh/?q=%25.{self.word}&output=json'
headers = {'User-Agent': Core.get_user_agent()}
request = requests.get(url, headers=headers, timeout=15)
if request.ok:
content = request.json()
data = set([dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in content])
return data
except Exception:
pass
data = set() # type: Set
url = f'https://crt.sh/?q=%25.{self.word}&output=json'
headers = {'User-Agent': Core.get_user_agent()}
request = requests.get(url, headers=headers, timeout=15)
if request.ok:
content = request.json()
data = set([dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in content])
return data
def process(self) -> None:
print('\tSearching results.')
theHarvester/discovery/otxsearch.py
+30
View File
@@ -0,0 +1,30 @@
from theHarvester.lib.core import *
from theHarvester.parsers import myparser
import grequests
class SearchOtx:
def __init__(self, word):
self.word = word
self.results = ''
self.totalresults = ''
def do_search(self):
base_url = f'https://otx.alienvault.com/api/v1/indicators/domain/{self.word}/passive_dns'
headers = {'User-Agent': Core.get_user_agent()}
try:
request = grequests.get(base_url, headers=headers)
data = grequests.map([request])
self.results = data[0].content.decode('UTF-8')
except Exception as e:
print(e)
self.totalresults += self.results
def get_hostnames(self) -> Set:
return myparser.Parser(self.totalresults, self.word).hostnames()
def process(self):
self.do_search()
self.get_hostnames()
print('\tSearching results.')
theHarvester/discovery/yandexsearch.py
-73
View File
@@ -1,73 +0,0 @@
from theHarvester.discovery.constants import *
from theHarvester.lib.core import *
from theHarvester.parsers import myparser
import re
import requests
import time
class SearchYandex:
def __init__(self, word, limit, start):
self.word = word
self.results = ""
self.totalresults = ""
self.server = 'yandex.com'
self.hostname = 'yandex.com'
self.limit = limit
self.counter = start
def do_search(self):
url = 'http://' + self.server + '/search?text=%40' + self.word + '&numdoc=50&lr=' + str(self.counter)
headers = {
'Host': self.hostname,
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.results = h.text
self.totalresults += self.results
print(self.results)
def do_search_files(self, files): # TODO
url = 'http://' + self.server + '/search?text=%40' + self.word + '&numdoc=50&lr=' + str(self.counter)
headers = {
'Host': self.hostname,
'User-agent': Core.get_user_agent()
}
h = requests.get(url=url, headers=headers)
self.results = h.text
self.totalresults += self.results
def check_next(self):
renext = re.compile('topNextUrl')
nextres = renext.findall(self.results)
if nextres != []:
nexty = '1'
print(str(self.counter))
else:
nexty = '0'
return nexty
def get_emails(self):
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.emails()
def get_hostnames(self):
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.hostnames()
def get_files(self):
rawres = myparser.Parser(self.totalresults, self.word)
return rawres.fileurls(self.files) # self.files is not init?
def process(self):
while self.counter <= self.limit:
self.do_search()
self.counter += 50
print(f'Searching {self.counter} results.')
def process_files(self, files):
while self.counter < self.limit:
self.do_search_files(files)
time.sleep(getDelay())
self.counter += 50
theHarvester/lib/core.py
+1
View File
@@ -80,6 +80,7 @@ class Core:
'linkedin',
'linkedin_links',
'netcraft',
'otx',
'securityTrails',
'threatcrowd',
'trello',