diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index eb29e14f..c78cb608 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -26,63 +26,85 @@ jobs: - name: Install dependencies run: | pip install -r requirements.txt + - name: Run theHarvester module baidu run: | python theHarvester.py -d metasploit.com -b baidu + - name: Run theHarvester module bing run: | python theHarvester.py -d metasploit.com -b bing + - name: Run theHarvester module censys run: | python theHarvester.py -d metasploit.com -b censys + - name: Run theHarvester module crtsh run: | python theHarvester.py -d metasploit.com -b crtsh + - name: Run theHarvester module dnsdumpster run: | python theHarvester.py -d metasploit.com -b dnsdumpster + - name: Run theHarvester module dogplie run: | python theHarvester.py -d metasploit.com -b dogpile + - name: Run theHarvester module duckduckgo run: | python theHarvester.py -d metasploit.com -b duckduckgo + - name: Run theHarvester module exalead run: | python theHarvester.py -d metasploit.com -b exalead + - name: Run theHarvester module google run: | python theHarvester.py -d metasploit.com -b google + - name: Run theHarvester module linkedin run: | python theHarvester.py -d metasploit.com -b linkedin + - name: Run theHarvester module linkedin_links run: | python theHarvester.py -d metasploit.com -b linkedin_links + - name: Run theHarvester module netcraft run: | python theHarvester.py -d metasploit.com -b netcraft + + - name: Run theHarvester module Otx + run: | + python theHarvester.py -d metasploit.com -b otx + - name: Run theHarvester module threatcrowd run: | python theHarvester.py -d metasploit.com -b threatcrowd - name: Run theHarvester module trello run: | python theHarvester.py -d metasploit.com -b trello + - name: Run theHarvester module twitter run: | python theHarvester.py -d metasploit.com -b twitter + - name: Run theHarvester module virustotal run: | python theHarvester.py -d metasploit.com -b virustotal + - name: Run theHarvester module yahoo run: | python theHarvester.py -d metasploit.com -b yahoo + - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-line-length=127 --statistics + - name: Test with pytest run: | pytest diff --git a/requirements.txt b/requirements.txt index dd796015..b47a4c4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,8 +5,8 @@ flake8==3.7.8 grequests==0.4.0 mypy==0.720 plotly==4.1.1 -pytest==5.1.2 +pytest==5.1.3 PyYaml==5.1.2 requests==2.22.0 -shodan==1.15.0 +shodan==1.17.0 texttable==1.6.2 \ No newline at end of file diff --git a/tests/discovery/test_otx.py b/tests/discovery/test_otx.py new file mode 100644 index 00000000..67d54d10 --- /dev/null +++ b/tests/discovery/test_otx.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# coding=utf-8 +from theHarvester.lib.core import * +from theHarvester.discovery import otxsearch +import requests +import pytest + + +class TestOtx(object): + @staticmethod + def domain() -> str: + return 'metasploit.com' + + def test_api(self): + base_url = f'https://otx.alienvault.com/api/v1/indicators/domain/{TestOtx.domain()}/passive_dns' + headers = {'User-Agent': Core.get_user_agent()} + request = requests.get(base_url, headers=headers) + assert request.status_code == 200 + + def test_search(self): + search = otxsearch.SearchOtx(TestOtx.domain()) + search.process() + assert type(search.get_hostnames()) == list + + def test_search_no_results(self): + search = otxsearch.SearchOtx('radiant.eu') + search.process() + assert len(search.get_hostnames()) == 0 + + +if __name__ == '__main__': + pytest.main() diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index f0dc79b7..55766741 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -35,7 +35,7 @@ def start(): parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, censys, crtsh, dnsdumpster, dogpile, duckduckgo, github-code, google, hunter, intelx, - linkedin, linkedin_links, netcraft, securityTrails, threatcrowd, + linkedin, linkedin_links, netcraft, otx, securityTrails, threatcrowd, trello, twitter, vhost, virustotal, yahoo''') args = parser.parse_args() @@ -308,6 +308,19 @@ def start(): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'netcraft') + elif engineitem == 'otx': + print('\033[94m[*] Searching AlienVault OTX. \033[0m') + from theHarvester.discovery import otxsearch + try: + otxsearch_search = otxsearch.SearchOtx(word) + otxsearch_search.process() + hosts = filter(otxsearch_search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'otx') + except Exception as e: + print(e) + elif engineitem == 'securityTrails': print('\033[94m[*] Searching SecurityTrails. \033[0m') from theHarvester.discovery import securitytrailssearch diff --git a/theHarvester/discovery/__init__.py b/theHarvester/discovery/__init__.py index 4438a5d8..c155cdc5 100644 --- a/theHarvester/discovery/__init__.py +++ b/theHarvester/discovery/__init__.py @@ -12,6 +12,7 @@ __all__ = ['baidusearch', 'intelxsearch', 'linkedinsearch', 'netcraft', + 'otxsearch', 'port_scanner', 'securitytrailssearch', 'shodansearch', @@ -21,4 +22,4 @@ __all__ = ['baidusearch', 'twittersearch', 'virustotal', 'yahoosearch', - 'yandexsearch'] + ] diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py index 27773d91..f0df5ecd 100644 --- a/theHarvester/discovery/crtsh.py +++ b/theHarvester/discovery/crtsh.py @@ -9,17 +9,14 @@ class SearchCrtsh: self.data = set() def do_search(self) -> Set: - try: - data = set() # type: Set - url = f'https://crt.sh/?q=%25.{self.word}&output=json' - headers = {'User-Agent': Core.get_user_agent()} - request = requests.get(url, headers=headers, timeout=15) - if request.ok: - content = request.json() - data = set([dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in content]) - return data - except Exception: - pass + data = set() # type: Set + url = f'https://crt.sh/?q=%25.{self.word}&output=json' + headers = {'User-Agent': Core.get_user_agent()} + request = requests.get(url, headers=headers, timeout=15) + if request.ok: + content = request.json() + data = set([dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in content]) + return data def process(self) -> None: print('\tSearching results.') diff --git a/theHarvester/discovery/otxsearch.py b/theHarvester/discovery/otxsearch.py new file mode 100644 index 00000000..03c1dbc2 --- /dev/null +++ b/theHarvester/discovery/otxsearch.py @@ -0,0 +1,30 @@ +from theHarvester.lib.core import * +from theHarvester.parsers import myparser +import grequests + + +class SearchOtx: + + def __init__(self, word): + self.word = word + self.results = '' + self.totalresults = '' + + def do_search(self): + base_url = f'https://otx.alienvault.com/api/v1/indicators/domain/{self.word}/passive_dns' + headers = {'User-Agent': Core.get_user_agent()} + try: + request = grequests.get(base_url, headers=headers) + data = grequests.map([request]) + self.results = data[0].content.decode('UTF-8') + except Exception as e: + print(e) + self.totalresults += self.results + + def get_hostnames(self) -> Set: + return myparser.Parser(self.totalresults, self.word).hostnames() + + def process(self): + self.do_search() + self.get_hostnames() + print('\tSearching results.') diff --git a/theHarvester/discovery/yandexsearch.py b/theHarvester/discovery/yandexsearch.py deleted file mode 100644 index 15f7a821..00000000 --- a/theHarvester/discovery/yandexsearch.py +++ /dev/null @@ -1,73 +0,0 @@ -from theHarvester.discovery.constants import * -from theHarvester.lib.core import * -from theHarvester.parsers import myparser -import re -import requests -import time - - -class SearchYandex: - - def __init__(self, word, limit, start): - self.word = word - self.results = "" - self.totalresults = "" - self.server = 'yandex.com' - self.hostname = 'yandex.com' - self.limit = limit - self.counter = start - - def do_search(self): - url = 'http://' + self.server + '/search?text=%40' + self.word + '&numdoc=50&lr=' + str(self.counter) - headers = { - 'Host': self.hostname, - 'User-agent': Core.get_user_agent() - } - h = requests.get(url=url, headers=headers) - self.results = h.text - self.totalresults += self.results - print(self.results) - - def do_search_files(self, files): # TODO - url = 'http://' + self.server + '/search?text=%40' + self.word + '&numdoc=50&lr=' + str(self.counter) - headers = { - 'Host': self.hostname, - 'User-agent': Core.get_user_agent() - } - h = requests.get(url=url, headers=headers) - self.results = h.text - self.totalresults += self.results - - def check_next(self): - renext = re.compile('topNextUrl') - nextres = renext.findall(self.results) - if nextres != []: - nexty = '1' - print(str(self.counter)) - else: - nexty = '0' - return nexty - - def get_emails(self): - rawres = myparser.Parser(self.totalresults, self.word) - return rawres.emails() - - def get_hostnames(self): - rawres = myparser.Parser(self.totalresults, self.word) - return rawres.hostnames() - - def get_files(self): - rawres = myparser.Parser(self.totalresults, self.word) - return rawres.fileurls(self.files) # self.files is not init? - - def process(self): - while self.counter <= self.limit: - self.do_search() - self.counter += 50 - print(f'Searching {self.counter} results.') - - def process_files(self, files): - while self.counter < self.limit: - self.do_search_files(files) - time.sleep(getDelay()) - self.counter += 50 diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 4f9307cd..b03e85af 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -80,6 +80,7 @@ class Core: 'linkedin', 'linkedin_links', 'netcraft', + 'otx', 'securityTrails', 'threatcrowd', 'trello',