From a18dd92cc965c5f5ec026c22e830a79a919f029e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 21 Sep 2019 14:02:30 +0000 Subject: [PATCH 01/10] Bump pytest from 5.1.2 to 5.1.3 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.1.2 to 5.1.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.1.2...5.1.3) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dd796015..5ad9547a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ flake8==3.7.8 grequests==0.4.0 mypy==0.720 plotly==4.1.1 -pytest==5.1.2 +pytest==5.1.3 PyYaml==5.1.2 requests==2.22.0 shodan==1.15.0 From 11fc174fee1a6d46c464fd4c2693738c9a0c54ba Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 21 Sep 2019 18:29:35 +0000 Subject: [PATCH 02/10] Bump shodan from 1.15.0 to 1.16.0 Bumps [shodan](https://github.com/achillean/shodan-python) from 1.15.0 to 1.16.0. - [Release notes](https://github.com/achillean/shodan-python/releases) - [Changelog](https://github.com/achillean/shodan-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/achillean/shodan-python/compare/1.15.0...1.16.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5ad9547a..910817ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,5 @@ plotly==4.1.1 pytest==5.1.3 PyYaml==5.1.2 requests==2.22.0 -shodan==1.15.0 +shodan==1.16.0 texttable==1.6.2 \ No newline at end of file From 581b006c7ac6e43fecac2825101a503f0403fce9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 21 Sep 2019 23:58:05 +0000 Subject: [PATCH 03/10] Bump shodan from 1.16.0 to 1.17.0 Bumps [shodan](https://github.com/achillean/shodan-python) from 1.16.0 to 1.17.0. - [Release notes](https://github.com/achillean/shodan-python/releases) - [Changelog](https://github.com/achillean/shodan-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/achillean/shodan-python/compare/1.16.0...1.17.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 910817ba..b47a4c4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,5 @@ plotly==4.1.1 pytest==5.1.3 PyYaml==5.1.2 requests==2.22.0 -shodan==1.16.0 +shodan==1.17.0 texttable==1.6.2 \ No newline at end of file From f5f4784010c749e70d280c636f9e89cb63cce211 Mon Sep 17 00:00:00 2001 From: Jay Townsend Date: Sun, 22 Sep 2019 04:54:53 +0100 Subject: [PATCH 04/10] remove yandex module as unused Signed-off-by: Jay Townsend --- theHarvester/discovery/yandexsearch.py | 73 -------------------------- 1 file changed, 73 deletions(-) delete mode 100644 theHarvester/discovery/yandexsearch.py diff --git a/theHarvester/discovery/yandexsearch.py b/theHarvester/discovery/yandexsearch.py deleted file mode 100644 index 15f7a821..00000000 --- a/theHarvester/discovery/yandexsearch.py +++ /dev/null @@ -1,73 +0,0 @@ -from theHarvester.discovery.constants import * -from theHarvester.lib.core import * -from theHarvester.parsers import myparser -import re -import requests -import time - - -class SearchYandex: - - def __init__(self, word, limit, start): - self.word = word - self.results = "" - self.totalresults = "" - self.server = 'yandex.com' - self.hostname = 'yandex.com' - self.limit = limit - self.counter = start - - def do_search(self): - url = 'http://' + self.server + '/search?text=%40' + self.word + '&numdoc=50&lr=' + str(self.counter) - headers = { - 'Host': self.hostname, - 'User-agent': Core.get_user_agent() - } - h = requests.get(url=url, headers=headers) - self.results = h.text - self.totalresults += self.results - print(self.results) - - def do_search_files(self, files): # TODO - url = 'http://' + self.server + '/search?text=%40' + self.word + '&numdoc=50&lr=' + str(self.counter) - headers = { - 'Host': self.hostname, - 'User-agent': Core.get_user_agent() - } - h = requests.get(url=url, headers=headers) - self.results = h.text - self.totalresults += self.results - - def check_next(self): - renext = re.compile('topNextUrl') - nextres = renext.findall(self.results) - if nextres != []: - nexty = '1' - print(str(self.counter)) - else: - nexty = '0' - return nexty - - def get_emails(self): - rawres = myparser.Parser(self.totalresults, self.word) - return rawres.emails() - - def get_hostnames(self): - rawres = myparser.Parser(self.totalresults, self.word) - return rawres.hostnames() - - def get_files(self): - rawres = myparser.Parser(self.totalresults, self.word) - return rawres.fileurls(self.files) # self.files is not init? - - def process(self): - while self.counter <= self.limit: - self.do_search() - self.counter += 50 - print(f'Searching {self.counter} results.') - - def process_files(self, files): - while self.counter < self.limit: - self.do_search_files(files) - time.sleep(getDelay()) - self.counter += 50 From 8c9f9ea6308c03e69f1e90591f6be222b12daf9a Mon Sep 17 00:00:00 2001 From: Jay Townsend Date: Sun, 22 Sep 2019 04:58:25 +0100 Subject: [PATCH 05/10] remove yandex from __all__ in __init__.py Signed-off-by: Jay Townsend --- theHarvester/discovery/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theHarvester/discovery/__init__.py b/theHarvester/discovery/__init__.py index 4438a5d8..12820c4c 100644 --- a/theHarvester/discovery/__init__.py +++ b/theHarvester/discovery/__init__.py @@ -21,4 +21,4 @@ __all__ = ['baidusearch', 'twittersearch', 'virustotal', 'yahoosearch', - 'yandexsearch'] + ] From 574403a9581c45bbc21dc54351f2b3200a72c840 Mon Sep 17 00:00:00 2001 From: Jay Townsend Date: Sun, 22 Sep 2019 15:10:03 +0100 Subject: [PATCH 06/10] Add AlienVault otx module WIP Signed-off-by: Jay Townsend --- theHarvester/__main__.py | 15 ++++++++++++++- theHarvester/discovery/__init__.py | 1 + theHarvester/discovery/otxsearch.py | 30 +++++++++++++++++++++++++++++ theHarvester/lib/core.py | 1 + 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 theHarvester/discovery/otxsearch.py diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 6ac1188e..150f08e9 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -35,7 +35,7 @@ def start(): parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, censys, crtsh, dnsdumpster, dogpile, duckduckgo, github-code, google, hunter, intelx, - linkedin, linkedin_links, netcraft, securityTrails, threatcrowd, + linkedin, linkedin_links, netcraft, otx, securityTrails, threatcrowd, trello, twitter, vhost, virustotal, yahoo''') args = parser.parse_args() @@ -308,6 +308,19 @@ def start(): db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'netcraft') + elif engineitem == 'otx': + print('\033[94m[*] Searching AlienVault OTX. \033[0m') + from theHarvester.discovery import otxsearch + try: + otxsearch_search = otxsearch.SearchOtx(word) + otxsearch_search.process() + hosts = filter(otxsearch_search.get_hostnames()) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'otx') + except Exception as e: + print(e) + elif engineitem == 'securityTrails': print('\033[94m[*] Searching SecurityTrails. \033[0m') from theHarvester.discovery import securitytrailssearch diff --git a/theHarvester/discovery/__init__.py b/theHarvester/discovery/__init__.py index 12820c4c..c155cdc5 100644 --- a/theHarvester/discovery/__init__.py +++ b/theHarvester/discovery/__init__.py @@ -12,6 +12,7 @@ __all__ = ['baidusearch', 'intelxsearch', 'linkedinsearch', 'netcraft', + 'otxsearch', 'port_scanner', 'securitytrailssearch', 'shodansearch', diff --git a/theHarvester/discovery/otxsearch.py b/theHarvester/discovery/otxsearch.py new file mode 100644 index 00000000..03c1dbc2 --- /dev/null +++ b/theHarvester/discovery/otxsearch.py @@ -0,0 +1,30 @@ +from theHarvester.lib.core import * +from theHarvester.parsers import myparser +import grequests + + +class SearchOtx: + + def __init__(self, word): + self.word = word + self.results = '' + self.totalresults = '' + + def do_search(self): + base_url = f'https://otx.alienvault.com/api/v1/indicators/domain/{self.word}/passive_dns' + headers = {'User-Agent': Core.get_user_agent()} + try: + request = grequests.get(base_url, headers=headers) + data = grequests.map([request]) + self.results = data[0].content.decode('UTF-8') + except Exception as e: + print(e) + self.totalresults += self.results + + def get_hostnames(self) -> Set: + return myparser.Parser(self.totalresults, self.word).hostnames() + + def process(self): + self.do_search() + self.get_hostnames() + print('\tSearching results.') diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 4f9307cd..b03e85af 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -80,6 +80,7 @@ class Core: 'linkedin', 'linkedin_links', 'netcraft', + 'otx', 'securityTrails', 'threatcrowd', 'trello', From c0edc7bea67d63c84b1a892572a1377423c981fb Mon Sep 17 00:00:00 2001 From: Jay Townsend Date: Sun, 22 Sep 2019 15:22:40 +0100 Subject: [PATCH 07/10] fix crtsh mypy error Signed-off-by: Jay Townsend --- theHarvester/discovery/crtsh.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py index 27773d91..f0df5ecd 100644 --- a/theHarvester/discovery/crtsh.py +++ b/theHarvester/discovery/crtsh.py @@ -9,17 +9,14 @@ class SearchCrtsh: self.data = set() def do_search(self) -> Set: - try: - data = set() # type: Set - url = f'https://crt.sh/?q=%25.{self.word}&output=json' - headers = {'User-Agent': Core.get_user_agent()} - request = requests.get(url, headers=headers, timeout=15) - if request.ok: - content = request.json() - data = set([dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in content]) - return data - except Exception: - pass + data = set() # type: Set + url = f'https://crt.sh/?q=%25.{self.word}&output=json' + headers = {'User-Agent': Core.get_user_agent()} + request = requests.get(url, headers=headers, timeout=15) + if request.ok: + content = request.json() + data = set([dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in content]) + return data def process(self) -> None: print('\tSearching results.') From e9b257ad8d46816db923edd680ce71fe7c96524f Mon Sep 17 00:00:00 2001 From: Jay Townsend Date: Sun, 22 Sep 2019 15:23:34 +0100 Subject: [PATCH 08/10] Update github action with otx module and formating changes to the file Signed-off-by: Jay Townsend --- .github/workflows/theHarvester.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index eb29e14f..c78cb608 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -26,63 +26,85 @@ jobs: - name: Install dependencies run: | pip install -r requirements.txt + - name: Run theHarvester module baidu run: | python theHarvester.py -d metasploit.com -b baidu + - name: Run theHarvester module bing run: | python theHarvester.py -d metasploit.com -b bing + - name: Run theHarvester module censys run: | python theHarvester.py -d metasploit.com -b censys + - name: Run theHarvester module crtsh run: | python theHarvester.py -d metasploit.com -b crtsh + - name: Run theHarvester module dnsdumpster run: | python theHarvester.py -d metasploit.com -b dnsdumpster + - name: Run theHarvester module dogplie run: | python theHarvester.py -d metasploit.com -b dogpile + - name: Run theHarvester module duckduckgo run: | python theHarvester.py -d metasploit.com -b duckduckgo + - name: Run theHarvester module exalead run: | python theHarvester.py -d metasploit.com -b exalead + - name: Run theHarvester module google run: | python theHarvester.py -d metasploit.com -b google + - name: Run theHarvester module linkedin run: | python theHarvester.py -d metasploit.com -b linkedin + - name: Run theHarvester module linkedin_links run: | python theHarvester.py -d metasploit.com -b linkedin_links + - name: Run theHarvester module netcraft run: | python theHarvester.py -d metasploit.com -b netcraft + + - name: Run theHarvester module Otx + run: | + python theHarvester.py -d metasploit.com -b otx + - name: Run theHarvester module threatcrowd run: | python theHarvester.py -d metasploit.com -b threatcrowd - name: Run theHarvester module trello run: | python theHarvester.py -d metasploit.com -b trello + - name: Run theHarvester module twitter run: | python theHarvester.py -d metasploit.com -b twitter + - name: Run theHarvester module virustotal run: | python theHarvester.py -d metasploit.com -b virustotal + - name: Run theHarvester module yahoo run: | python theHarvester.py -d metasploit.com -b yahoo + - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-line-length=127 --statistics + - name: Test with pytest run: | pytest From 33ae370c81c082e947640ff0120241f27dafddc3 Mon Sep 17 00:00:00 2001 From: Jay Townsend Date: Sun, 22 Sep 2019 15:59:53 +0100 Subject: [PATCH 09/10] Add otx module to travis Signed-off-by: Jay Townsend --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d56623c5..7cf52def 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ before_install: install: - python setup.py test script: -- python theHarvester.py -d metasploit.com -b baidu,bing,censys,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,threatcrowd,trello,twitter,virustotal,yahoo +- python theHarvester.py -d metasploit.com -b baidu,bing,censys,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,otx,threatcrowd,trello,twitter,virustotal,yahoo - pytest - flake8 . --count --show-source --statistics #- mypy *.py From 6de05139bfb6dfbd3532bbe85db95810bbfee982 Mon Sep 17 00:00:00 2001 From: Jay Townsend Date: Sun, 22 Sep 2019 17:03:53 +0100 Subject: [PATCH 10/10] Add otx tests Signed-off-by: Jay Townsend --- tests/discovery/test_otx.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/discovery/test_otx.py diff --git a/tests/discovery/test_otx.py b/tests/discovery/test_otx.py new file mode 100644 index 00000000..67d54d10 --- /dev/null +++ b/tests/discovery/test_otx.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# coding=utf-8 +from theHarvester.lib.core import * +from theHarvester.discovery import otxsearch +import requests +import pytest + + +class TestOtx(object): + @staticmethod + def domain() -> str: + return 'metasploit.com' + + def test_api(self): + base_url = f'https://otx.alienvault.com/api/v1/indicators/domain/{TestOtx.domain()}/passive_dns' + headers = {'User-Agent': Core.get_user_agent()} + request = requests.get(base_url, headers=headers) + assert request.status_code == 200 + + def test_search(self): + search = otxsearch.SearchOtx(TestOtx.domain()) + search.process() + assert type(search.get_hostnames()) == list + + def test_search_no_results(self): + search = otxsearch.SearchOtx('radiant.eu') + search.process() + assert len(search.get_hostnames()) == 0 + + +if __name__ == '__main__': + pytest.main()