diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index a958f8ee..0d28c31d 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -35,6 +35,10 @@ jobs: run: | python theHarvester.py -d metasploit.com -b bing + - name: Run theHarvester module certspotter + run: | + python theHarvester.py -d metasploit.com -b certspotter + - name: Run theHarvester module crtsh run: | python theHarvester.py -d metasploit.com -b crtsh diff --git a/.travis.yml b/.travis.yml index 92df7b7e..2e952d9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ before_install: install: - python setup.py test script: -- python theHarvester.py -d metasploit.com -b baidu,bing,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,otx,intelx,threatcrowd,trello,twitter,virustotal,yahoo -l 200 +- python theHarvester.py -d metasploit.com -b baidu,bing,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,otx,intelx,threatcrowd,trello,twitter,virustotal,yahoo -l 200 - pytest - flake8 . --count --show-source --statistics #- mypy --pretty *.py diff --git a/README.md b/README.md index 8f17e7cc..6638cdab 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ Passive: * bingapi: Microsoft search engine, through the API (Requires API key, see below.) +* CertSporter: Cert Spotter monitors Certificate Transparency logs - https://sslmate.com/certspotter/ + * crtsh: Comodo Certificate search - www.crt.sh * dnsdumpster: DNSdumpster search engine - dnsdumpster.com diff --git a/tests/discovery/test_certspotter.py b/tests/discovery/test_certspotter.py new file mode 100644 index 00000000..a96c49ba --- /dev/null +++ b/tests/discovery/test_certspotter.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# coding=utf-8 +from theHarvester.lib.core import * +from theHarvester.discovery import certspottersearch +import requests +import pytest + + +class TestCertspotter(object): + @staticmethod + def domain() -> str: + return 'metasploit.com' + + def test_api(self): + base_url = f'https://api.certspotter.com/v1/issuances?domain={TestCertspotter.domain()}&expand=dns_names' + headers = {'User-Agent': Core.get_user_agent()} + request = requests.get(base_url, headers=headers) + assert request.status_code == 200 + + def test_search(self): + search = certspottersearch.SearchCertspoter(TestCertspotter.domain()) + search.process() + assert isinstance(search.get_hostnames(), set) + + def test_search_no_results(self): + search = certspottersearch.SearchCertspoter('radiant.eu') + search.process() + assert len(search.get_hostnames()) == 0 + + +if __name__ == '__main__': + pytest.main() diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 102df6c1..d8eabd59 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -34,7 +34,7 @@ def start(): parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False, action='store_true') parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true') parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) - parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, crtsh, dnsdumpster, + parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, certspotter, crtsh, dnsdumpster, dogpile, duckduckgo, github-code, google, hunter, intelx, linkedin, linkedin_links, netcraft, otx, securityTrails, spyse(disabled for now), threatcrowd, @@ -117,6 +117,20 @@ def start(): else: pass + elif engineitem == 'certspotter': + print('\033[94m[*] Searching CertSpotter. \033[0m') + from theHarvester.discovery import certspottersearch + try: + certspotter_search = certspottersearch.SearchCertspoter(word) + certspotter_search.process() + hosts = filter(certspotter_search.get_hostnames()) + all_hosts.extend(list(hosts)) + all_hosts.extend(hosts) + db = stash.stash_manager() + db.store_all(word, all_hosts, 'host', 'certspotter') + except Exception as e: + print(e) + elif engineitem == 'crtsh': try: print('\033[94m[*] Searching CRT.sh. \033[0m') diff --git a/theHarvester/discovery/__init__.py b/theHarvester/discovery/__init__.py index bf99b950..e9ab54b1 100644 --- a/theHarvester/discovery/__init__.py +++ b/theHarvester/discovery/__init__.py @@ -1,6 +1,7 @@ __all__ = ['baidusearch', 'bingsearch', 'crtsh', + 'certspottersearch', 'dnssearch', 'dogpilesearch', 'duckduckgosearch', diff --git a/theHarvester/discovery/certspottersearch.py b/theHarvester/discovery/certspottersearch.py new file mode 100644 index 00000000..0eb9cd78 --- /dev/null +++ b/theHarvester/discovery/certspottersearch.py @@ -0,0 +1,29 @@ +from theHarvester.lib.core import * +import requests + + +class SearchCertspoter: + + def __init__(self, word): + self.word = word + self.totalhosts = set() + + def do_search(self) -> None: + base_url = f'https://api.certspotter.com/v1/issuances?domain={self.word}&expand=dns_names' + headers = {'User-Agent': Core.get_user_agent()} + try: + request = requests.get(base_url, headers=headers) + response = request.json() + for dct in response: + for key, value in dct.items(): + if key == 'dns_names': + self.totalhosts.update({name for name in value if name}) + except Exception as e: + print(e) + + def get_hostnames(self) -> set: + return self.totalhosts + + def process(self): + self.do_search() + print('\tSearching results.') diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 7a912b6f..0c4439fd 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -73,6 +73,7 @@ class Core: supportedengines = {'baidu', 'bing', 'bingapi', + 'certspotter', 'crtsh', 'dnsdumpster', 'dogpile',