diff --git a/requirements.txt b/requirements.txt index 44940d3b..2bd9cb06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ dnspython==1.16.0 flake8==3.7.8 grequests==0.4.0 mypy==0.720 +netaddr==0.7.19 plotly==4.1.1 pytest==5.1.3 PyYaml==5.1.2 diff --git a/tests/discovery/test_otx.py b/tests/discovery/test_otx.py index 15c2ce9b..58d6d018 100644 --- a/tests/discovery/test_otx.py +++ b/tests/discovery/test_otx.py @@ -20,7 +20,7 @@ class TestOtx(object): def test_search(self): search = otxsearch.SearchOtx(TestOtx.domain()) search.process() - assert type(search.get_hostnames()) == set + assert isinstance(search.get_hostnames(), set) def test_search_no_results(self): search = otxsearch.SearchOtx('radiant.eu') diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 1dc04571..f86edce3 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -9,7 +9,7 @@ from theHarvester.lib import statichtmlgenerator from theHarvester.lib.core import * import argparse import datetime -import ipaddress +import netaddr import re import sys import time @@ -315,9 +315,10 @@ def start(): otxsearch_search = otxsearch.SearchOtx(word) otxsearch_search.process() hosts = filter(otxsearch_search.get_hostnames()) - all_hosts.extend(hosts) + all_hosts.extend(list(hosts)) ips = filter(otxsearch_search.get_ips()) - all_ip.extend(ips) + all_ip.extend(list(ips)) + all_hosts.extend(hosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'otx') db.store_all(word, all_ip, 'ip', 'otx') @@ -433,8 +434,11 @@ def start(): else: print('\n[*] IPs found: ' + str(len(all_ip))) print('-------------------') - ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip)) - print('\n'.join(map(str, ips))) + # ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip)) + # print('\n'.join(map(str, ips))) + ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)]) + # use netaddr as the list may contain ipv4 and ipv6 addresses + print('\n'.join(map(str, ip_list))) if len(all_emails) == 0: print('\n[*] No emails found.') diff --git a/theHarvester/discovery/otxsearch.py b/theHarvester/discovery/otxsearch.py index 328b7beb..7d0da415 100644 --- a/theHarvester/discovery/otxsearch.py +++ b/theHarvester/discovery/otxsearch.py @@ -1,6 +1,6 @@ from theHarvester.lib.core import * -import grequests import json +import grequests class SearchOtx: @@ -21,14 +21,16 @@ class SearchOtx: self.results = data[0].content.decode('UTF-8') except Exception as e: print(e) + self.totalresults += self.results dct = json.loads(self.totalresults) - self.totalhosts = {host['hostname'] for host in dct['passive_dns']} - self.totalips = {ip['address'] for ip in dct['passive_dns'] if 'NXDOMAIN' not in ip['address']} + self.totalhosts: set = {host['hostname'] for host in dct['passive_dns']} + self.totalips: set = {ip['address'] for ip in dct['passive_dns'] if 'NXDOMAIN' not in ip['address']} + # filter out ips that are just called NXDOMAIN def get_hostnames(self) -> set: return self.totalhosts - + def get_ips(self) -> set: return self.totalips