Merge pull request #22 from NotoriousRebel/master

Fixed ip sorting.

requirements.txt

@@ -5,6 +5,7 @@ dnspython==1.16.0
 flake8==3.7.8
 grequests==0.4.0
 mypy==0.720
+netaddr==0.7.19
 plotly==4.1.1
 pytest==5.1.3
 PyYaml==5.1.2

tests/discovery/test_otx.py

@@ -20,7 +20,7 @@ class TestOtx(object):
     def test_search(self):
         search = otxsearch.SearchOtx(TestOtx.domain())
         search.process()
-        assert type(search.get_hostnames()) == set
+        assert isinstance(search.get_hostnames(), set)
 
     def test_search_no_results(self):
         search = otxsearch.SearchOtx('radiant.eu')

theHarvester/__main__.py

@@ -9,7 +9,7 @@ from theHarvester.lib import statichtmlgenerator
 from theHarvester.lib.core import *
 import argparse
 import datetime
-import ipaddress
+import netaddr
 import re
 import sys
 import time
@@ -315,9 +315,10 @@ def start():
                         otxsearch_search = otxsearch.SearchOtx(word)
                         otxsearch_search.process()
                         hosts = filter(otxsearch_search.get_hostnames())
-                        all_hosts.extend(hosts)
+                        all_hosts.extend(list(hosts))
                         ips = filter(otxsearch_search.get_ips())
-                        all_ip.extend(ips)
+                        all_ip.extend(list(ips))
+                        all_hosts.extend(hosts)
                         db = stash.stash_manager()
                         db.store_all(word, all_hosts, 'host', 'otx')
                         db.store_all(word, all_ip, 'ip', 'otx')
@@ -433,8 +434,11 @@ def start():
     else:
         print('\n[*] IPs found: ' + str(len(all_ip)))
         print('-------------------')
-        ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip))
-        print('\n'.join(map(str, ips)))
+        # ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip))
+        # print('\n'.join(map(str, ips)))
+        ip_list = sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)])
+        # use netaddr as the list may contain ipv4 and ipv6 addresses
+        print('\n'.join(map(str, ip_list)))
 
     if len(all_emails) == 0:
         print('\n[*] No emails found.')

theHarvester/discovery/otxsearch.py

@@ -1,6 +1,6 @@
 from theHarvester.lib.core import *
-import grequests
 import json
+import grequests
 
 
 class SearchOtx:
@@ -21,14 +21,16 @@ class SearchOtx:
             self.results = data[0].content.decode('UTF-8')
         except Exception as e:
             print(e)
+
         self.totalresults += self.results
         dct = json.loads(self.totalresults)
-        self.totalhosts = {host['hostname'] for host in dct['passive_dns']}
-        self.totalips = {ip['address'] for ip in dct['passive_dns'] if 'NXDOMAIN' not in ip['address']}
+        self.totalhosts: set = {host['hostname'] for host in dct['passive_dns']}
+        self.totalips: set = {ip['address'] for ip in dct['passive_dns'] if 'NXDOMAIN' not in ip['address']}
+        # filter out ips that are just called NXDOMAIN
 
     def get_hostnames(self) -> set:
         return self.totalhosts
-
+      
     def get_ips(self) -> set:
         return self.totalips