Merge pull request #436 from NotoriousRebel/dev

Fixed dnsbrute and made it fully async

theHarvester/__main__.py

@@ -468,20 +468,18 @@ async def start():
             print(url)
 
     # DNS brute force
-    # dnsres = []
+
     if dnsbrute is True:
         print('\n[*] Starting DNS brute force.')
-        a = dnssearch.DnsForce(word, dnsserver, verbose=True)
-        a.process()
-        # print('\n[*] Hosts found after DNS brute force:')
-        # for y in res:
-        # print('-------------------------------------')
-        #    print(y)
-        #   dnsres.append(y.split(':')[0])
-        #    if y not in full:
-        #        full.append(y)
-        # db = stash.stash_manager()
-        # db.store_all(word, dnsres, 'host', 'dns_bruteforce')
+        dns_force = dnssearch.DnsForce(word, dnsserver, verbose=True)
+        hosts, ips = await dns_force.run()
+        hosts = list({host for host in hosts if ':' in host})
+        hosts.sort(key=lambda el: el.split(':')[0])
+        print('\n[*] Hosts found after DNS brute force:')
+        db = stash.StashManager()
+        for host in hosts:
+            print(host)
+        await db.store_all(word, hosts, 'host', 'dns_bruteforce')
 
     # TakeOver Checking
 

theHarvester/discovery/crtsh.py

@@ -1,5 +1,5 @@
 from theHarvester.lib.core import *
-from typing import Set
+from typing import List
 
 
 class SearchCrtsh:
@@ -9,7 +9,7 @@ class SearchCrtsh:
         self.data = set()
         self.proxy = False
 
-    async def do_search(self) -> Set:
+    async def do_search(self) -> List:
         data: set = set()
         try:
             url = f'https://crt.sh/?q=%25.{self.word}&output=json'

theHarvester/discovery/dnssearch.py

@@ -8,13 +8,13 @@ DNS Browsing
 Explore the space around known hosts & ips for extra catches.
 """
 
-import dns
 import re
 import sys
 
 from aiodns import DNSResolver
 from ipaddress import IPv4Network
 from typing import Callable, List, Optional
+from theHarvester.lib import hostchecker
 
 # TODO: need big focus on performance and results parsing, now does the basic.
 
@@ -29,39 +29,21 @@ class DnsForce:
         self.domain = domain
         self.subdo = False
         self.verbose = verbose
-        dns.resolver.default_resolver = dns.resolver.Resolver(configure=False)
-        dns.resolver.default_resolver.nameservers = [dnsserver]
+        self.dnsserver = [dnsserver] if isinstance(dnsserver, str) else dnsserver
         try:
             with open('wordlists/dns-names.txt', 'r') as file:
                 self.list = file.readlines()
         except FileNotFoundError:
             with open('/etc/theHarvester/dns-names.txt', 'r') as file:
                 self.list = file.readlines()
+        self.domain = domain.replace('www.', '')
+        self.list = [f'{word.strip()}.{self.domain}' for word in self.list]
 
-    def run(self, host):
-        hostname = str(host.split('\n')[0]) + '.' + str(self.domain)
-        if self.verbose:
-            esc = chr(27)
-            sys.stdout.write(esc + '[2K' + esc + '[G')
-            sys.stdout.write('\r' + hostname + ' - ')
-            sys.stdout.flush()
-        try:
-            answer = dns.resolver.query(hostname, 'A')
-            print(answer.canonical_name)
-            return answer.canonical_name  # TODO: need rework all this results
-
-        except Exception:
-            pass
-
-    def process(self):
-        results = []
-        for entry in self.list:
-            host = self.run(entry)
-            if host is not None:
-                # print(' : ' + host.split(':')[1])
-                results.append(host)
-        return results
-
+    async def run(self):
+        print(f'Created checker with this many words {len(self.list)}')
+        checker = hostchecker.Checker(self.list)
+        hosts, ips = await checker.check()
+        return hosts, ips
 #####################################################################
 # DNS REVERSE
 #####################################################################

theHarvester/discovery/linkedinsearch.py

@@ -38,7 +38,9 @@ class SearchLinkedin:
 
     async def get_people(self):
         rawres = myparser.Parser(self.totalresults, self.word)
-        return await rawres.people_linkedin()
+        temp = await rawres.people_linkedin()
+        return [person for person in temp
+                if person[0] != '.' and '...' not in person and len(person.split()) != 1]
 
     async def get_links(self):
         links = myparser.Parser(self.totalresults, self.word)

theHarvester/lib/hostchecker.py

@@ -13,10 +13,13 @@ from typing import Tuple, Any
 
 class Checker:
 
-    def __init__(self, hosts: list):
+    def __init__(self, hosts: list, nameserver=False):
         self.hosts = hosts
         self.realhosts: list = []
         self.addresses: set = set()
+        self.nameserver = []
+        if nameserver:
+            self.nameserver = nameserver
 
     @staticmethod
     async def query(host, resolver) -> Tuple[str, Any]:
@@ -37,7 +40,8 @@ class Checker:
 
     async def check(self):
         loop = asyncio.get_event_loop()
-        resolver = aiodns.DNSResolver(loop=loop, timeout=4)
+        resolver = aiodns.DNSResolver(loop=loop, timeout=4) if len(self.nameserver) == 0\
+            else aiodns.DNSResolver(loop=loop, timeout=4, nameservers=self.nameserver)
         results = await self.query_all(resolver)
         for host, address in results:
             self.realhosts.append(host)