From 87c2cb8a27086020ca3dd7bd193cd2525cab4841 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Fri, 15 May 2020 14:45:08 -0400 Subject: [PATCH 1/5] updated linkedin filtering. --- theHarvester/discovery/linkedinsearch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/theHarvester/discovery/linkedinsearch.py b/theHarvester/discovery/linkedinsearch.py index a3e10dce..11c5fb41 100644 --- a/theHarvester/discovery/linkedinsearch.py +++ b/theHarvester/discovery/linkedinsearch.py @@ -38,7 +38,9 @@ class SearchLinkedin: async def get_people(self): rawres = myparser.Parser(self.totalresults, self.word) - return await rawres.people_linkedin() + temp = await rawres.people_linkedin() + return [person for person in temp + if person[0] != '.' and '...' not in person and len(person.split()) != 1] async def get_links(self): links = myparser.Parser(self.totalresults, self.word) From 080c21a753ef44d0e0d314974bb55fada7f1ccbf Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Fri, 15 May 2020 18:34:13 -0400 Subject: [PATCH 2/5] Updated dnsbrute force to be async. --- theHarvester/__main__.py | 22 +++++++++--------- theHarvester/discovery/dnssearch.py | 35 ++++++++--------------------- theHarvester/lib/hostchecker.py | 8 +++++-- 3 files changed, 25 insertions(+), 40 deletions(-) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index f69889ef..3fe34451 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -468,20 +468,18 @@ async def start(): print(url) # DNS brute force - # dnsres = [] + if dnsbrute is True: print('\n[*] Starting DNS brute force.') - a = dnssearch.DnsForce(word, dnsserver, verbose=True) - a.process() - # print('\n[*] Hosts found after DNS brute force:') - # for y in res: - # print('-------------------------------------') - # print(y) - # dnsres.append(y.split(':')[0]) - # if y not in full: - # full.append(y) - # db = stash.stash_manager() - # db.store_all(word, dnsres, 'host', 'dns_bruteforce') + dns_force = dnssearch.DnsForce(word, dnsserver, verbose=True) + hosts, ips = await dns_force.run() + hosts = list({host for host in hosts if ':' in host}) + hosts.sort(key=lambda el: el.split(':')[0]) + print('\n[*] Hosts found after DNS brute force:') + db = stash.StashManager() + for host in hosts: + print(host) + await db.store_all(word, host, 'host', 'dns_bruteforce') # TakeOver Checking diff --git a/theHarvester/discovery/dnssearch.py b/theHarvester/discovery/dnssearch.py index 323f54fa..efaacdd2 100644 --- a/theHarvester/discovery/dnssearch.py +++ b/theHarvester/discovery/dnssearch.py @@ -15,6 +15,7 @@ import sys from aiodns import DNSResolver from ipaddress import IPv4Network from typing import Callable, List, Optional +from theHarvester.lib import hostchecker # TODO: need big focus on performance and results parsing, now does the basic. @@ -29,39 +30,21 @@ class DnsForce: self.domain = domain self.subdo = False self.verbose = verbose - dns.resolver.default_resolver = dns.resolver.Resolver(configure=False) - dns.resolver.default_resolver.nameservers = [dnsserver] + self.dnsserver = [dnsserver] if isinstance(dnsserver, str) else dnsserver try: with open('wordlists/dns-names.txt', 'r') as file: self.list = file.readlines() except FileNotFoundError: with open('/etc/theHarvester/dns-names.txt', 'r') as file: self.list = file.readlines() + self.domain = domain.replace('www.', '') + self.list = [f'{word.strip()}.{self.domain}' for word in self.list] - def run(self, host): - hostname = str(host.split('\n')[0]) + '.' + str(self.domain) - if self.verbose: - esc = chr(27) - sys.stdout.write(esc + '[2K' + esc + '[G') - sys.stdout.write('\r' + hostname + ' - ') - sys.stdout.flush() - try: - answer = dns.resolver.query(hostname, 'A') - print(answer.canonical_name) - return answer.canonical_name # TODO: need rework all this results - - except Exception: - pass - - def process(self): - results = [] - for entry in self.list: - host = self.run(entry) - if host is not None: - # print(' : ' + host.split(':')[1]) - results.append(host) - return results - + async def run(self): + print(f'Created checker with this many words {len(self.list)}') + checker = hostchecker.Checker(self.list) + hosts, ips = await checker.check() + return hosts, ips ##################################################################### # DNS REVERSE ##################################################################### diff --git a/theHarvester/lib/hostchecker.py b/theHarvester/lib/hostchecker.py index bc19a18d..b8127eb5 100644 --- a/theHarvester/lib/hostchecker.py +++ b/theHarvester/lib/hostchecker.py @@ -13,10 +13,13 @@ from typing import Tuple, Any class Checker: - def __init__(self, hosts: list): + def __init__(self, hosts: list, nameserver=False): self.hosts = hosts self.realhosts: list = [] self.addresses: set = set() + self.nameserver = [] + if nameserver: + self.nameserver = nameserver @staticmethod async def query(host, resolver) -> Tuple[str, Any]: @@ -37,7 +40,8 @@ class Checker: async def check(self): loop = asyncio.get_event_loop() - resolver = aiodns.DNSResolver(loop=loop, timeout=4) + resolver = aiodns.DNSResolver(loop=loop, timeout=4) if len(self.nameserver) == 0\ + else aiodns.DNSResolver(loop=loop, timeout=4, nameservers=self.nameserver) results = await self.query_all(resolver) for host, address in results: self.realhosts.append(host) From a37d3cbd5dc7767337f973a554a3e1fc26aca1f9 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Fri, 15 May 2020 19:01:22 -0400 Subject: [PATCH 3/5] updated way data is stored in database. --- theHarvester/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 3fe34451..81b8ae74 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -479,7 +479,7 @@ async def start(): db = stash.StashManager() for host in hosts: print(host) - await db.store_all(word, host, 'host', 'dns_bruteforce') + await db.store_all(word, hosts, 'host', 'dns_bruteforce') # TakeOver Checking From 2359a94500e4e7f38027250db3fcbf35457410cf Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Fri, 15 May 2020 19:09:08 -0400 Subject: [PATCH 4/5] fixed pep8 error. --- theHarvester/discovery/dnssearch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/theHarvester/discovery/dnssearch.py b/theHarvester/discovery/dnssearch.py index efaacdd2..26114e6f 100644 --- a/theHarvester/discovery/dnssearch.py +++ b/theHarvester/discovery/dnssearch.py @@ -8,7 +8,6 @@ DNS Browsing Explore the space around known hosts & ips for extra catches. """ -import dns import re import sys From 03b48dfe01594cf8524a8d4fbb62af897808e423 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Fri, 15 May 2020 19:38:06 -0400 Subject: [PATCH 5/5] fixed mypy error. --- theHarvester/discovery/crtsh.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py index 97fb3341..8c1696f5 100644 --- a/theHarvester/discovery/crtsh.py +++ b/theHarvester/discovery/crtsh.py @@ -1,5 +1,5 @@ from theHarvester.lib.core import * -from typing import Set +from typing import List class SearchCrtsh: @@ -9,7 +9,7 @@ class SearchCrtsh: self.data = set() self.proxy = False - async def do_search(self) -> Set: + async def do_search(self) -> List: data: set = set() try: url = f'https://crt.sh/?q=%25.{self.word}&output=json'