From 82e1b23f749664af5e0fa4c4a6618172532f4c11 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Mon, 30 Dec 2019 22:56:54 -0500 Subject: [PATCH 1/3] Ported netcraft to use aiohttp. --- theHarvester/discovery/netcraft.py | 77 +++++++++++++++++++----------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/theHarvester/discovery/netcraft.py b/theHarvester/discovery/netcraft.py index 0e5d4519..1998208b 100644 --- a/theHarvester/discovery/netcraft.py +++ b/theHarvester/discovery/netcraft.py @@ -1,9 +1,10 @@ from theHarvester.lib.core import * from theHarvester.parsers import myparser -import requests import hashlib import urllib.parse as urllib import re +import aiohttp +import asyncio class SearchNetcraft: @@ -14,30 +15,36 @@ class SearchNetcraft: self.totalresults = "" self.server = 'netcraft.com' self.base_url = f'https://searchdns.netcraft.com/?restriction=site+ends+with&host={word}' - self.session = requests.session() + self.session = None self.headers = { 'User-Agent': Core.get_user_agent() } - self.timeout = 25 + self.timeout = aiohttp.ClientTimeout(total=25) self.domain = f"https://searchdns.netcraft.com/?restriction=site+ends+with&host={self.word}" - def request(self, url, cookies=None): - cookies = cookies or {} + async def request(self, url, first=False): try: - resp = self.session.get(url, headers=self.headers, timeout=self.timeout, cookies=cookies) + if first: + async with aiohttp.ClientSession(headers=self.headers, timeout=self.timeout) as sess: + async with sess.get(url) as resp: + await asyncio.sleep(3) + return resp.headers + else: + async with self.session.get(url) as sess: + await asyncio.sleep(2) + return await sess.text() except Exception as e: - print(e) resp = None return resp - def get_next(self, resp): + async def get_next(self, resp): link_regx = re.compile('Next page') link = link_regx.findall(resp) link = re.sub(f'host=.*?{self.word}', f'host={self.domain}', link[0]) url = f'https://searchdns.netcraft.com{link.replace(" ", "%20")}' return url - def create_cookies(self, cookie): + async def create_cookies(self, cookie): cookies = dict() cookies_list = cookie[0:cookie.find(';')].split("=") cookies[cookies_list[0]] = cookies_list[1] @@ -46,27 +53,43 @@ class SearchNetcraft: urllib.unquote(cookies_list[1]).encode('utf-8')).hexdigest() return cookies - def get_cookies(self, headers): - if 'set-cookie' in headers: - cookies = self.create_cookies(headers['set-cookie']) - else: - cookies = {} + async def get_cookies(self, headers): + try: + if headers is None: + return {} + elif 'set-cookie' in headers: + cookies = await self.create_cookies(headers['set-cookie']) + else: + cookies = {} + except Exception as e: + return {} return cookies - def do_search(self): - start_url = self.base_url - resp = self.request(start_url) - cookies = self.get_cookies(resp.headers) - while True: - resp = self.request(self.base_url, cookies).text - self.totalresults += resp - if 'Next page' not in resp or resp is None: - break - self.base_url = self.get_next(resp) + async def do_search(self): + try: + start_url = self.base_url + resp = await self.request(start_url, first=True) + # indicates this is the start_url to retrieve cookie we need + cookies = await self.get_cookies(resp) + self.session = aiohttp.ClientSession(headers=self.headers, timeout=self.timeout, cookies=cookies) + while True: + resp = await self.request(self.base_url) + if isinstance(resp, str): + self.totalresults += resp + if 'Next page' not in resp or resp is None: + await self.session.close() + break + self.base_url = await self.get_next(resp) + except Exception as e: + try: + await self.session.close() + except Exception: + pass - def get_hostnames(self): + async def get_hostnames(self): rawres = myparser.Parser(self.totalresults, self.word) return rawres.hostnames() - def process(self): - self.do_search() + async def process(self): + await self.do_search() + From 6ef32cfce5ddf806f526f486a9fea26143f7d1f3 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Mon, 30 Dec 2019 23:00:06 -0500 Subject: [PATCH 2/3] Made pep8 compliant. --- theHarvester/discovery/netcraft.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theHarvester/discovery/netcraft.py b/theHarvester/discovery/netcraft.py index 1998208b..ac18edc9 100644 --- a/theHarvester/discovery/netcraft.py +++ b/theHarvester/discovery/netcraft.py @@ -34,6 +34,7 @@ class SearchNetcraft: await asyncio.sleep(2) return await sess.text() except Exception as e: + print(e) resp = None return resp @@ -92,4 +93,3 @@ class SearchNetcraft: async def process(self): await self.do_search() - From e5c4c9de36c158a103ba1abb8d09e6ab36c7dc88 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Mon, 30 Dec 2019 23:07:38 -0500 Subject: [PATCH 3/3] Added docstring. --- theHarvester/discovery/netcraft.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/theHarvester/discovery/netcraft.py b/theHarvester/discovery/netcraft.py index ac18edc9..966aa385 100644 --- a/theHarvester/discovery/netcraft.py +++ b/theHarvester/discovery/netcraft.py @@ -25,6 +25,7 @@ class SearchNetcraft: async def request(self, url, first=False): try: if first: + # indicates first request to extract cookie async with aiohttp.ClientSession(headers=self.headers, timeout=self.timeout) as sess: async with sess.get(url) as resp: await asyncio.sleep(3) @@ -57,6 +58,7 @@ class SearchNetcraft: async def get_cookies(self, headers): try: if headers is None: + # In this case just return default dict return {} elif 'set-cookie' in headers: cookies = await self.create_cookies(headers['set-cookie'])