From de1f8a3adae3d763c0887a5d75ea1952d4e1d6a7 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Sat, 4 Jul 2020 15:38:11 -0400 Subject: [PATCH] Reporting should be working on linux at least... --- theHarvester.py | 4 +- theHarvester/__main__.py | 15 +++-- theHarvester/lib/statichtmlgenerator.py | 75 +++++++++++++++++++++++++ theHarvester/screenshot/screenshot.py | 24 +++++--- 4 files changed, 100 insertions(+), 18 deletions(-) diff --git a/theHarvester.py b/theHarvester.py index d9514216..71adfae0 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -21,10 +21,8 @@ if __name__ == '__main__': else: import uvloop uvloop.install() - if platform == "linux": + if "linux" in platform: import aiomultiprocess # As we are not using Windows we can change the spawn method to fork for greater performance aiomultiprocess.set_context("fork") asyncio.run(__main__.entry_point()) - -# __main__ diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index f1ad47eb..efc4f185 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -47,7 +47,6 @@ async def start(): rapiddns, securityTrails, spyse, sublist3r, suip, threatcrowd, threatminer, trello, twitter, urlscan, virustotal, yahoo, all''') - args = parser.parse_args() filename: str = args.filename dnsbrute = (args.dns_brute, False) @@ -445,8 +444,6 @@ async def start(): await asyncio.gather(*tasks, return_exceptions=True) await handler(lst=stor_lst) - return_ips = [] - # Sanity check to see if all_emails and all_hosts are defined. try: all_emails @@ -590,9 +587,11 @@ async def start(): pass # Screenshots + screenshot_tups = [] if len(args.screenshot) > 0: import time from aiomultiprocess import Pool + from itertools import chain from theHarvester.screenshot.screenshot import ScreenShotter screen_shotter = ScreenShotter(args.screenshot) await screen_shotter.verify_installation() @@ -600,7 +599,6 @@ async def start(): start = time.perf_counter() print('Filtering domains for ones we can reach') unique_resolved_domains = {url.split(':')[0]for url in full if ':' in url and 'www.' not in url} - if len(unique_resolved_domains) > 0: # First filter out ones that didn't resolve print('Attempting to visit unique resolved domains, this is ACTIVE RECON') @@ -614,12 +612,16 @@ async def start(): chunk_number = 25 for chunk in screen_shotter.chunk_list(unique_resolved_domains, chunk_number): try: - await pool.map(screen_shotter.take_screenshot, chunk) + screenshot_tups.extend(await pool.map(screen_shotter.take_screenshot, chunk)) + # screenshot_tups.append(list(chain(*await pool.map(screen_shotter.take_screenshot, chunk)))) except Exception as ee: print(f'An exception has occurred while mapping: {ee}') end = time.perf_counter() print(f"Finished taking screenshots in {end - start} seconds") + import pprint as p + p.pprint(screenshot_tups, indent=4) + # Shodan shodanres = [] if shodan is True: @@ -675,6 +677,8 @@ async def start(): HTMLcode = await generator.beginhtml() HTMLcode += await generator.generatedashboardcode(scanboarddata) HTMLcode += await generator.generatelatestscanresults(latestscanresults) + if len(screenshot_tups) > 0: + HTMLcode += await generator.generatescreenshots(screenshot_tups) HTMLcode += await generator.generatepreviousscanresults(previousscanresults) graph = reportgraph.GraphGenerator(word) await graph.init_db() @@ -698,7 +702,6 @@ async def start(): print('[*] Reporting finished.') print('[*] Saving files.') - try: # filename = filename.rsplit('.', 1)[0] + '.xml' # file = open(filename, 'w') diff --git a/theHarvester/lib/statichtmlgenerator.py b/theHarvester/lib/statichtmlgenerator.py index 8e5d1ecc..f9db0501 100644 --- a/theHarvester/lib/statichtmlgenerator.py +++ b/theHarvester/lib/statichtmlgenerator.py @@ -1,3 +1,6 @@ +from typing import List +import json + class HtmlGenerator: def __init__(self, word): @@ -189,6 +192,13 @@ var table = new Tabulator("#example-table", { theHarvester Scan Report +

theHarvester Scan Report

''' return html @@ -262,3 +272,68 @@ var table = new Tabulator("#example-table", { return html except Exception as e: print(f'Error generating scan statistics HTML code: {e}') + + @staticmethod + async def generatescreenshots(tups: List): + """ + Append screenshot content + :param tups: List of tuples, each tuple is comprised of 3 elements + 1. Domain + 2. Path to screenshot if it doesn't exist will be an empty string + 3. Html content for website + :return: html content that displays screenshots + """ + try: + html = ''' +

 

+

 

+ +

+ Screenshots +

+ +
+ +

 

+

 

+ + ''' + # var tabledata = [{date:"2020-07-03", domain:"box.netflix.com", plugin:"DNS-resolver", record:"ip", + # result:"3.220.163.143"},{date:"2020-07-03", domain:"netflix.com", plugin:"DNS-resolver", record:"ip", + # result:"34.209.106.197"},{date:"2020-07-03", domain:"netflix.com", plugin:"DNS-resolver", record:"ip", + # result:"34.212.161.97"},{date:"2020-07-03", domain:"netflix.com", plugin:"DNS-resolver", record:"ip", + # result:"34.223.232.157"}]; + + base = "var tabledata = [ " + print(f'Iterating through: {len(tups)} tuples') + for tup in tups: + date = tup[0] + domain = tup[1] + path = tup[2] + data = f'{{date:"{date}", domain:"{domain}", path:"{path}"}},' + base += data + base += '];' + return html.replace('xxxxxxxxx', base) + except Exception as e: + print(f'Error generating screenshot section: {e}') + return "" diff --git a/theHarvester/screenshot/screenshot.py b/theHarvester/screenshot/screenshot.py index 9c9d37d4..9ac9cf3a 100644 --- a/theHarvester/screenshot/screenshot.py +++ b/theHarvester/screenshot/screenshot.py @@ -1,14 +1,17 @@ """ -Screenshot module that utilizes pyppeteer in async fashion -to break urls into list and assign them to workers in a queue +Screenshot module that utilizes pyppeteer to asynchronously +take screenshots """ from pyppeteer import launch import aiohttp +import asyncio +from datetime import datetime +import json import sys -class ScreenShotter(): +class ScreenShotter: def __init__(self, output): self.output = output @@ -43,30 +46,33 @@ class ScreenShotter(): text = await resp.text("UTF-8") return f'http://{url}' if ('http' not in url and 'https' not in url) else url, text except Exception as e: - print(f'An exception has occurred while attempting to screenshot {url}: {e}') + print(f'An exception has occurred while attempting to visit {url} : {e}') return "", "" async def take_screenshot(self, url): url = f'http://{url}' if ('http' not in url and 'https' not in url) else url - # url = f'https://{url}' if ('http' not in url and 'https' not in url) else url url = url.replace('www.', '') print(f'Attempting to take a screenshot of: {url}') browser = await launch(headless=True, ignoreHTTPSErrors=True, args=["--no-sandbox"]) context = await browser.createIncognitoBrowserContext() page = await browser.newPage() + path = fr'{self.output}{self.slash}{url.replace("http://", "").replace("https://", "")}.png' + print(f'path: {path}') + date = str(datetime.utcnow()) try: # change default timeout from 30 to 35 seconds page.setDefaultNavigationTimeout(35000) await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/83.0.4103.106 Safari/537.36') - # await page.goto(url, waitUntil='networkidle0') await page.goto(url) - await page.screenshot( - {'path': f'{self.output}{self.slash}{url.replace("http://", "").replace("https://", "")}.png'}) + await page.screenshot({'path': path}) except Exception as e: - print(f'Exception occurred: {e} for: {url} ') + print(f'An exception has occurred attempting to screenshot: {url} : {e}') + path = "" finally: # Clean up everything whether screenshot is taken or not + await asyncio.sleep(2) await page.close() await context.close() await browser.close() + return date, url, path