mirror of
https://github.com/laramies/theHarvester.git
synced 2026-09-22 01:24:56 +02:00
Reporting should be working on linux at least...
This commit is contained in:
NotoriousRebel
+1
-3
@@ -21,10 +21,8 @@ if __name__ == '__main__':
|
||||
else:
|
||||
import uvloop
|
||||
uvloop.install()
|
||||
if platform == "linux":
|
||||
if "linux" in platform:
|
||||
import aiomultiprocess
|
||||
# As we are not using Windows we can change the spawn method to fork for greater performance
|
||||
aiomultiprocess.set_context("fork")
|
||||
asyncio.run(__main__.entry_point())
|
||||
|
||||
# __main__
|
||||
|
||||
@@ -47,7 +47,6 @@ async def start():
|
||||
rapiddns, securityTrails, spyse, sublist3r, suip, threatcrowd, threatminer,
|
||||
trello, twitter, urlscan, virustotal, yahoo, all''')
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
filename: str = args.filename
|
||||
dnsbrute = (args.dns_brute, False)
|
||||
@@ -445,8 +444,6 @@ async def start():
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
await handler(lst=stor_lst)
|
||||
return_ips = []
|
||||
|
||||
# Sanity check to see if all_emails and all_hosts are defined.
|
||||
try:
|
||||
all_emails
|
||||
@@ -590,9 +587,11 @@ async def start():
|
||||
pass
|
||||
|
||||
# Screenshots
|
||||
screenshot_tups = []
|
||||
if len(args.screenshot) > 0:
|
||||
import time
|
||||
from aiomultiprocess import Pool
|
||||
from itertools import chain
|
||||
from theHarvester.screenshot.screenshot import ScreenShotter
|
||||
screen_shotter = ScreenShotter(args.screenshot)
|
||||
await screen_shotter.verify_installation()
|
||||
@@ -600,7 +599,6 @@ async def start():
|
||||
start = time.perf_counter()
|
||||
print('Filtering domains for ones we can reach')
|
||||
unique_resolved_domains = {url.split(':')[0]for url in full if ':' in url and 'www.' not in url}
|
||||
|
||||
if len(unique_resolved_domains) > 0:
|
||||
# First filter out ones that didn't resolve
|
||||
print('Attempting to visit unique resolved domains, this is ACTIVE RECON')
|
||||
@@ -614,12 +612,16 @@ async def start():
|
||||
chunk_number = 25
|
||||
for chunk in screen_shotter.chunk_list(unique_resolved_domains, chunk_number):
|
||||
try:
|
||||
await pool.map(screen_shotter.take_screenshot, chunk)
|
||||
screenshot_tups.extend(await pool.map(screen_shotter.take_screenshot, chunk))
|
||||
# screenshot_tups.append(list(chain(*await pool.map(screen_shotter.take_screenshot, chunk))))
|
||||
except Exception as ee:
|
||||
print(f'An exception has occurred while mapping: {ee}')
|
||||
end = time.perf_counter()
|
||||
print(f"Finished taking screenshots in {end - start} seconds")
|
||||
|
||||
import pprint as p
|
||||
p.pprint(screenshot_tups, indent=4)
|
||||
|
||||
# Shodan
|
||||
shodanres = []
|
||||
if shodan is True:
|
||||
@@ -675,6 +677,8 @@ async def start():
|
||||
HTMLcode = await generator.beginhtml()
|
||||
HTMLcode += await generator.generatedashboardcode(scanboarddata)
|
||||
HTMLcode += await generator.generatelatestscanresults(latestscanresults)
|
||||
if len(screenshot_tups) > 0:
|
||||
HTMLcode += await generator.generatescreenshots(screenshot_tups)
|
||||
HTMLcode += await generator.generatepreviousscanresults(previousscanresults)
|
||||
graph = reportgraph.GraphGenerator(word)
|
||||
await graph.init_db()
|
||||
@@ -698,7 +702,6 @@ async def start():
|
||||
print('[*] Reporting finished.')
|
||||
print('[*] Saving files.')
|
||||
|
||||
|
||||
try:
|
||||
# filename = filename.rsplit('.', 1)[0] + '.xml'
|
||||
# file = open(filename, 'w')
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
from typing import List
|
||||
import json
|
||||
|
||||
class HtmlGenerator:
|
||||
|
||||
def __init__(self, word):
|
||||
@@ -189,6 +192,13 @@ var table = new Tabulator("#example-table", {
|
||||
</head>
|
||||
<title>theHarvester Scan Report</title>
|
||||
<body>
|
||||
<style>
|
||||
.infoImage {
|
||||
height:500px;
|
||||
width:500px;
|
||||
cursor:pointer;
|
||||
}
|
||||
</style>
|
||||
<h1 style="text-align: center;"><span>theHarvester Scan Report</span></h1>
|
||||
'''
|
||||
return html
|
||||
@@ -262,3 +272,68 @@ var table = new Tabulator("#example-table", {
|
||||
return html
|
||||
except Exception as e:
|
||||
print(f'Error generating scan statistics HTML code: {e}')
|
||||
|
||||
@staticmethod
|
||||
async def generatescreenshots(tups: List):
|
||||
"""
|
||||
Append screenshot content
|
||||
:param tups: List of tuples, each tuple is comprised of 3 elements
|
||||
1. Domain
|
||||
2. Path to screenshot if it doesn't exist will be an empty string
|
||||
3. Html content for website
|
||||
:return: html content that displays screenshots
|
||||
"""
|
||||
try:
|
||||
html = '''
|
||||
<p> </p>
|
||||
<p> </p>
|
||||
|
||||
<h2 style="background-color:DodgerBlue; text-align:center">
|
||||
Screenshots
|
||||
</h2>
|
||||
|
||||
<div id="screenshot-table"></div>
|
||||
<script type="text/javascript">
|
||||
xxxxxxxxx
|
||||
var iconFormatter= function(value, data, cell, row, options){ //plain text value
|
||||
var test = `${value}`;
|
||||
return "<img class='infoImage' src=test>";
|
||||
//return "<img class='infoImage' src='" + value + "'>";
|
||||
};
|
||||
|
||||
//create Tabulator on DOM element with id "example-table"
|
||||
var table = new Tabulator("#screenshot-table", {
|
||||
height:650, // set height of table (in CSS or here), this enables the Virtual DOM and improves render speed dramatically (can be any valid css height value)
|
||||
data:tabledata, //assign data to table
|
||||
layout:"fitColumns", //fit columns to width of table (optional)
|
||||
columns:[ //Define Table Columns
|
||||
{title:"Date", field:"date", width:150},
|
||||
{title:"Domain", field:"domain", hozAlign:"left", headerFilter:"select" },
|
||||
{title:"Screenshot", field:"icon", align:"center", formatter:iconFormatter},
|
||||
]
|
||||
},
|
||||
);
|
||||
</script>
|
||||
<p> </p>
|
||||
<p> </p>
|
||||
|
||||
'''
|
||||
# var tabledata = [{date:"2020-07-03", domain:"box.netflix.com", plugin:"DNS-resolver", record:"ip",
|
||||
# result:"3.220.163.143"},{date:"2020-07-03", domain:"netflix.com", plugin:"DNS-resolver", record:"ip",
|
||||
# result:"34.209.106.197"},{date:"2020-07-03", domain:"netflix.com", plugin:"DNS-resolver", record:"ip",
|
||||
# result:"34.212.161.97"},{date:"2020-07-03", domain:"netflix.com", plugin:"DNS-resolver", record:"ip",
|
||||
# result:"34.223.232.157"}];
|
||||
|
||||
base = "var tabledata = [ "
|
||||
print(f'Iterating through: {len(tups)} tuples')
|
||||
for tup in tups:
|
||||
date = tup[0]
|
||||
domain = tup[1]
|
||||
path = tup[2]
|
||||
data = f'{{date:"{date}", domain:"{domain}", path:"{path}"}},'
|
||||
base += data
|
||||
base += '];'
|
||||
return html.replace('xxxxxxxxx', base)
|
||||
except Exception as e:
|
||||
print(f'Error generating screenshot section: {e}')
|
||||
return ""
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
"""
|
||||
Screenshot module that utilizes pyppeteer in async fashion
|
||||
to break urls into list and assign them to workers in a queue
|
||||
Screenshot module that utilizes pyppeteer to asynchronously
|
||||
take screenshots
|
||||
"""
|
||||
|
||||
from pyppeteer import launch
|
||||
import aiohttp
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
class ScreenShotter():
|
||||
class ScreenShotter:
|
||||
|
||||
def __init__(self, output):
|
||||
self.output = output
|
||||
@@ -43,30 +46,33 @@ class ScreenShotter():
|
||||
text = await resp.text("UTF-8")
|
||||
return f'http://{url}' if ('http' not in url and 'https' not in url) else url, text
|
||||
except Exception as e:
|
||||
print(f'An exception has occurred while attempting to screenshot {url}: {e}')
|
||||
print(f'An exception has occurred while attempting to visit {url} : {e}')
|
||||
return "", ""
|
||||
|
||||
async def take_screenshot(self, url):
|
||||
url = f'http://{url}' if ('http' not in url and 'https' not in url) else url
|
||||
# url = f'https://{url}' if ('http' not in url and 'https' not in url) else url
|
||||
url = url.replace('www.', '')
|
||||
print(f'Attempting to take a screenshot of: {url}')
|
||||
browser = await launch(headless=True, ignoreHTTPSErrors=True, args=["--no-sandbox"])
|
||||
context = await browser.createIncognitoBrowserContext()
|
||||
page = await browser.newPage()
|
||||
path = fr'{self.output}{self.slash}{url.replace("http://", "").replace("https://", "")}.png'
|
||||
print(f'path: {path}')
|
||||
date = str(datetime.utcnow())
|
||||
try:
|
||||
# change default timeout from 30 to 35 seconds
|
||||
page.setDefaultNavigationTimeout(35000)
|
||||
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
|
||||
'Chrome/83.0.4103.106 Safari/537.36')
|
||||
# await page.goto(url, waitUntil='networkidle0')
|
||||
await page.goto(url)
|
||||
await page.screenshot(
|
||||
{'path': f'{self.output}{self.slash}{url.replace("http://", "").replace("https://", "")}.png'})
|
||||
await page.screenshot({'path': path})
|
||||
except Exception as e:
|
||||
print(f'Exception occurred: {e} for: {url} ')
|
||||
print(f'An exception has occurred attempting to screenshot: {url} : {e}')
|
||||
path = ""
|
||||
finally:
|
||||
# Clean up everything whether screenshot is taken or not
|
||||
await asyncio.sleep(2)
|
||||
await page.close()
|
||||
await context.close()
|
||||
await browser.close()
|
||||
return date, url, path
|
||||
|
||||
Reference in New Issue
Block a user