From eabbb30c4d90a07d504cbede348a2c86e22ddb4e Mon Sep 17 00:00:00 2001 From: Janos Zold Date: Wed, 19 Dec 2018 09:08:42 +0000 Subject: [PATCH] New reporting module with new format and contents --- lib/statichtmlgenerator.py | 63 ++++++++++++++++++++++++----- stash.py | 82 ++++++++++++++++++++++++++++++++++++-- theHarvester.py | 22 +++++----- 3 files changed, 143 insertions(+), 24 deletions(-) diff --git a/lib/statichtmlgenerator.py b/lib/statichtmlgenerator.py index d43f8dbe..5b8c510e 100644 --- a/lib/statichtmlgenerator.py +++ b/lib/statichtmlgenerator.py @@ -2,6 +2,50 @@ class htmlgenerator: def __init__(self,word): self.domain = word + def generatelatestscanresults(self, latestscanresults): + try: + html=''' +

theHarvester Scan Report

+

Latest scan report

+

 

+ + + + + + + + + + +''' + for i in latestscanresults: + html += '" + html += '" + html += '" + html += '" + html += '" + html +='' + html +=''' + +
DateDomainPluginRecord typeResult
' + str(i[0]) + "' + str(i[1]) + "' + str(i[2]) + "' + str(i[3]) + "' + str(i[4]) + "
+

 

+

 

+

 

+

 

+''' + return html + except Exception as e: + print("Error generating the latest scan results HTML code: " + str(e)) + + def beginhtml(self): + html =''' + + + + ''' + return html + def generatedashboardcode(self, scanboarddata): try: totalnumberofdomains = scanboarddata['domains'] @@ -11,12 +55,8 @@ class htmlgenerator: totalnumberofemail= scanboarddata['email'] totalnumberofshodan= scanboarddata['shodan'] html=''' - - - -

theHarvester Scan Report

-

TheHarvester scanning dashboard

- +

Scan dashboard

+
@@ -40,15 +80,17 @@ class htmlgenerator:

 

 

 

+

 

+

 

''' return html except Exception as e: print("Error generating dashboard HTML code: " + str(e)) - def generatescanstatistics(self, scanhistorystatistics): + def generatepluginscanstatistics(self, scanstatistics): try: html=''' -

theHarvester scan statistics

+

theHarvester plugin statistics

 

Domains
@@ -61,7 +103,7 @@ class htmlgenerator: ''' - for i in scanhistorystatistics: + for i in scanstatistics: html += '" html += '" html += '" @@ -74,12 +116,11 @@ class htmlgenerator:

 

 

''' - print("END") return html except Exception as e: print("Error generating scan statistics HTML code: " + str(e)) - def generatescandetailsdomain(self, word, latestscandomain): + def generatescandetailsdomain(self, word, latestscandomain): #This is the old scan details try: emails = latestscandomain['scandetailsemail'] hosts = latestscandomain['scandetailshost'] diff --git a/stash.py b/stash.py index eda27a6a..2209b520 100644 --- a/stash.py +++ b/stash.py @@ -1,7 +1,6 @@ import sqlite3 import datetime - class stash_manager: def __init__(self): @@ -12,6 +11,7 @@ class stash_manager: self.domainscanhistory = [] self.scanboarddata = {} self.scanstats = [] + self.latestscanresults = [] def do_init(self): conn = sqlite3.connect(self.db) @@ -54,7 +54,7 @@ class stash_manager: print(e) return - def getlatestscandomain(self,domain): + def generatedashboardcode(self,domain): try: self.latestscandomain["domain"] = domain conn = sqlite3.connect(self.db) @@ -99,6 +99,38 @@ class stash_manager: finally: conn.close() + def getlatestscanresults(self,domain): + #CONTINUE WITH NEW FUNCTION FOR THE PREVIOUS DAY SQL + ''' + SELECT find_date + FROM results + WHERE find_date = ( SELECT MAX(find_date) + FROM results + WHERE find_date < ( SELECT MAX(find_date) + FROM results + ) + ) + ''' + try: + conn = sqlite3.connect(self.db) + c = conn.cursor() + c.execute('''SELECT MAX(find_date) FROM results WHERE domain=?''',(domain,)) + latestscandate = c.fetchone() + c = conn.cursor() + c.execute(''' + SELECT find_date, domain, source,type,resource + FROM results + WHERE find_date =? and domain=? + ORDER BY source,type + ''',(latestscandate[0],domain,)) + results = c.fetchall() + self.latestscanresults = results + return self.latestscanresults + except Exception as e: + print("Error in getting the latest scan results from the database: " + str(e)) + finally: + conn.close() + def getscanboarddata(self): try: conn = sqlite3.connect(self.db) @@ -164,7 +196,7 @@ class stash_manager: finally: conn.close() - def getscanstatistics(self): + def getpluginscanstatistics(self): try: conn = sqlite3.connect(self.db) c = conn.cursor() @@ -181,5 +213,49 @@ class stash_manager: finally: conn.close() + def latestscanchartdata(self,domain): + try: + self.latestscandomain["domain"] = domain + conn = sqlite3.connect(self.db) + c = conn.cursor() + c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="host"''',(domain,)) + data = c.fetchone() + self.latestscandomain["host"] = data[0] + c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="email"''',(domain,)) + data = c.fetchone() + self.latestscandomain["email"] = data[0] + c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="ip"''',(domain,)) + data = c.fetchone() + self.latestscandomain["ip"] = data[0] + c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="vhost"''',(domain,)) + data = c.fetchone() + self.latestscandomain["vhost"] = data[0] + c.execute('''SELECT COUNT(*) from results WHERE domain=? AND type="shodan"''',(domain,)) + data = c.fetchone() + self.latestscandomain["shodan"] = data[0] + c.execute('''SELECT MAX(find_date) FROM results WHERE domain=?''',(domain,)) + data = c.fetchone() + self.latestscandomain["latestdate"] = data[0] + latestdate = data [0] + c.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="host"''',(domain,latestdate,)) + scandetailshost = c.fetchall() + self.latestscandomain["scandetailshost"] = scandetailshost + c.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="email"''',(domain,latestdate,)) + scandetailsemail = c.fetchall() + self.latestscandomain["scandetailsemail"] = scandetailsemail + c.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="ip"''',(domain,latestdate,)) + scandetailsip = c.fetchall() + self.latestscandomain["scandetailsip"] = scandetailsip + c.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="vhost"''',(domain,latestdate,)) + scandetailsvhost = c.fetchall() + self.latestscandomain["scandetailsvhost"] = scandetailsvhost + c.execute('''SELECT * FROM results WHERE domain=? AND find_date=? AND type="shodan"''',(domain,latestdate,)) + scandetailsshodan = c.fetchall() + self.latestscandomain["scandetailsshodan"] = scandetailsshodan + return self.latestscandomain + except Exception as e: + print(e) + finally: + conn.close() \ No newline at end of file diff --git a/theHarvester.py b/theHarvester.py index e4132a65..21df0c69 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -146,8 +146,8 @@ def start(argv): search = googlesearch.search_google(word, limit, start) search.process(google_dorking) emails = search.get_emails() - all_emails.extend(emails) hosts = search.get_hostnames() + all_emails.extend(emails) all_hosts.extend(hosts) db=stash.stash_manager() db.store_all(word,all_hosts,'host','google') @@ -691,24 +691,26 @@ def start(argv): print("NEW REPORTING BEGINS:") db = stash.stash_manager() scanboarddata = db.getscanboarddata() - latestscandomain = db.getlatestscandomain(word) + latestscanresults = db.getlatestscanresults(word) + latestscanchartdata = db.latestscanchartdata(word) scanhistorydomain = db.getscanhistorydomain(word) - scanstatistics = db.getscanstatistics() + pluginscanstatistics = db.getpluginscanstatistics() from lib import statichtmlgenerator generator = statichtmlgenerator.htmlgenerator(word) - HTMLcode = generator.generatedashboardcode(scanboarddata) - HTMLcode += generator.generatescandetailsdomain(word, latestscandomain) + HTMLcode = generator.beginhtml() + HTMLcode += generator.generatelatestscanresults(pluginscanstatistics) from lib import reportgraph import datetime graph = reportgraph.graphgenerator(word) - HTMLcode += graph.drawlatestscangraph(word, latestscandomain) + HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata) HTMLcode += graph.drawscattergraphscanhistory(word, scanhistorydomain) - HTMLcode += generator.generatescanstatistics(scanstatistics) + HTMLcode += generator.generatepluginscanstatistics(pluginscanstatistics) + HTMLcode += generator.generatedashboardcode(scanboarddata) HTMLcode += '

Report generated on '+ str(datetime.datetime.now())+'

' HTMLcode +=''' - - - ''' + + +''' Html_file= open("report.html","w") Html_file.write(HTMLcode) Html_file.close()
' + str(i[0]) + "' + str(i[1]) + "' + str(i[2]) + "