diff --git a/requirements.txt b/requirements.txt index 8dd48ea2..9bff7a66 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ beautifulsoup4==4.8.0 censys==0.0.8 -chart-studio==1.0.0 flake8==3.7.8 grequests==0.4.0 plotly==4.1.0 diff --git a/setup.cfg b/setup.cfg index 7ce8946d..9b4ba780 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [flake8] -ignore = E501, F405, \ No newline at end of file +ignore = E501, F405, F403 \ No newline at end of file diff --git a/tests/discovery/test_githubcode.py b/tests/discovery/test_githubcode.py index 49ab3fa7..d4b00e5a 100644 --- a/tests/discovery/test_githubcode.py +++ b/tests/discovery/test_githubcode.py @@ -112,5 +112,4 @@ class TestSearchGithubCode: assert(None is test_class_instance.next_page_or_end(test_result)) if __name__ == '__main__': - pytest.main() - + pytest.main() \ No newline at end of file diff --git a/tests/test_myparser.py b/tests/test_myparser.py index 2494bbeb..72cd6243 100755 --- a/tests/test_myparser.py +++ b/tests/test_myparser.py @@ -16,5 +16,4 @@ class TestMyParser(object): if __name__ == '__main__': - pytest.main() - + pytest.main() \ No newline at end of file diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 8958bcba..7ef7daf3 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -912,7 +912,7 @@ def start(): latestscanchartdata = db.latestscanchartdata(word) scanhistorydomain = db.getscanhistorydomain(word) pluginscanstatistics = db.getpluginscanstatistics() - generator = statichtmlgenerator.htmlgenerator(word) + generator = statichtmlgenerator.HtmlGenerator(word) HTMLcode = generator.beginhtml() HTMLcode += generator.generatelatestscanresults(latestscanresults) HTMLcode += generator.generatepreviousscanresults(previousscanresults) diff --git a/theHarvester/discovery/censys.py b/theHarvester/discovery/censys.py index 697de756..9adeae3f 100644 --- a/theHarvester/discovery/censys.py +++ b/theHarvester/discovery/censys.py @@ -2,8 +2,8 @@ from theHarvester.lib.core import * from theHarvester.parsers import censysparser import requests - # TODO rewrite this module to use the censys api as the current way does notwork - # TODO And not really that maintainable as it currently stands +# TODO rewrite this module to use the censys api as the current way does notwork +# TODO And not really that maintainable as it currently stands class SearchCensys: diff --git a/theHarvester/lib/HtmlExport.py b/theHarvester/lib/HtmlExport.py index 57c19b79..d2c1bda4 100644 --- a/theHarvester/lib/HtmlExport.py +++ b/theHarvester/lib/HtmlExport.py @@ -80,7 +80,7 @@ class HtmlExport: self.style = a def writehtml(self): - page = markup.page() + page = markup.Page() page.html() self.styler() page.head(self.style) diff --git a/theHarvester/lib/graphs.py b/theHarvester/lib/graphs.py index 7d82767e..42005d8d 100644 --- a/theHarvester/lib/graphs.py +++ b/theHarvester/lib/graphs.py @@ -35,9 +35,9 @@ class BarGraph: """Creates horizontal and vertical bar graphs, progress bars, and faders.""" def __init__(self, type=''): - #------------------------------------------------------------------------- + # ------------------------------------------------------------------------- # Configuration - #------------------------------------------------------------------------- + # ------------------------------------------------------------------------- # graph type: "hBar", "vBar", "pBar", or "fader" self.type = type and type or 'hBar' self.values = [] # graph data: list diff --git a/theHarvester/lib/hostchecker.py b/theHarvester/lib/hostchecker.py index 9d3119b0..cb0bca88 100644 --- a/theHarvester/lib/hostchecker.py +++ b/theHarvester/lib/hostchecker.py @@ -7,7 +7,7 @@ Created by laramies on 2008-08-21. import socket -class Checker(): +class Checker: def __init__(self, hosts): self.hosts = hosts diff --git a/theHarvester/lib/markup.py b/theHarvester/lib/markup.py index 0891d7b2..6f6f6cde 100644 --- a/theHarvester/lib/markup.py +++ b/theHarvester/lib/markup.py @@ -10,7 +10,7 @@ in an intuitive, lightweight, customizable and pythonic way. The code is in the public domain. -Version: %s as of %s. +Version: {0} as of {1}. Documentation and further info is at http://markup.sourceforge.net/ @@ -18,10 +18,10 @@ Please send bug reports, feature requests, enhancement ideas or questions to nogradi at gmail dot com. Installation: drop markup.py somewhere into your Python path. -""" % ( __version__, __date__ ) +""".format(__version__, __date__) -class element: +class Element: """This class handles the addition of a new element.""" @@ -114,7 +114,7 @@ class element: raise DeprecationError(self.tag) -class page: +class Page: """This is our main class representing a document. Elements are added as attributes of an instance of this class.""" @@ -203,8 +203,8 @@ class page: elif (onetags and not twotags) or (twotags and not onetags): raise CustomizationError() else: - self.onetags = russell() - self.twotags = russell() + self.onetags = Russell() + self.twotags = Russell() self.mode = mode else: raise ModeError(mode) @@ -212,7 +212,7 @@ class page: def __getattr__(self, attr): if attr.startswith("__") and attr.endswith("__"): raise AttributeError(attr) - return element(attr, case=self.case, parent=self) + return Element(attr, case=self.case, parent=self) def __str__(self): @@ -381,7 +381,7 @@ class page: "Script should be given a dictionary of src:type pairs.") -class _oneliner: +class _OneLiner: """An instance of oneliner returns a string corresponding to one element. This class can be used to write 'oneliners' that return a string @@ -393,11 +393,11 @@ class _oneliner: def __getattr__(self, attr): if attr.startswith("__") and attr.endswith("__"): raise AttributeError(attr) - return element(attr, case=self.case, parent=None) + return Element(attr, case=self.case, parent=None) -oneliner = _oneliner(case='lower') -upper_oneliner = _oneliner(case='upper') +oneliner = _OneLiner(case='lower') +upper_oneliner = _OneLiner(case='upper') def _argsdicts(args, mydict): @@ -484,18 +484,19 @@ def unescape(text): return text -class dummy: +class Dummy: """A dummy class for attaching attributes.""" pass -doctype = dummy() + +doctype = Dummy() doctype.frameset = "" doctype.strict = "" doctype.loose = "" -class russell: +class Russell: """A dummy class that contains anything.""" @@ -514,25 +515,25 @@ class MarkupError(Exception): class ClosingError(MarkupError): def __init__(self, tag): - self.message = "The element '%s' does not accept non-keyword arguments (has no closing tag)." % tag + self.message = "The element '{}' does not accept non-keyword arguments (has no closing tag)".format(tag) class OpeningError(MarkupError): def __init__(self, tag): - self.message = "The element '%s' can not be opened." % tag + self.message = "The element '{}' can not be opened.".format(tag) class ArgumentError(MarkupError): def __init__(self, tag): - self.message = "The element '%s' was called with more than one non-keyword argument." % tag + self.message = "The element '{}' was called with more than one non-keyword argument.".format(tag) class InvalidElementError(MarkupError): def __init__(self, tag, mode): - self.message = "The element '%s' is not valid for your mode '%s'." % ( + self.message = "The element '{0}' is not valid for your mode '{1}'.".format( tag, mode) @@ -540,13 +541,13 @@ class InvalidElementError(MarkupError): class DeprecationError(MarkupError): def __init__(self, tag): - self.message = "The element '%s' is deprecated, instantiate markup.page with mode='loose_html' to allow it." % tag + self.message = "The element '{0}' is deprecated, instantiate markup.page with mode='loose_html' to allow it.".format(tag) class ModeError(MarkupError): def __init__(self, mode): - self.message = "Mode '%s' is invalid, possible values: strict_html, loose_html, xml." % mode + self.message = "Mode '{}' is invalid, possible values: strict_html, loose_html, xml.".format(mode) class CustomizationError(MarkupError): @@ -556,4 +557,4 @@ class CustomizationError(MarkupError): if __name__ == '__main__': - print(__doc__) + print(__doc__) \ No newline at end of file diff --git a/theHarvester/lib/reportgraph.py b/theHarvester/lib/reportgraph.py index 7ee2e443..7a5aa121 100644 --- a/theHarvester/lib/reportgraph.py +++ b/theHarvester/lib/reportgraph.py @@ -2,7 +2,6 @@ from theHarvester.lib import stash from datetime import datetime import plotly import plotly.graph_objs as go -import chart_studio.plotly as py try: db = stash.stash_manager() @@ -32,11 +31,11 @@ except Exception: self.bardata.append(latestscandata['shodan']) self.bardata.append(latestscandata['vhost']) layout = dict(title='Latest scan - number of targets identified for ' + domain, - xaxis=dict(title='Targets'), - yaxis=dict(title='Hits'),) + xaxis=dict(title='Targets'), + yaxis=dict(title='Hits'),) barchartcode = plotly.offline.plot({ - 'data': [go.Bar(x=self.barcolumns, y=self.bardata)], - 'layout': layout, + 'data': [go.Bar(x=self.barcolumns, y=self.bardata)], + 'layout': layout, }, auto_open=False, include_plotlyjs=False, filename='report.html', output_type='div') return barchartcode except Exception as e: @@ -54,40 +53,40 @@ except Exception: self.scattercountvhosts.append(int(i['vhost'])) trace0 = go.Scatter( - x=self.scatterxdata, - y=self.scattercounthosts, - mode='lines+markers', - name='hosts') + x=self.scatterxdata, + y=self.scattercounthosts, + mode='lines+markers', + name='hosts') trace1 = go.Scatter( - x=self.scatterxdata, - y=self.scattercountips, - mode='lines+markers', - name='IP address') + x=self.scatterxdata, + y=self.scattercountips, + mode='lines+markers', + name='IP address') trace2 = go.Scatter( - x=self.scatterxdata, - y=self.scattercountvhosts, - mode='lines+markers', - name='vhost') + x=self.scatterxdata, + y=self.scattercountvhosts, + mode='lines+markers', + name='vhost') trace3 = go.Scatter( - x=self.scatterxdata, - y=self.scattercountshodans, - mode='lines+markers', - name='shodan') + x=self.scatterxdata, + y=self.scattercountshodans, + mode='lines+markers', + name='shodan') trace4 = go.Scatter( - x=self.scatterxdata, - y=self.scattercountemails, - mode='lines+markers', - name='email') + x=self.scatterxdata, + y=self.scattercountemails, + mode='lines+markers', + name='email') data = [trace0, trace1, trace2, trace3, trace4] layout = dict(title='Scanning history for ' + domain, xaxis=dict(title='Date'), yaxis=dict(title='Results')) scatterchartcode = plotly.offline.plot({ - 'data': data, - 'layout': layout}, auto_open=False, include_plotlyjs=False, filename='report.html', output_type='div') + 'data': data, + 'layout': layout}, auto_open=False, include_plotlyjs=False, filename='report.html', output_type='div') return scatterchartcode except Exception as e: print(f'Error generating HTML for the historical graph for domain: {e}') diff --git a/theHarvester/lib/stash.py b/theHarvester/lib/stash.py index 2bf751c4..02115e45 100644 --- a/theHarvester/lib/stash.py +++ b/theHarvester/lib/stash.py @@ -10,7 +10,6 @@ class stash_manager: self.totalresults = "" self.latestscandomain = {} self.domainscanhistory = [] - #self.scanboarddata = [] self.scanboarddata = {} self.scanstats = [] self.latestscanresults = [] diff --git a/theHarvester/lib/statichtmlgenerator.py b/theHarvester/lib/statichtmlgenerator.py index 23c9fcb7..7c64a12a 100644 --- a/theHarvester/lib/statichtmlgenerator.py +++ b/theHarvester/lib/statichtmlgenerator.py @@ -1,12 +1,12 @@ -class htmlgenerator: +class HtmlGenerator: def __init__(self, word): self.domain = word def generatepreviousscanresults(self, previousscanresults): try: - if previousscanresults[0]=='No results': - html=''' + if previousscanresults[0] == 'No results': + html = '''

Previous scan report

 

@@ -94,7 +94,7 @@ class htmlgenerator: ''' return html except Exception as e: - print('Error generating the latest scan results HTML code: ' + str(e)) + print(f'Error generating the latest scan results HTML code: {e}') def beginhtml(self): html = ''' @@ -142,7 +142,7 @@ class htmlgenerator: ''' return html except Exception as e: - print('Error generating dashboard HTML code: ' + str(e)) + print(f'Error generating dashboard HTML code: {e}') def generatepluginscanstatistics(self, scanstatistics): try: @@ -175,4 +175,4 @@ class htmlgenerator: ''' return html except Exception as e: - print('Error generating scan statistics HTML code: ' + str(e)) + print(f'Error generating scan statistics HTML code: {e}') diff --git a/theHarvester/parsers/cymonparser.py b/theHarvester/parsers/cymonparser.py deleted file mode 100644 index f424e97b..00000000 --- a/theHarvester/parsers/cymonparser.py +++ /dev/null @@ -1,19 +0,0 @@ -import re -from bs4 import BeautifulSoup - - -class Parser: - - def __init__(self, results): - self.results = results - self.ipaddresses = [] - self.soup = BeautifulSoup(results.results, features='html.parser') - - def search_ipaddresses(self): - try: - tags = self.soup.findAll('td') - allip = re.findall(r'[0-9]+(?:\.[0-9]+){3}', str(tags)) - self.ipaddresses = set(allip) - return self.ipaddresses - except Exception as e: - print('Error occurred: ' + str(e)) diff --git a/theHarvester/parsers/myparser.py b/theHarvester/parsers/myparser.py index 0f212730..a293b344 100644 --- a/theHarvester/parsers/myparser.py +++ b/theHarvester/parsers/myparser.py @@ -9,9 +9,9 @@ class Parser: self.temp = [] def genericClean(self): - self.results = self.results.replace('', '').replace('', '').replace('', '').replace('', - '').replace('%2f', '').replace('%3a', '').replace('', '').replace('','')\ - .replace('','').replace('','') + self.results = self.results.replace('', '').replace('', '').replace('', '').replace('', '')\ + .replace('%2f', '').replace('%3a', '').replace('', '').replace('', '')\ + .replace('', '').replace('', '') for e in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '/', '\\'): self.results = self.results.replace(e, ' ')