From 930a60345f28d40adadf491879b9db5bdbebf31d Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Mon, 21 Oct 2019 01:02:10 +0100 Subject: [PATCH] Bug fixes for file locations and more static linting, pep8 fixes and ci updates --- .github/workflows/theHarvester.yml | 2 +- .gitignore | 5 ++++- setup.py | 2 +- theHarvester/__main__.py | 30 ++++++++++++------------- theHarvester/discovery/dnssearch.py | 15 ++++++------- theHarvester/lib/core.py | 2 +- theHarvester/lib/reportgraph.py | 2 +- theHarvester/lib/stash.py | 8 +++---- theHarvester/lib/statichtmlgenerator.py | 2 +- 9 files changed, 35 insertions(+), 33 deletions(-) mode change 100644 => 100755 setup.py diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index 44300381..91f7c260 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -114,4 +114,4 @@ jobs: pytest - name: Static type checking with mypy run: | - mypy --pretty theHarvester/discovery/*.py \ No newline at end of file + mypy --pretty theHarvester/*/*.py \ No newline at end of file diff --git a/.gitignore b/.gitignore index ffe2562b..3f36fa1e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,7 @@ debug_results.txt venv .mypy_cache -.pytest_cache \ No newline at end of file +.pytest_cache +build/ +dist/ +theHarvester.egg-info/ \ No newline at end of file diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 3bd24953..b81b2b8a --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ import setuptools from theHarvester.lib.core import Core -with open("README.md", "r") as fh: +with open('README.md', 'r') as fh: long_description = fh.read() setuptools.setup( diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 249fccab..2c3cc5b4 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -42,32 +42,32 @@ def start(): args = parser.parse_args() try: - db = stash.stash_manager() + db = stash.StashManager() db.do_init() except Exception: pass - all_emails = [] - all_hosts = [] - all_ip = [] + all_emails: list = [] + all_hosts: list = [] + all_ip: list = [] dnsbrute = args.dns_brute dnslookup = args.dns_lookup dnsserver = args.dns_server dnstld = args.dns_tld engines = [] - filename = args.filename # type: str - full = [] + filename: str = args.filename + full: list = [] google_dorking = args.google_dork - host_ip = [] - limit = args.limit # type: int + host_ip: list = [] + limit: int = args.limit ports_scanning = args.port_scan shodan = args.shodan - start = args.start # type: int + start: int = args.start takeover_check = False - all_urls = [] - vhost = [] + all_urls: list = [] + vhost: list = [] virtual = args.virtual_host - word = args.domain # type: str + word: str = args.domain def store(search_engine: Any, source: str, process_param: Any = None, store_host: bool = False, store_emails: bool = False, store_ip: bool = False, store_people: bool = False, @@ -89,7 +89,7 @@ def start(): :param store_results: whether to fetch details from get_results() and persist """ search_engine.process() if process_param is None else search_engine.process(process_param) - db_stash = stash.stash_manager() + db_stash = stash.StashManager() if store_host: host_names = filter(search_engine.get_hostnames()) @@ -402,7 +402,7 @@ def start(): all_hosts = sorted(list(set(all_hosts))) full_host = hostchecker.Checker(all_hosts) full, ips = asyncio.run(full_host.check()) - db = stash.stash_manager() + db = stash.StashManager() for host in full: host = str(host) print(host) @@ -563,7 +563,7 @@ def start(): if filename != "": try: print('\n[*] Reporting started.') - db = stash.stash_manager() + db = stash.StashManager() scanboarddata = db.getscanboarddata() latestscanresults = db.getlatestscanresults(word) previousscanresults = db.getlatestscanresults(word, previousday=True) diff --git a/theHarvester/discovery/dnssearch.py b/theHarvester/discovery/dnssearch.py index ef5c0caa..a1759022 100644 --- a/theHarvester/discovery/dnssearch.py +++ b/theHarvester/discovery/dnssearch.py @@ -8,21 +8,20 @@ class DnsForce: def __init__(self, domain, dnsserver, verbose=False): self.domain = domain - self.file = 'wordlists/dns-names.txt' self.subdo = False self.verbose = verbose try: - f = open(self.file, 'r') - except Exception: - print('Error opening DNS dictionary file.') - sys.exit() - self.list = f.readlines() + with open('wordlists/dns-names.txt', 'r') as file: + self.list = file.readlines() + except FileNotFoundError: + with open('/etc/theHarvester/dns-names.txt', 'r') as file: + self.list = file.readlines() def run(self, host): hostname = str(host.split('\n')[0]) + '.' + str(self.domain) if self.verbose: - ESC = chr(27) - sys.stdout.write(ESC + '[2K' + ESC + '[G') + esc = chr(27) + sys.stdout.write(esc + '[2K' + esc + '[G') sys.stdout.write('\r' + hostname + ' - ') sys.stdout.flush() try: diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index b35aeedd..17526fe5 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -8,7 +8,7 @@ import yaml class Core: @staticmethod def version() -> str: - return '3.1.1-dev3' + return '3.1.1dev3' @staticmethod def bing_key() -> str: diff --git a/theHarvester/lib/reportgraph.py b/theHarvester/lib/reportgraph.py index 60a2f03e..b50f5862 100644 --- a/theHarvester/lib/reportgraph.py +++ b/theHarvester/lib/reportgraph.py @@ -4,7 +4,7 @@ import plotly import plotly.graph_objs as go try: - db = stash.stash_manager() + db = stash.StashManager() db.do_init() except Exception as error: print(f'{error}') diff --git a/theHarvester/lib/stash.py b/theHarvester/lib/stash.py index 02115e45..db4439a0 100644 --- a/theHarvester/lib/stash.py +++ b/theHarvester/lib/stash.py @@ -2,7 +2,7 @@ import datetime import sqlite3 -class stash_manager: +class StashManager: def __init__(self): self.db = "stash.sqlite" @@ -131,7 +131,7 @@ class stash_manager: self.previousscanresults = results return self.previousscanresults except Exception as e: - print('Error in getting the previous scan results from the database: ' + str(e)) + print(f'Error in getting the previous scan results from the database: {e}') else: try: c = conn.cursor() @@ -148,9 +148,9 @@ class stash_manager: self.latestscanresults = results return self.latestscanresults except Exception as e: - print('Error in getting the latest scan results from the database: ' + str(e)) + print(f'Error in getting the latest scan results from the database: {e}') except Exception as e: - print('Error connecting to theHarvester database: ' + str(e)) + print(f'Error connecting to theHarvester database: {e}') finally: conn.close() diff --git a/theHarvester/lib/statichtmlgenerator.py b/theHarvester/lib/statichtmlgenerator.py index 39dfd1a4..dfe01a76 100644 --- a/theHarvester/lib/statichtmlgenerator.py +++ b/theHarvester/lib/statichtmlgenerator.py @@ -59,7 +59,7 @@ class HtmlGenerator: ''' return html except Exception as e: - print('Error generating the previous scan results HTML code: ' + str(e)) + print(f'Error generating the previous scan results HTML code: {e}') def generatelatestscanresults(self, latestscanresults): try: