From 8176e948026e6a5d83fd3bd5942083ecdd2f8684 Mon Sep 17 00:00:00 2001 From: Lee Baird Date: Sun, 13 Jan 2019 17:58:38 -0600 Subject: [PATCH 1/5] Removed dbl quotes. --- discovery/censys.py | 26 +++---- discovery/constants.py | 6 +- discovery/crtsh.py | 6 +- discovery/cymon.py | 6 +- discovery/dnssearch.py | 108 +++++++++++++++--------------- discovery/dogpilesearch.py | 8 +-- discovery/duckduckgosearch.py | 10 +-- discovery/exaleadsearch.py | 20 +++--- discovery/googleCSE.py | 32 ++++----- discovery/googlecertificates.py | 8 +-- discovery/googlesearch.py | 46 ++++++------- discovery/googlesets.py | 10 +-- discovery/huntersearch.py | 4 +- discovery/linkedinsearch.py | 8 +-- discovery/netcraft.py | 10 +-- discovery/pgpsearch.py | 10 +-- discovery/s3_scanner.py | 16 ++--- discovery/securitytrailssearch.py | 4 +- discovery/shodansearch.py | 6 +- discovery/takeover.py | 28 ++++---- discovery/threatcrowd.py | 10 +-- discovery/trello.py | 14 ++-- discovery/twittersearch.py | 8 +-- discovery/virustotal.py | 10 +-- discovery/wfuzz_search.py | 2 +- discovery/yahoosearch.py | 9 ++- discovery/yandexsearch.py | 12 ++-- theHarvester.py | 16 ++--- 28 files changed, 226 insertions(+), 227 deletions(-) diff --git a/discovery/censys.py b/discovery/censys.py index 07fb91df..c4b0f6c9 100644 --- a/discovery/censys.py +++ b/discovery/censys.py @@ -14,7 +14,7 @@ class SearchCensys: self.resultcerts = "" self.total_resultshosts = "" self.total_resultscerts = "" - self.server = "censys.io" + self.server = 'censys.io' self.ips = [] self.hostnamesall = [] self.limit = limit @@ -39,8 +39,8 @@ class SearchCensys: def process(self): try: - self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=1" - self.urlcert = "https://" + self.server + "/certificates/_search?q=" + str(self.word) + "&page=1" + self.urlhost = 'https://' + self.server + '/ipv4/_search?q=' + str(self.word) + '&page=1' + self.urlcert = 'https://' + self.server + '/certificates/_search?q=' + str(self.word) + '&page=1' self.do_searchhosturl() self.do_searchcertificateurl() counter = 2 @@ -51,9 +51,9 @@ class SearchCensys: while counter <= totalpages: try: self.page = str(counter) - self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str( + self.urlhost = 'https://' + self.server + '/ipv4/_search?q=' + str(self.word) + '&page=' + str( self.page) - print("\tSearching Censys IP results page " + self.page + ".") + print('\tSearching Censys IP results page ' + self.page + '.') self.do_searchhosturl() counter += 1 except Exception as e: @@ -62,9 +62,9 @@ class SearchCensys: while counter <= pagestosearch: try: self.page = str(counter) - self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str( + self.urlhost = 'https://' + self.server + '/ipv4/_search?q=' + str(self.word) + '&page=' + str( self.page) - print(f'\tSearching Censys IP results page {self.page} ...') + print(f'\tSearching Censys IP results page {self.page}.') self.do_searchhosturl() counter += 1 except Exception as e: @@ -75,9 +75,9 @@ class SearchCensys: while counter <= totalpages: try: self.page = str(counter) - self.urlhost = "https://" + self.server + "/certificates/_search?q=" + str( - self.word) + "&page=" + str(self.page) - print(f'\tSearching Censys certificates results page {self.page} ...') + self.urlhost = 'https://' + self.server + '/certificates/_search?q=' + str( + self.word) + '&page=' + str(self.page) + print(f'\tSearching Censys certificates results page {self.page}.') self.do_searchcertificateurl() counter += 1 except Exception as e: @@ -86,9 +86,9 @@ class SearchCensys: while counter <= pagestosearch: try: self.page = str(counter) - self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str( + self.urlhost = 'https://' + self.server + '/ipv4/_search?q=' + str(self.word) + '&page=' + str( self.page) - print("\tSearching Censys IP results page " + self.page + ".") + print('\tSearching Censys IP results page ' + self.page + '.') self.do_searchhosturl() counter += 1 except Exception as e: @@ -101,7 +101,7 @@ class SearchCensys: try: ips = self.get_ipaddresses() headers = {'user-agent': Core.get_user_agent(), 'Accept': '*/*', 'Referer': self.urlcert} - response = requests.post("https://censys.io/ipv4/getdns", json={"ips": ips}, headers=headers) + response = requests.post('https://censys.io/ipv4/getdns', json={'ips': ips}, headers=headers) responsejson = response.json() domainsfromcensys = [] for key, jdata in responsejson.items(): diff --git a/discovery/constants.py b/discovery/constants.py index d59affdb..ca1b7230 100644 --- a/discovery/constants.py +++ b/discovery/constants.py @@ -8,7 +8,7 @@ User-Agents from: https://github.com/tamimibrahim17/List-of-user-agents import random -googleUA = "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36" +googleUA = 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36' bingAPI_key = '' @@ -61,9 +61,9 @@ class MissingKey(Exception): def __init__(self, identity_flag): if identity_flag: - self.message = '\n\033[93m[!] Missing API key.\n \033[0m' + self.message = '\n\033[93m[!] Missing API key. \033[0m' else: - self.message = '\n\033[93m[!] Missing CSE id.\n \033[0m' + self.message = '\n\033[93m[!] Missing CSE id. \033[0m' def __str__(self): return self.message diff --git a/discovery/crtsh.py b/discovery/crtsh.py index 9e331280..faa8fd02 100644 --- a/discovery/crtsh.py +++ b/discovery/crtsh.py @@ -11,8 +11,8 @@ class search_crtsh: self.word = word.replace(' ', '%20') self.results = "" self.totalresults = "" - self.server = "https://crt.sh/?q=" - self.quantity = "100" + self.server = 'https://crt.sh/?q=' + self.quantity = '100' self.counter = 0 def do_search(self): @@ -64,4 +64,4 @@ class search_crtsh: def process(self): self.do_search() - print("\tSearching CRT.sh results.") + print('\tSearching results.') diff --git a/discovery/cymon.py b/discovery/cymon.py index a6e3723d..09e4638c 100644 --- a/discovery/cymon.py +++ b/discovery/cymon.py @@ -11,7 +11,7 @@ class search_cymon: self.word = word self.url = "" self.results = "" - self.server = "cymon.io" + self.server = 'cymon.io' def do_search(self): try: @@ -24,8 +24,8 @@ class search_cymon: def process(self): try: - self.url = "https://" + self.server + "/domain/" + str(self.word) - print('\tSearching Cymon results.') + self.url = 'https://' + self.server + '/domain/' + str(self.word) + print('\tSearching results.') self.do_search() except Exception as e: print(f'Error occurred: {e}') diff --git a/discovery/dnssearch.py b/discovery/dnssearch.py index c384c1ba..17f5f99e 100644 --- a/discovery/dnssearch.py +++ b/discovery/dnssearch.py @@ -12,10 +12,10 @@ class dns_reverse(): self.results = [] self.verbose = verbose try: - DNS.ParseResolvConf("/etc/resolv.conf") + DNS.ParseResolvConf('/etc/resolv.conf') nameserver = DNS.defaults['server'][0] except: - print("Error in DNS resolvers") + print('Error in DNS resolvers') sys.exit() def run(self, host): @@ -27,20 +27,20 @@ class dns_reverse(): if self.verbose: ESC = chr(27) sys.stdout.write(ESC + '[2K' + ESC + '[G') - sys.stdout.write("\r\t" + host) + sys.stdout.write('\r\t' + host) sys.stdout.flush() try: name = DNS.Base.DnsRequest(b, qtype='ptr').req().answers[0]['data'] - return host + ":" + name + return host + ':' + name except: pass def get_ip_list(self, ips): - """Generates the list of ips to reverse""" + """Generates the list of IPs to reverse""" try: list = IPy.IP(ips) except: - print("Error in IP format, check the input and try again. (Eg. 192.168.1.0/24)") + print('Error in IP format, check the input and try again. (Eg. 192.168.1.0/24)') sys.exit() name = [] for x in list: @@ -64,7 +64,7 @@ class dns_force(): def __init__(self, domain, dnsserver, verbose=False): self.domain = domain self.nameserver = dnsserver - self.file = "wordlists/dns-big.txt" + self.file = 'wordlists/dns-big.txt' self.subdo = False self.verbose = verbose try: @@ -73,21 +73,21 @@ class dns_force(): with open(res_path) as f: self.resolvers = f.read().splitlines() except Exception: - print("Resolvers file can't be open") + print("Resolvers file can't be open.") try: - f = open(self.file, "r") + f = open(self.file, 'r') except: - print("Error opening dns dictionary file") + print('Error opening DNS dictionary file.') sys.exit() self.list = f.readlines() def getdns(self, domain): - DNS.ParseResolvConf("/etc/resolv.conf") + DNS.ParseResolvConf('/etc/resolv.conf') dom = domain if self.subdo is True: - dom = domain.split(".") + dom = domain.split('.') dom.pop(0) - rootdom = ".".join(dom) + rootdom = '.'.join(dom) else: rootdom = dom if self.nameserver == "": @@ -107,26 +107,26 @@ class dns_force(): # Check if variable is defined. test except NameError: - print("Error, test is not defined") + print('Error, test is not defined.') sys.exit() - if test.header['status'] != "NOERROR": - print("Error") + if test.header['status'] != 'NOERROR': + print('[!] Error') sys.exit() self.nameserver = test.answers[0]['data'] - elif self.nameserver == "local": + elif self.nameserver == 'local': self.nameserver = nameserver return self.nameserver def run(self, host): if self.nameserver == "": self.nameserver = self.getdns(self.domain) - print("\n\033[94m[-] Using DNS server: " + self.nameserver + "\033[1;33;40m\n") + print('\n\033[94m[-] Using DNS server: ' + self.nameserver + '\033[1;33;40m\n') - hostname = str(host.split("\n")[0]) + "." + str(self.domain) + hostname = str(host.split('\n')[0]) + '.' + str(self.domain) if self.verbose: ESC = chr(27) sys.stdout.write(ESC + '[2K' + ESC + '[G') - sys.stdout.write("\r" + hostname) + sys.stdout.write('\r' + hostname) sys.stdout.flush() try: test = DNS.Request( @@ -136,7 +136,7 @@ class dns_force(): ) # TODO FIX test is sometimes not getting answers and leads to an indexing error. hostip = test.answers[0]['data'] - return hostname + ":" + hostip + return hostname + ':' + hostip except Exception: pass @@ -145,7 +145,7 @@ class dns_force(): for x in self.list: host = self.run(x) if host is not None: - print(" : " + host.split(":")[1]) + print(' : ' + host.split(':')[1]) results.append(host) return results @@ -159,37 +159,37 @@ class dns_tld(): self.verbose = verbose # Updated from http://data.iana.org/TLD/tlds-alpha-by-domain.txt self.tlds = [ - "ac", "academy", "ad", "ae", "aero", "af", "ag", "ai", "al", "am", "an", "ao", "aq", "ar", "arpa", "as", - "asia", "at", "au", "aw", "ax", "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bike", "biz", "bj", - "bm", "bn", "bo", "br", "bs", "bt", "builders", "buzz", "bv", "bw", "by", "bz", "ca", "cab", "camera", - "camp", "careers", "cat", "cc", "cd", "center", "ceo", "cf", "cg", "ch", "ci", "ck", "cl", "clothing", - "cm", "cn", "co", "codes", "coffee", "com", "company", "computer", "construction", "contractors", "coop", - "cr", "cu", "cv", "cw", "cx", "cy", "cz", "de", "diamonds", "directory", "dj", "dk", "dm", "do", - "domains", "dz", "ec", "edu", "education", "ee", "eg", "email", "enterprises", "equipment", "er", "es", - "estate", "et", "eu", "farm", "fi", "fj", "fk", "florist", "fm", "fo", "fr", "ga", "gallery", "gb", "gd", - "ge", "gf", "gg", "gh", "gi", "gl", "glass", "gm", "gn", "gov", "gp", "gq", "gr", "graphics", "gs", "gt", - "gu", "guru", "gw", "gy", "hk", "hm", "hn", "holdings", "holiday", "house", "hr", "ht", "hu", "id", "ie", - "il", "im", "immobilien", "in", "info", "institute", "int", "international", "io", "iq", "ir", "is", "it", - "je", "jm", "jo", "jobs", "jp", "kaufen", "ke", "kg", "kh", "ki", "kitchen", "kiwi", "km", "kn", "kp", - "kr", "kw", "ky", "kz", "la", "land", "lb", "lc", "li", "lighting", "limo", "lk", "lr", "ls", "lt", "lu", - "lv", "ly", "ma", "management", "mc", "md", "me", "menu", "mg", "mh", "mil", "mk", "ml", "mm", "mn", "mo", - "mobi", "mp", "mq", "mr", "ms", "mt", "mu", "museum", "mv", "mw", "mx", "my", "mz", "na", "name", "nc", - "ne", "net", "nf", "ng", "ni", "ninja", "nl", "no", "np", "nr", "nu", "nz", "om", "onl", "org", "pa", "pe", - "pf", "pg", "ph", "photography", "photos", "pk", "pl", "plumbing", "pm", "pn", "post", "pr", "pro", "ps", - "pt", "pw", "py", "qa", "re", "recipes", "repair", "ro", "rs", "ru", "ruhr", "rw", "sa", "sb", "sc", "sd", - "se", "sexy", "sg", "sh", "shoes", "si", "singles", "sj", "sk", "sl", "sm", "sn", "so", "solar", - "solutions", "sr", "st", "su", "support", "sv", "sx", "sy", "systems", "sz", "tattoo", "tc", "td", - "technology", "tel", "tf", "tg", "th", "tips", "tj", "tk", "tl", "tm", "tn", "to", "today", "tp", "tr", - "training", "travel", "tt", "tv", "tw", "tz", "ua", "ug", "uk", "uno", "us", "uy", "uz", "va", "vc", - "ve", "ventures", "vg", "vi", "viajes", "vn", "voyage", "vu", "wang", "wf", "wien", "ws", "xxx", "ye", - "yt", "za", "zm", "zw"] + 'ac', 'academy', 'ad', 'ae', 'aero', 'af', 'ag', 'ai', 'al', 'am', 'an', 'ao', 'aq', 'ar', 'arpa', 'as', + 'asia', 'at', 'au', 'aw', 'ax', 'az', 'ba', 'bb', 'bd', 'be', 'bf', 'bg', 'bh', 'bi', 'bike', 'biz', 'bj', + 'bm', 'bn', 'bo', 'br', 'bs', 'bt', 'builders', 'buzz', 'bv', 'bw', 'by', 'bz', 'ca', 'cab', 'camera', + 'camp', 'careers', 'cat', 'cc', 'cd', 'center', 'ceo', 'cf', 'cg', 'ch', 'ci', 'ck', 'cl', 'clothing', + 'cm', 'cn', 'co', 'codes', 'coffee', 'com', 'company', 'computer', 'construction', 'contractors', 'coop', + 'cr', 'cu', 'cv', 'cw', 'cx', 'cy', 'cz', 'de', 'diamonds', 'directory', 'dj', 'dk', 'dm', 'do', + 'domains', 'dz', 'ec', 'edu', 'education', 'ee', 'eg', 'email', 'enterprises', 'equipment', 'er', 'es', + 'estate', 'et', 'eu', 'farm', 'fi', 'fj', 'fk', 'florist', 'fm', 'fo', 'fr', 'ga', 'gallery', 'gb', 'gd', + 'ge', 'gf', 'gg', 'gh', 'gi', 'gl', 'glass', 'gm', 'gn', 'gov', 'gp', 'gq', 'gr', 'graphics', 'gs', 'gt', + 'gu', 'guru', 'gw', 'gy', 'hk', 'hm', 'hn', 'holdings', 'holiday', 'house', 'hr', 'ht', 'hu', 'id', 'ie', + 'il', 'im', 'immobilien', 'in', 'info', 'institute', 'int', 'international', 'io', 'iq', 'ir', 'is', 'it', + 'je', 'jm', 'jo', 'jobs', 'jp', 'kaufen', 'ke', 'kg', 'kh', 'ki', 'kitchen', 'kiwi', 'km', 'kn', 'kp', + 'kr', 'kw', 'ky', 'kz', 'la', 'land', 'lb', 'lc', 'li', 'lighting', 'limo', 'lk', 'lr', 'ls', 'lt', 'lu', + 'lv', 'ly', 'ma', 'management', 'mc', 'md', 'me', 'menu', 'mg', 'mh', 'mil', 'mk', 'ml', 'mm', 'mn', 'mo', + 'mobi', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu', 'museum', 'mv', 'mw', 'mx', 'my', 'mz', 'na', 'name', 'nc', + 'ne', 'net', 'nf', 'ng', 'ni', 'ninja', 'nl', 'no', 'np', 'nr', 'nu', 'nz', 'om', 'onl', 'org', 'pa', 'pe', + 'pf', 'pg', 'ph', 'photography', 'photos', 'pk', 'pl', 'plumbing', 'pm', 'pn', 'post', 'pr', 'pro', 'ps', + 'pt', 'pw', 'py', 'qa', 're', 'recipes', 'repair', 'ro', 'rs', 'ru', 'ruhr', 'rw', 'sa', 'sb', 'sc', 'sd', + 'se', 'sexy', 'sg', 'sh', 'shoes', 'si', 'singles', 'sj', 'sk', 'sl', 'sm', 'sn', 'so', 'solar', + 'solutions', 'sr', 'st', 'su', 'support', 'sv', 'sx', 'sy', 'systems', 'sz', 'tattoo', 'tc', 'td', + 'technology', 'tel', 'tf', 'tg', 'th', 'tips', 'tj', 'tk', 'tl', 'tm', 'tn', 'to', 'today', 'tp', 'tr', + 'training', 'travel', 'tt', 'tv', 'tw', 'tz', 'ua', 'ug', 'uk', 'uno', 'us', 'uy', 'uz', 'va', 'vc', + 've', 'ventures', 'vg', 'vi', 'viajes', 'vn', 'voyage', 'vu', 'wang', 'wf', 'wien', 'ws', 'xxx', 'ye', + 'yt', 'za', 'zm', 'zw'] def getdns(self, domain): dom = domain if self.subdo is True: - dom = domain.split(".") + dom = domain.split('.') dom.pop(0) - rootdom = ".".join(dom) + rootdom = '.'.join(dom) else: rootdom = dom if self.nameserver is False: @@ -197,21 +197,21 @@ class dns_tld(): primary, email, serial, refresh, retry, expire, minimum = r.answers[ 0]['data'] test = DNS.Request(rootdom, qtype='NS', server=primary, aa=1).req() - if test.header['status'] != "NOERROR": - print("Error") + if test.header['status'] != 'NOERROR': + print('Error') sys.exit() self.nameserver = test.answers[0]['data'] - elif self.nameserver == "local": + elif self.nameserver == 'local': self.nameserver = nameserver return self.nameserver def run(self, tld): self.nameserver = self.getdns(self.domain) - hostname = self.domain.split(".")[0] + "." + tld + hostname = self.domain.split('.')[0] + '.' + tld if self.verbose: ESC = chr(27) sys.stdout.write(ESC + '[2K' + ESC + '[G') - sys.stdout.write("\r\tSearching for: " + hostname) + sys.stdout.write('\r\tSearching for: ' + hostname) sys.stdout.flush() try: test = DNS.Request( @@ -220,7 +220,7 @@ class dns_tld(): server=self.nameserver).req( ) hostip = test.answers[0]['data'] - return hostip + ":" + hostname + return hostip + ':' + hostname except Exception: pass diff --git a/discovery/dogpilesearch.py b/discovery/dogpilesearch.py index 1fa7649b..ef64de98 100644 --- a/discovery/dogpilesearch.py +++ b/discovery/dogpilesearch.py @@ -10,15 +10,15 @@ class SearchDogpile: def __init__(self, word, limit): self.word = word self.total_results = "" - self.server = "www.dogpile.com" - self.hostname = "www.dogpile.com" + self.server = 'www.dogpile.com' + self.hostname = 'www.dogpile.com' self.limit = limit self.counter = 0 def do_search(self): # Dogpile is hardcoded to return 10 results. - url = 'http://' + self.server + "/search/web?qsi=" + str(self.counter) \ - + "&q=\"%40" + self.word + "\"" + url = 'http://' + self.server + '/search/web?qsi=' + str(self.counter) \ + + '&q=\"%40' + self.word + '\"' headers = { 'Host': self.hostname, 'User-agent': Core.get_user_agent() diff --git a/discovery/duckduckgosearch.py b/discovery/duckduckgosearch.py index 3f3bb740..bd882770 100644 --- a/discovery/duckduckgosearch.py +++ b/discovery/duckduckgosearch.py @@ -14,9 +14,9 @@ class SearchDuckDuckGo: self.totalresults = "" self.dorks = [] self.links = [] - self.database = "https://duckduckgo.com/?q=" - self.api = "https://api.duckduckgo.com/?q=x&format=json&pretty=1" # Currently using API. - self.quantity = "100" + self.database = 'https://duckduckgo.com/?q=' + self.api = 'https://api.duckduckgo.com/?q=x&format=json&pretty=1' # Currently using API. + self.quantity = '100' self.limit = limit def do_search(self): @@ -39,9 +39,9 @@ class SearchDuckDuckGo: def crawl(self, text): """ - function parses json and returns urls + Function parses json and returns URLs. :param text: formatted json - :return: set of urls + :return: set of URLs """ urls = set() try: diff --git a/discovery/exaleadsearch.py b/discovery/exaleadsearch.py index 0f57c2ed..0c71d5a8 100644 --- a/discovery/exaleadsearch.py +++ b/discovery/exaleadsearch.py @@ -10,20 +10,20 @@ class search_exalead: def __init__(self, word, limit, start): self.word = word - self.files = "pdf" + self.files = 'pdf' self.results = "" self.totalresults = "" - self.server = "www.exalead.com" - self.hostname = "www.exalead.com" + self.server = 'www.exalead.com' + self.hostname = 'www.exalead.com' self.limit = limit self.counter = start def do_search(self): url = 'http:// ' + self.server + '/search/web/results/?q=%40' + self.word \ - + "&elements_per_page=50&start_index=" + str(self.counter) + + '&elements_per_page=50&start_index=' + str(self.counter) headers = { 'Host': self.hostname, - 'Referer': ("http://" + self.hostname + "/search/web/results/?q=%40" + self.word), + 'Referer': ('http://' + self.hostname + '/search/web/results/?q=%40' + self.word), 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) @@ -32,10 +32,10 @@ class search_exalead: def do_search_files(self, files): url = 'http:// ' + self.server + '/search/web/results/?q=%40' + self.word \ - + "filetype:" + self.files + "&elements_per_page=50&start_index=" + str(self.counter) + + 'filetype:' + self.files + '&elements_per_page=50&start_index=' + str(self.counter) headers = { 'Host': self.hostname, - 'Referer': ("http://" + self.hostname + "/search/web/results/?q=%40" + self.word), + 'Referer': ('http://' + self.hostname + '/search/web/results/?q=%40' + self.word), 'User-agent': Core.get_user_agent() } h = requests.get(url=url, headers=headers) @@ -46,10 +46,10 @@ class search_exalead: renext = re.compile('topNextUrl') nextres = renext.findall(self.results) if nextres != []: - nexty = "1" + nexty = '1' print(str(self.counter)) else: - nexty = "0" + nexty = '0' return nexty def get_emails(self): @@ -75,7 +75,7 @@ class search_exalead: self.do_search_files(files) time.sleep(getDelay()) more = self.check_next() - if more == "1": + if more == '1': self.counter += 50 else: break diff --git a/discovery/googleCSE.py b/discovery/googleCSE.py index 32c5067f..0c7719d6 100644 --- a/discovery/googleCSE.py +++ b/discovery/googleCSE.py @@ -10,13 +10,13 @@ class SearchGoogleCSE: def __init__(self, word, limit, start): self.word = word - self.files = "pdf" + self.files = 'pdf' self.results = "" self.totalresults = "" - self.server = "www.googleapis.com" - self.hostname = "www.googleapis.com" - self.userAgent = "(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6" - self.quantity = "10" + self.server = 'www.googleapis.com' + self.hostname = 'www.googleapis.com' + self.userAgent = '(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6' + self.quantity = '10' self.limit = limit self.counter = 1 self.api_key = googleCSEAPI_key @@ -29,9 +29,9 @@ class SearchGoogleCSE: self.highRange = start + 100 def do_search(self): - url = 'https://' + self.server + "/customsearch/v1?key=" + self.api_key + "&highrange=" + str(self.highRange) \ - + '&lowrange=' + str(self.lowRange) + '&cx=' + self.cse_id + "&start=" + str(self.counter) + \ - "&q=" + self.word + url = 'https://' + self.server + '/customsearch/v1?key=' + self.api_key + '&highrange=' + str(self.highRange) \ + + '&lowrange=' + str(self.lowRange) + '&cx=' + self.cse_id + '&start=' + str(self.counter) + \ + '&q=' + self.word headers = { 'Host': self.server, 'User-agent': self.userAgent @@ -42,9 +42,9 @@ class SearchGoogleCSE: self.totalresults += self.results def do_search_files(self, files): - url = 'https://' + self.server + "/customsearch/v1?key=" + self.api_key + "&highRange=" + str(self.highRange) \ - + '&lowRange=' + str(self.lowRange) + '&cx=' + self.cse_id + "&start=" + str(self.counter) + \ - "&q=filetype:" + files + "%20site:" + self.word + url = 'https://' + self.server + '/customsearch/v1?key=' + self.api_key + '&highRange=' + str(self.highRange) \ + + '&lowRange=' + str(self.lowRange) + '&cx=' + self.cse_id + '&start=' + str(self.counter) + \ + '&q=filetype:' + files + '%20site:' + self.word headers = { 'Host': self.server, 'User-agent': self.userAgent @@ -57,9 +57,9 @@ class SearchGoogleCSE: renext = re.compile('> Next <') nextres = renext.findall(self.results) if nextres != []: - nexty = "1" + nexty = '1' else: - nexty = "0" + nexty = '0' return nexty def get_emails(self): @@ -80,7 +80,7 @@ class SearchGoogleCSE: self.do_search() ESC = chr(27) sys.stdout.write(ESC + '[2K' + ESC + '[G') - sys.stdout.write("\r\t" + "Searching " + str(self.counter + self.lowRange) + " results.") + sys.stdout.write('\r\t' + 'Searching ' + str(self.counter + self.lowRange) + ' results.') sys.stdout.flush() if self.counter == 101: self.counter = 1 @@ -91,7 +91,7 @@ class SearchGoogleCSE: tracker = self.counter + self.lowRange def store_results(self): - filename = "debug_results.txt" + filename = 'debug_results.txt' file = open(filename, 'w') file.write(self.totalresults) @@ -100,4 +100,4 @@ class SearchGoogleCSE: self.do_search_files(files) time.sleep(1) self.counter += 100 - print("\tSearching " + str(self.counter) + " results.") + print('\tSearching ' + str(self.counter) + ' results.') diff --git a/discovery/googlecertificates.py b/discovery/googlecertificates.py index 4fe57427..c59688f5 100644 --- a/discovery/googlecertificates.py +++ b/discovery/googlecertificates.py @@ -9,14 +9,14 @@ class SearchGoogleCertificates: self.word = word self.results = "" self.totalresults = "" - self.server = "www.google.com" - self.quantity = "100" + self.server = 'www.google.com' + self.quantity = '100' self.limit = limit self.counter = start def do_search(self): try: - urly = "https://" + self.server + "/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=true&include_subdomains=true&domain=" + self.word + urly = 'https://' + self.server + '/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=true&include_subdomains=true&domain=' + self.word except Exception as e: print(e) try: @@ -29,7 +29,7 @@ class SearchGoogleCertificates: def get_domains(self): domains = [] - rawres = json.loads(self.totalresults.split("\n", 2)[2]) + rawres = json.loads(self.totalresults.split('\n', 2)[2]) for array in rawres[0][1]: domains.append(array[1]) return list(set(domains)) diff --git a/discovery/googlesearch.py b/discovery/googlesearch.py index 8ac2c699..320a3d0e 100644 --- a/discovery/googlesearch.py +++ b/discovery/googlesearch.py @@ -10,18 +10,18 @@ class search_google: self.word = word self.results = "" self.totalresults = "" - self.server = "www.google.com" + self.server = 'www.google.com' self.dorks = [] self.links = [] - self.database = "https://www.google.com/search?q=" - self.quantity = "100" + self.database = 'https://www.google.com/search?q=' + self.quantity = '100' self.limit = limit self.counter = start def do_search(self): try: # Do normal scraping. - urly = "http://" + self.server + "/search?num=" + self.quantity + "&start=" + str( - self.counter) + "&hl=en&meta=&q=%40\"" + self.word + "\"" + urly = 'http://' + self.server + '/search?num=' + self.quantity + '&start=' + str( + self.counter) + '&hl=en&meta=&q=%40\"' + self.word + '\"' except Exception as e: print(e) try: @@ -38,8 +38,8 @@ class search_google: def do_search_profiles(self): try: - urly = "http://" + self.server + "/search?num=" + self.quantity + "&start=" + str( - self.counter) + "&hl=en&meta=&q=site:www.google.com%20intitle:\"Google%20Profile\"%20\"Companies%20I%27ve%20worked%20for\"%20\"at%20" + self.word + "\"" + urly = 'http://' + self.server + '/search?num=' + self.quantity + '&start=' + str( + self.counter) + '&hl=en&meta=&q=site:www.google.com%20intitle:\"Google%20Profile\"%20\"Companies%20I%27ve%20worked%20for\"%20\"at%20' + self.word + '\"' except Exception as e: print(e) try: @@ -79,7 +79,7 @@ class search_google: else: # Google dorking is true. self.counter = 0 # Reset counter. print('\n') - print("[-] Searching with Google Dorks: ") + print('[-] Searching with Google Dorks: ') while self.counter <= self.limit and self.counter <= 200: # Only 200 dorks in list. self.googledork() # Call Google dorking method if user wanted it! print(f'\tSearching {self.counter} results.') @@ -102,27 +102,27 @@ class search_google: def construct_dorks(self): # Format is: site:targetwebsite.com + space + inurl:admindork - colon = "%3A" - plus = "%2B" + colon = '%3A' + plus = '%2B' space = '+' - period = "%2E" - double_quote = "%22" - asterick = "%2A" - left_bracket = "%5B" - right_bracket = "%5D" - question_mark = "%3F" - slash = "%2F" - single_quote = "%27" - ampersand = "%26" - left_peren = "%28" - right_peren = "%29" + period = '%2E' + double_quote = '%22' + asterick = '%2A' + left_bracket = '%5B' + right_bracket = '%5D' + question_mark = '%3F' + slash = '%2F' + single_quote = '%27' + ampersand = '%26' + left_peren = '%28' + right_peren = '%29' pipe = '%7C' # Replace links with html encoding. self.links = [self.database + space + self.word + space + str(dork).replace(':', colon).replace('+', plus).replace('.', period).replace('"', double_quote) - .replace("*", asterick).replace('[', left_bracket).replace(']', right_bracket) + .replace('*', asterick).replace('[', left_bracket).replace(']', right_bracket) .replace('?', question_mark).replace(' ', space).replace('/', slash).replace("'",single_quote) - .replace("&", ampersand).replace('(', left_peren).replace(')', right_peren).replace('|', pipe) + .replace('&', ampersand).replace('(', left_peren).replace(')', right_peren).replace('|', pipe) for dork in self.dorks] def googledork(self): diff --git a/discovery/googlesets.py b/discovery/googlesets.py index 2428f69a..795ecb9f 100644 --- a/discovery/googlesets.py +++ b/discovery/googlesets.py @@ -8,19 +8,19 @@ class search_google_labs: def __init__(self, list): self.results = "" self.totalresults = "" - self.server = "labs.google.com" - self.hostname = "labs.google.com" + self.server = 'labs.google.com' + self.hostname = 'labs.google.com' id = 0 self.set = "" for x in list: id += 1 if id == 1: - self.set = self.set + "q" + str(id) + "=" + str(x) + self.set = self.set + 'q' + str(id) + '=' + str(x) else: - self.set = self.set + "&q" + str(id) + "=" + str(x) + self.set = self.set + '&q' + str(id) + '=' + str(x) def do_search(self): - url = 'http://' + self.server + "/sets?hl-en&" + self.set + url = 'http://' + self.server + '/sets?hl-en&' + self.set headers = { 'Host': self.server, 'User-agent': Core.get_user_agent() diff --git a/discovery/huntersearch.py b/discovery/huntersearch.py index e4f6ab3f..8e74f846 100644 --- a/discovery/huntersearch.py +++ b/discovery/huntersearch.py @@ -15,7 +15,7 @@ class SearchHunter: self.results = "" self.totalresults = "" self.counter = start - self.database = "https://api.hunter.io/v2/domain-search?domain=" + word + "&api_key=" + self.key + "&limit=" + str(self.limit) + self.database = 'https://api.hunter.io/v2/domain-search?domain=' + word + '&api_key=' + self.key + '&limit=' + str(self.limit) def do_search(self): try: @@ -27,7 +27,7 @@ class SearchHunter: def process(self): self.do_search() # Only need to do it once. - print('\tDone Searching Results') + print('\tSearching results.') def get_emails(self): rawres = myparser.Parser(self.totalresults, self.word) diff --git a/discovery/linkedinsearch.py b/discovery/linkedinsearch.py index 6aa069f4..dce71807 100644 --- a/discovery/linkedinsearch.py +++ b/discovery/linkedinsearch.py @@ -11,15 +11,15 @@ class SearchLinkedin: self.word = word.replace(' ', '%20') self.results = "" self.totalresults = "" - self.server = "www.google.com" - self.userAgent = "(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6" - self.quantity = "100" + self.server = 'www.google.com' + self.userAgent = '(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6' + self.quantity = '100' self.limit = int(limit) self.counter = 0 def do_search(self): try: - urly = "http://" + self.server + "/search?num=100&start=" + str(self.counter) + "&hl=en&meta=&q=site%3Alinkedin.com/in%20" + self.word + urly = 'http://' + self.server + '/search?num=100&start=' + str(self.counter) + '&hl=en&meta=&q=site%3Alinkedin.com/in%20' + self.word except Exception as e: print(e) try: diff --git a/discovery/netcraft.py b/discovery/netcraft.py index 4cd4e919..0b4959e9 100644 --- a/discovery/netcraft.py +++ b/discovery/netcraft.py @@ -9,14 +9,14 @@ class SearchNetcraft: self.word = word.replace(' ', '%20') self.results = "" self.totalresults = "" - self.server = "www.google.com" - self.hostname = "www.google.com" - self.quantity = "100" + self.server = 'www.google.com' + self.hostname = 'www.google.com' + self.quantity = '100' self.counter = 0 def do_search(self): try: - urly = "https://searchdns.netcraft.com/?restriction=site+ends+with&host=" + self.word + urly = 'https://searchdns.netcraft.com/?restriction=site+ends+with&host=' + self.word except Exception as e: print(e) headers = {'User-Agent': Core.get_user_agent()} @@ -33,4 +33,4 @@ class SearchNetcraft: def process(self): self.do_search() - print("\tSearching Netcraft results.") + print('\tSearching results.') diff --git a/discovery/pgpsearch.py b/discovery/pgpsearch.py index 4d1ba61e..a87456a9 100644 --- a/discovery/pgpsearch.py +++ b/discovery/pgpsearch.py @@ -8,13 +8,13 @@ class SearchPgp: def __init__(self, word): self.word = word self.results = "" - self.server = "pgp.mit.edu" - self.hostname = "pgp.mit.edu" + self.server = 'pgp.mit.edu' + self.hostname = 'pgp.mit.edu' def process(self): - print("\tSearching PGP results.") + print('\tSearching results.') try: - url = 'http://' + self.server + "/pks/lookup?search=" + self.word + "&op=index" + url = 'http://' + self.server + '/pks/lookup?search=' + self.word + '&op=index' headers = { 'Host': self.hostname, 'User-agent': Core.get_user_agent() @@ -23,7 +23,7 @@ class SearchPgp: self.results = h.text self.results += self.results except Exception as e: - print("Unable to connect to PGP server: ", str(e)) + print('Unable to connect to PGP server: ', str(e)) def get_emails(self): rawres = myparser.Parser(self.results, self.word) diff --git a/discovery/s3_scanner.py b/discovery/s3_scanner.py index 0c8293b6..a27800ba 100644 --- a/discovery/s3_scanner.py +++ b/discovery/s3_scanner.py @@ -8,21 +8,21 @@ class s3_scanner: self.host = host self.results = "" self.totalresults = "" - self.fingerprints = ["www.herokucdn.com/error-pages/no-such-app.html", "Squarespace - No Such Account", "

If you're trying to publish one, read the full documentation to learn how to set up GitHub Pages for your repository, organization, or user account.

","

If you\'re trying to publish one, read the full documentation to learn how to set up GitHub Pages for your repository, organization, or user account.

","Bummer. It looks like the help center that you are trying to reach no longer exists."," The page you\'re looking for could not be found (404) "] + self.fingerprints = ['www.herokucdn.com/error-pages/no-such-app.html', 'Squarespace - No Such Account', "

If you're trying to publish one, read the full documentation to learn how to set up GitHub Pages for your repository, organization, or user account.

","

If you\'re trying to publish one, read the full documentation to learn how to set up GitHub Pages for your repository, organization, or user account.

","Bummer. It looks like the help center that you are trying to reach no longer exists."," The page you\'re looking for could not be found (404) "] def __check_http(self, bucket_url): check_response = self.session.head( - S3_URL, timeout=3, headers={"Host": bucket_url}) + S3_URL, timeout=3, headers={'Host': bucket_url}) # if not ARGS.ignore_rate_limiting\ -# and (check_response.status_code == 503 and check_response.reason == "Slow Down"): +# and (check_response.status_code == 503 and check_response.reason == 'Slow Down'): # self.q.rate_limited = True - # add it back to the bucket for re-processing + # Add it back to the bucket for re-processing. # self.q.put(bucket_url) if check_response.status_code == 307: # valid bucket, lets check if its public - new_bucket_url = check_response.headers["Location"] + new_bucket_url = check_response.headers['Location'] bucket_response = requests.request( - "GET" if ARGS.only_interesting else "HEAD", new_bucket_url, timeout=3) + 'GET' if ARGS.only_interesting else 'HEAD', new_bucket_url, timeout=3) if bucket_response.status_code == 200\ and (not ARGS.only_interesting or @@ -32,13 +32,13 @@ class s3_scanner: def do_s3(self): try: - print("\t Searching takeovers for " + self.host) + print('\t Searching takeovers for ' + self.host) r = requests.get('https://' + self.host, verify=False) for x in self.fingerprints: take_reg = re.compile(x) self.temp = take_reg.findall(r.text) if self.temp != []: - print("\t\033[91m Takeover detected! - " + self.host + "\033[1;32;40m ") + print('\t\033[91m Takeover detected! - ' + self.host + '\033[1;32;40m') except Exception as e: print(e) diff --git a/discovery/securitytrailssearch.py b/discovery/securitytrailssearch.py index 30cc57ec..28b350e1 100644 --- a/discovery/securitytrailssearch.py +++ b/discovery/securitytrailssearch.py @@ -14,7 +14,7 @@ class search_securitytrail: raise MissingKey(True) self.results = "" self.totalresults = "" - self.database = "https://api.securitytrails.com/v1/" + self.database = 'https://api.securitytrails.com/v1/' self.info = () def authenticate(self): @@ -52,7 +52,7 @@ class search_securitytrail: parser = securitytrailsparser.Parser(word=self.word, text=self.totalresults) self.info = parser.parse_text() # Create parser and set self.info to tuple returned from parsing text. - print('\tDone Searching Results') + print('\tSearching results.') def get_ips(self): return self.info[0] diff --git a/discovery/shodansearch.py b/discovery/shodansearch.py index 7b1480e9..588609ee 100644 --- a/discovery/shodansearch.py +++ b/discovery/shodansearch.py @@ -33,10 +33,10 @@ class search_shodan: str(results.get('org')), str(servicesports).replace('\'', '').strip('[]'), str(technologies).replace('\'', '').strip('[]')] except exception.APIError: - print(f'{ipaddress}: Not in Shodan') - self.hostdatarow = [ipaddress, "Not in Shodan", "Not in Shodan", "Not in Shodan", "Not in Shodan"] + print(f'{ipaddress}: Not in Shodan.') + self.hostdatarow = [ipaddress, 'Not in Shodan', 'Not in Shodan', 'Not in Shodan', 'Not in Shodan'] except Exception as e: - print(f'Error occurred in the Shodan IP search module: {e}') + print(f'[!] Error occurred in the Shodan IP search module: {e}') finally: return self.hostdatarow diff --git a/discovery/takeover.py b/discovery/takeover.py index b03b9748..3a3979d1 100644 --- a/discovery/takeover.py +++ b/discovery/takeover.py @@ -8,25 +8,25 @@ class take_over: self.host = host self.results = "" self.totalresults = "" - self.fingerprints = ["Squarespace - Domain Not Claimed" - ,"www.herokucdn.com/error-pages/no-such-app.html", - "Squarespace - No Such Account", + self.fingerprints = ["Squarespace - Domain Not Claimed", + 'www.herokucdn.com/error-pages/no-such-app.html', + 'Squarespace - No Such Account', "

If you're trying to publish one, read the full documentation to learn how to set up GitHub Pages for your repository, organization, or user account.

", "

If you\'re trying to publish one, read the full documentation to learn how to set up GitHub Pages for your repository, organization, or user account.

", "Bummer. It looks like the help center that you are trying to reach no longer exists.", " The page you\'re looking for could not be found (404) ", - "The specified bucket does not exist", - "Bad Request: ERROR: The request could not be satisfied", - "Fastly error: unknown domain:", + 'The specified bucket does not exist', + 'Bad Request: ERROR: The request could not be satisfied', + 'Fastly error: unknown domain:', "There isn't a Github Pages site here.", - "No such app", - "Unrecognized domain", - "Sorry, this shop is currently unavailable.", + 'No such app', + 'Unrecognized domain', + 'Sorry, this shop is currently unavailable.', "Whatever you were looking for doesn't currently exist at this address", - "The requested URL was not found on this server.", - "This UserVoice subdomain is currently available!", - "Do you want to register *.wordpress.com?", - "Help Center Closed"] + 'The requested URL was not found on this server.', + 'This UserVoice subdomain is currently available!', + 'Do you want to register *.wordpress.com?', + 'Help Center Closed'] def do_take(self): try: @@ -36,7 +36,7 @@ class take_over: take_reg = re.compile(x) self.temp = take_reg.findall(r.text) if self.temp != []: - print(f"\t\033[91m Takeover detected! - {self.host} \033[1;32;40m") + print(f'\t\033[91m Takeover detected! - {self.host} \033[1;32;40m') except Exception as e: print(e) diff --git a/discovery/threatcrowd.py b/discovery/threatcrowd.py index f92dfa9c..e9620ddd 100644 --- a/discovery/threatcrowd.py +++ b/discovery/threatcrowd.py @@ -9,14 +9,14 @@ class search_threatcrowd: self.word = word.replace(' ', '%20') self.results = "" self.totalresults = "" - self.server = "www.google.com" - self.hostname = "www.google.com" - self.quantity = "100" + self.server = 'www.google.com' + self.hostname = 'www.google.com' + self.quantity = '100' self.counter = 0 def do_search(self): try: - urly = "https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=" + self.word + urly = 'https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=' + self.word except Exception as e: print(e) headers = {'User-Agent': Core.get_user_agent()} @@ -33,4 +33,4 @@ class search_threatcrowd: def process(self): self.do_search() - print('\tSearching Threatcrowd results.') + print('\tSearching results.') diff --git a/discovery/trello.py b/discovery/trello.py index b9a21282..200c5df2 100644 --- a/discovery/trello.py +++ b/discovery/trello.py @@ -10,16 +10,16 @@ class search_trello: self.word = word.replace(' ', '%20') self.results = "" self.totalresults = "" - self.server = "www.google.com" - self.hostname = "www.google.com" - self.quantity = "100" + self.server = 'www.google.com' + self.hostname = 'www.google.com' + self.quantity = '100' self.limit = limit self.counter = 0 def do_search(self): try: - urly = "https://" + self.server + "/search?num=100&start=" + str( - self.counter) + "&hl=en&q=site%3Atrello.com%20" + self.word + urly = 'https://' + self.server + '/search?num=100&start=' + str( + self.counter) + '&hl=en&q=site%3Atrello.com%20' + self.word except Exception as e: print(e) headers = {'User-Agent': googleUA} @@ -36,9 +36,9 @@ class search_trello: return rawres.emails() def get_urls(self): - print('\tSearching Trello URLs.') + print('\tSearching URLs.') try: - rawres = myparser.Parser(self.totalresults, "trello.com") + rawres = myparser.Parser(self.totalresults, 'trello.com') trello_urls = rawres.urls() visited = set() for url in trello_urls: diff --git a/discovery/twittersearch.py b/discovery/twittersearch.py index 486c76b4..bc4bcbab 100644 --- a/discovery/twittersearch.py +++ b/discovery/twittersearch.py @@ -11,15 +11,15 @@ class search_twitter: self.word = word.replace(' ', '%20') self.results = "" self.totalresults = "" - self.server = "www.google.com" - self.hostname = "www.google.com" - self.quantity = "100" + self.server = 'www.google.com' + self.hostname = 'www.google.com' + self.quantity = '100' self.limit = int(limit) self.counter = 0 def do_search(self): try: - urly = "https://" + self.server + "/search?num=100&start=" + str(self.counter) + "&hl=en&meta=&q=site%3Atwitter.com%20intitle%3A%22on+Twitter%22%20" + self.word + urly = 'https://' + self.server + '/search?num=100&start=' + str(self.counter) + '&hl=en&meta=&q=site%3Atwitter.com%20intitle%3A%22on+Twitter%22%20' + self.word except Exception as e: print(e) headers = {'User-Agent': Core.get_user_agent()} diff --git a/discovery/virustotal.py b/discovery/virustotal.py index ce2dbe5c..5e00fa66 100644 --- a/discovery/virustotal.py +++ b/discovery/virustotal.py @@ -9,14 +9,14 @@ class search_virustotal: self.word = word.replace(' ', '%20') self.results = "" self.totalresults = "" - self.server = "www.google.com" - self.hostname = "www.google.com" - self.quantity = "100" + self.server = 'www.google.com' + self.hostname = 'www.google.com' + self.quantity = '100' self.counter = 0 def do_search(self): try: - urly = "https://www.virustotal.com/en/domain/" + self.word + "/information/" + urly = 'https://www.virustotal.com/en/domain/' + self.word + '/information/' except Exception as e: print(e) headers = {'User-Agent': Core.get_user_agent()} @@ -33,4 +33,4 @@ class search_virustotal: def process(self): self.do_search() - print('\tSearching Virustotal results.') + print('\tSearching results.') diff --git a/discovery/wfuzz_search.py b/discovery/wfuzz_search.py index 37bc73a5..c62447ec 100644 --- a/discovery/wfuzz_search.py +++ b/discovery/wfuzz_search.py @@ -14,7 +14,7 @@ class search_wfuzz: def do_search(self): print('elo') try: - for r in wfuzz.fuzz(url="https://"+self.host+"/FUZZ", hc=[404], payloads=[("file", dict(fn="wordlists/general/common.txt"))]): + for r in wfuzz.fuzz(url='https://'+self.host+'/FUZZ', hc=[404], payloads=[('file', dict(fn='wordlists/general/common.txt'))]): print(r) self.results += r except Exception as e: diff --git a/discovery/yahoosearch.py b/discovery/yahoosearch.py index 6236002d..4c2b5fd3 100644 --- a/discovery/yahoosearch.py +++ b/discovery/yahoosearch.py @@ -10,14 +10,13 @@ class search_yahoo: def __init__(self, word, limit): self.word = word self.total_results = "" - self.server = "search.yahoo.com" - self.hostname = "search.yahoo.com" + self.server = 'search.yahoo.com' + self.hostname = 'search.yahoo.com' self.limit = limit self.counter = 0 def do_search(self): - url = 'http://' + self.server + "/search?p=\"%40" + self.word \ - + "\"&b=" + str(self.counter) + "&pz=10" + url = 'http://' + self.server + '/search?p=\"%40' + self.word + '\"&b=' + str(self.counter) + '&pz=10' headers = { 'Host': self.hostname, 'User-agent': Core.get_user_agent() @@ -29,7 +28,7 @@ class search_yahoo: while self.counter <= self.limit and self.counter <= 1000: self.do_search() time.sleep(getDelay()) - print(f'\tSearching {self.counter} results.') + print(f'\tSearching {self.counter} results.') self.counter += 10 def get_emails(self): diff --git a/discovery/yandexsearch.py b/discovery/yandexsearch.py index b74c7bb8..7ce5f3ea 100644 --- a/discovery/yandexsearch.py +++ b/discovery/yandexsearch.py @@ -12,13 +12,13 @@ class search_yandex: self.word = word self.results = "" self.totalresults = "" - self.server = "yandex.com" - self.hostname = "yandex.com" + self.server = 'yandex.com' + self.hostname = 'yandex.com' self.limit = limit self.counter = start def do_search(self): - url = 'http://' + self.server + "/search?text=%40" + self.word + "&numdoc=50&lr=" + str(self.counter) + url = 'http://' + self.server + '/search?text=%40' + self.word + '&numdoc=50&lr=' + str(self.counter) headers = { 'Host': self.hostname, 'User-agent': Core.get_user_agent() @@ -29,7 +29,7 @@ class search_yandex: print(self.results) def do_search_files(self, files): # TODO - url = 'http://' + self.server + "/search?text=%40" + self.word + "&numdoc=50&lr=" + str(self.counter) + url = 'http://' + self.server + '/search?text=%40' + self.word + '&numdoc=50&lr=' + str(self.counter) headers = { 'Host': self.hostname, 'User-agent': Core.get_user_agent() @@ -42,10 +42,10 @@ class search_yandex: renext = re.compile('topNextUrl') nextres = renext.findall(self.results) if nextres != []: - nexty = "1" + nexty = '1' print(str(self.counter)) else: - nexty = "0" + nexty = '0' return nexty def get_emails(self): diff --git a/theHarvester.py b/theHarvester.py index 6660651d..92875a8c 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -101,7 +101,7 @@ def start(argv): 'linkedin', 'netcraft', 'pgp', 'securityTrails', 'threatcrowd', 'trello', 'twitter', 'vhost', 'virustotal', 'yahoo', 'all']) if set(engines).issubset(supportedengines): - print(f'\033[94m[*] Target domain: {word} \n \033[0m') + print(f'\033[94m[*] Target: {word} \n \033[0m') for engineitem in engines: if engineitem == 'baidu': print('\033[94m[*] Searching Baidu. \033[0m') @@ -208,7 +208,7 @@ def start(argv): db.store_all(word, all_emails, 'email', 'google') elif engineitem == 'googleCSE': - print('\033[94m[*] Searching Google Custom Search. \033[0m') + print('\033[94m[*] Searching Google Custom Search Engine. \033[0m') try: search = googleCSE.SearchGoogleCSE(word, limit, start) search.process() @@ -227,7 +227,7 @@ def start(argv): pass elif engineitem == 'google-certificates': - print('\033[94m[*] Searching Google Certificate transparency report. \033[0m') + print('\033[94m[*] Searching Google Certificate Transparency Report. \033[0m') search = googlecertificates.SearchGoogleCertificates(word, limit, start) search.process() hosts = filter(search.get_domains()) @@ -633,7 +633,7 @@ def start(argv): else: print('\n[*] IPs found: ' + str(len(all_ip))) print('-------------------') - ips = sorted(ipaddress.ip_address(line.strip()) for line in all_ip) + ips = sorted(ipaddress.ip_address(line.strip()) for line in set(all_ip)) print('\n'.join(map(str, ips))) if len(all_emails) == 0: @@ -646,8 +646,8 @@ def start(argv): if len(all_hosts) == 0: print('\n[*] No hosts found.\n\n') else: - print('\n[*] Hosts found: ' + str(len(all_hosts))) - print('---------------------') + print('\n[*] Resolving hosts found: ' + str(len(all_hosts))) + print('-------------------------------') all_hosts = sorted(list(set(all_hosts))) full_host = hostchecker.Checker(all_hosts) full = full_host.check() @@ -725,7 +725,7 @@ def start(argv): s = '.' range = s.join(range) if not analyzed_ranges.count(range): - print('[*] Performing reverse lookup in ' + range) + print('[*] Performing a reverse lookup in ' + range) a = dnssearch.dns_reverse(range, True) a.list() res = a.process() @@ -918,7 +918,7 @@ if __name__ == '__main__': try: start(sys.argv[1:]) except KeyboardInterrupt: - print('\n\n\033[93m[!] ctrl+c detected from user, quitting.\n\n \033[0m') + print('\n\n\033[93m[!] ctrl+c detected, stopping program.\n\n \033[0m') except Exception: import traceback print(traceback.print_exc()) From 5bc000296d42ad4680dd8b6d649ae52fcf91f2e6 Mon Sep 17 00:00:00 2001 From: Lee Baird Date: Sun, 13 Jan 2019 21:19:40 -0600 Subject: [PATCH 2/5] Missing period. --- discovery/duckduckgosearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery/duckduckgosearch.py b/discovery/duckduckgosearch.py index 5d591a27..55de0b09 100644 --- a/discovery/duckduckgosearch.py +++ b/discovery/duckduckgosearch.py @@ -51,7 +51,7 @@ class SearchDuckDuckGo: if isinstance(val, int) or isinstance(val, dict) or val is None: continue if isinstance(val, list): - if len(val) == 0: # Make sure not indexing an empty list + if len(val) == 0: # Make sure not indexing an empty list. continue val = val[0] # First value should be dict. if isinstance(val, dict): # Sanity check. From bd582f09746591ebec5711dd180ac0dc4b66b137 Mon Sep 17 00:00:00 2001 From: Lee Baird Date: Sun, 13 Jan 2019 21:23:45 -0600 Subject: [PATCH 3/5] Updated version number. --- README.md | 2 +- lib/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7edf62ff..8d305848 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ * | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | * * \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| * * * -* theHarvester 3.0.6 v183 * +* theHarvester 3.0.6 v193 * * Coded by Christian Martorella * * Edge-Security Research * * cmartorella@edge-security.com * diff --git a/lib/core.py b/lib/core.py index 305368f0..03fae9cc 100644 --- a/lib/core.py +++ b/lib/core.py @@ -15,7 +15,7 @@ class Core: print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *") print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *") print('* *') - print('* theHarvester 3.0.6 v183 *') + print('* theHarvester 3.0.6 v193 *') print('* Coded by Christian Martorella *') print('* Edge-Security Research *') print('* cmartorella@edge-security.com *') From 900929a00305d47abfcfe74881a90481c4ec6207 Mon Sep 17 00:00:00 2001 From: Lee Baird Date: Sun, 13 Jan 2019 22:38:06 -0600 Subject: [PATCH 4/5] Updated verbiage. --- theHarvester.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/theHarvester.py b/theHarvester.py index 50fe53f8..2c4b18a0 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -725,7 +725,7 @@ def start(argv): s = '.' range = s.join(range) if not analyzed_ranges.count(range): - print('[*] Performing a reverse lookup in ' + range) + print('[*] Performing a reverse lookup on ' + range) a = dnssearch.dns_reverse(range, True) a.list() res = a.process() @@ -737,8 +737,8 @@ def start(argv): dnsrev.append(x) if x not in full: full.append(x) - print('Hosts found after reverse lookup (in target domain):') - print('----------------------------------------------------') + print('[*] Hosts found after reverse lookup:') + print('-------------------------------------') for xh in dnsrev: print(xh) @@ -796,7 +796,7 @@ def start(argv): time.sleep(2) tab.add_row(rowdata) printedtable = tab.draw() - print('\n [*] Shodan results:') + print('\n[*] Shodan results:') print('-------------------') print(printedtable) except Exception as e: @@ -850,20 +850,11 @@ def start(argv): Html_file.close() print('NEW REPORTING FINISHED!') print('[*] Saving files.') - html = htmlExport.htmlExport( - all_emails, - full, - vhost, - dnsres, - dnsrev, - filename, - word, - shodanres, - dnstldres) + html = htmlExport.htmlExport(all_emails, full, vhost, dnsres, dnsrev, filename, word, shodanres, dnstldres) save = html.writehtml() except Exception as e: print(e) - print('Error creating the file.') + print('[!] Error creating the file.') try: filename = filename.split('.')[0] + '.xml' file = open(filename, 'w') From e92e9ea2f185df9df8864633dfb53d2276927831 Mon Sep 17 00:00:00 2001 From: Lee Baird Date: Tue, 22 Jan 2019 21:04:38 -0500 Subject: [PATCH 5/5] Bug fix and tweaks. --- .gitignore | 4 ++-- README.md | 2 +- discovery/censys.py | 2 +- lib/core.py | 12 ++++-------- requirements.txt | 2 +- theHarvester.py | 3 ++- 6 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 5a073d2c..d82d0b1c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ .html .vscode .xml +api-keys.yaml debug_results.txt tests/myparser.py -venv -api-keys.yaml \ No newline at end of file +venv \ No newline at end of file diff --git a/README.md b/README.md index 84031ae7..f7b24937 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ * | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | * * \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| * * * -* theHarvester 3.0.6 v206 * +* theHarvester 3.0.6 v213 * * Coded by Christian Martorella * * Edge-Security Research * * cmartorella@edge-security.com * diff --git a/discovery/censys.py b/discovery/censys.py index c4b0f6c9..4f315a83 100644 --- a/discovery/censys.py +++ b/discovery/censys.py @@ -64,7 +64,7 @@ class SearchCensys: self.page = str(counter) self.urlhost = 'https://' + self.server + '/ipv4/_search?q=' + str(self.word) + '&page=' + str( self.page) - print(f'\tSearching Censys IP results page {self.page}.') + print(f'\tSearching results page {self.page}.') self.do_searchhosturl() counter += 1 except Exception as e: diff --git a/lib/core.py b/lib/core.py index ab2a8a54..c4b71c6f 100644 --- a/lib/core.py +++ b/lib/core.py @@ -1,11 +1,6 @@ # coding=utf-8 -#from discovery import * -# from lib import stash -import os import random -# import re -import sys import yaml @@ -49,7 +44,7 @@ class Core: print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *") print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *") print('* *') - print('* theHarvester 3.0.6 v206 *') + print('* theHarvester 3.0.6 v213 *') print('* Coded by Christian Martorella *') print('* Edge-Security Research *') print('* cmartorella@edge-security.com *') @@ -321,8 +316,9 @@ class Core: return random.choice(user_agents) # TODO use this method when -b all is called to replace lines 383-635 in theHarvester.py - # TODO and to find the best approch of getting the - # word, limit and start etc vars from the arguments and importing libs that are needed + # TODO and to find the best approach of getting the word, limit, and start etc vars from + # the arguments and importing libs that are needed. + # # @staticmethod # def engine_all_search(): # print(('Full harvest on ' + word)) diff --git a/requirements.txt b/requirements.txt index 244b3b73..a6ff9bb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ beautifulsoup4>=4.7.1 plotly>=3.5.0 -PyYaml==3.13 pytest>=4.1.1 +PyYaml==3.13 requests>=2.21.0 shodan>=1.10.0 texttable>=1.6.0 \ No newline at end of file diff --git a/theHarvester.py b/theHarvester.py index a16535b6..8c0b5790 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -14,6 +14,7 @@ import datetime import ipaddress import re import time +import sys try: import bs4 @@ -80,7 +81,7 @@ def start(): word = args.domain engines = set(args.source.split(',')) if set(engines).issubset(Core.get_supportedengines()): - print(f'\033[94m[*] Target domain: {word} \n \033[0m') + print(f'\033[94m[*] Target: {word} \n \033[0m') for engineitem in engines: if engineitem == 'baidu': print('\033[94m[*] Searching Baidu. \033[0m')