diff --git a/censysparser.py b/censysparser.py index a3c51b9f..0d1cad8b 100755 --- a/censysparser.py +++ b/censysparser.py @@ -3,39 +3,59 @@ import re class parser: - - def __init__(self, results): - self.results = results + + def __init__(self, resultstoparse): self.ipaddresses = [] - self.soup = BeautifulSoup(results.results, features="html.parser") + self.souphosts = BeautifulSoup(resultstoparse.total_resultshosts,features="html.parser") + self.soupcerts = BeautifulSoup(resultstoparse.total_resultscerts,features="html.parser") self.hostnames = [] + self.hostnamesfromcerts = [] self.urls = [] - self.numberofpages = 0 + self.numberofpageshosts = 0 + self.numberofpagescerts = 0 - def search_hostnames(self): + def search_hostnamesfromcerts(self): try: - hostnamelist = self.soup.findAll('tt') + hostnamelist = self.soupcerts.findAll("i", "fa fa-fw fa-home") for hostnameitem in hostnamelist: - self.hostnames.append(hostnameitem.text) - return self.hostnames + hostitems = hostnameitem.next_sibling + hostnames = str(hostitems) + hostnamesclean = re.sub('[ \'\[\]]','', hostnames) + hostnamesclean = re.sub(r'\.\.\.',r'',hostnamesclean) + self.hostnamesfromcerts.extend(hostnamesclean.split(",")) + self.hostnamesfromcerts = list(filter(None, self.hostnamesfromcerts)) #filter out duplicates + return self.hostnamesfromcerts except Exception as e: - print("Error occurred: " + str(e)) + print("Error occurred in the Censys module: certificate hostname parser: " + str(e)) def search_ipaddresses(self): try: - ipaddresslist = self.soup.findAll('a', 'SearchResult__title-text') + ipaddresslist = self.souphosts.findAll('a','SearchResult__title-text') for ipaddressitem in ipaddresslist: self.ipaddresses.append(ipaddressitem.text.strip()) return self.ipaddresses except Exception as e: - print("Error occurred: " + str(e)) + print("Error occurred in the Censys module: IP address parser: " + str(e)) - def search_numberofpages(self): + def search_numberofpageshosts(self): try: - items = self.soup.findAll(href=re.compile("page")) + items = self.souphosts.findAll(href=re.compile("page")) for item in items: - if (item.text != 'next'): # to filter out pagination - self.numberofpages += 1 - return self.numberofpages + if (item.text !='next'): #to filter out pagination + self.numberofpageshosts+=1 + return self.numberofpageshosts except Exception as e: - print("Error occurred: " + str(e)) + print("Error occurred in the Censys module IP search: page parser: " + str(e)) + + def search_numberofpagescerts(self): + try: + items = self.soupcerts.findAll(href=re.compile("page")) + for item in items: + if (item.text != 'next'): #to filter out pagination + self.numberofpagescerts += 1 + return self.numberofpagescerts + except Exception as e: + print("Error occurred in the Censys module certificate search: page parser: " + str(e)) + + + diff --git a/discovery/censys.py b/discovery/censys.py index 55ab916c..49374778 100644 --- a/discovery/censys.py +++ b/discovery/censys.py @@ -7,48 +7,89 @@ class search_censys: def __init__(self, word): self.word = word - self.url = "" + self.urlhost = "" + self.urlcert = "" self.page = "" - self.results = "" - self.total_results = "" + self.resultshosts = "" + self.resultcerts = "" + self.total_resultshosts = "" + self.total_resultscerts = "" self.server = "censys.io" - - def do_search(self): + self.ips = [] + self.hostnamesall = [] + + def do_searchhosturl(self): try: - headers = {'user-agent': getUserAgent(), 'Accept': '*/*', 'Referer': self.url} - response = requests.get(self.url, headers=headers) - self.results = response.text - self.total_results += self.results + headers = {'user-agent': getUserAgent(), 'Accept':'*/*','Referer': self.urlhost} + responsehost = requests.get(self.urlhost, headers=headers) + self.resultshosts = responsehost.text + self.total_resultshosts += self.resultshosts except Exception as e: - print(e) + print("Error occurred in the Censys module downloading pages from Censys - IP search: " + str(e)) + + def do_searchcertificateurl(self): + try: + headers = {'user-agent': getUserAgent(), 'Accept':'*/*','Referer': self.urlcert} + responsecert = requests.get(self.urlcert, headers=headers) + self.resultcerts = responsecert.text + self.total_resultscerts += self.resultcerts + except Exception as e: + print("Error occurred in the Censys module downloading pages from Censys - certificates search: " + str(e)) def process(self): - self.url = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=1" - self.do_search() - self.counter = 2 - pages = censysparser.parser(self) - totalpages = pages.search_numberofpages() - while self.counter <= totalpages: - try: - self.page = str(self.counter) - self.url = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page) - print("\t -Searching Censys results page " + self.page + "...") - self.do_search() - time.sleep(getDelay()) - except Exception as e: - print("Error occurred: " + str(e)) - self.counter += 1 + try: + self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=1" + self.urlcert = "https://"+ self.server + "/certificates/_search?q=" + str(self.word) + "&page=1" + self.do_searchhosturl() + self.do_searchcertificateurl() + counter = 2 + pages = censysparser.parser(self) + totalpages = pages.search_numberofpageshosts() + while counter <= totalpages: + try: + self.page =str(counter) + self.urlhost = "https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page) + print("\tSearching Censys IP results page " + self.page + "...") + self.do_searchhosturl() + counter+= 1 + except Exception as e: + print("Error occurred in the Censys module requesting the pages: " + str(e)) + counter = 2 + totalpages = pages.search_numberofpagescerts() + while counter <= totalpages: + try: + self.page = str(counter) + self.urlhost = "https://" + self.server + "/certificates/_search?q=" + str(self.word) + "&page=" + str(self.page) + print("\tSearching Censys certificates results page " + self.page + "...") + self.do_searchcertificateurl() + counter += 1 + except Exception as e: + print("Error occurred in the Censys module requesting the pages: " + str(e)) + except Exception as e: + print("Error occurred in the main Censys module: " + str(e)) def get_hostnames(self): try: - hostnames = censysparser.parser(self) - return hostnames.search_hostnames() + ips = self.get_ipaddresses() + headers = {'user-agent': getUserAgent(), 'Accept':'*/*','Referer': self.urlcert} + response = requests.post("https://censys.io/ipv4/getdns", json={"ips": ips}, headers=headers) + responsejson = response.json() + for key, jdata in responsejson.items(): + if jdata is not None: + self.hostnamesall.append(jdata) + else: + pass + hostnamesfromcerts = censysparser.parser(self) + self.hostnamesall.extend(hostnamesfromcerts.search_hostnamesfromcerts()) + return self.hostnamesall except Exception as e: - print("Error occurred: " + str(e)) + print("Error occurred in the Censys module - hostname search: " + str(e)) def get_ipaddresses(self): try: ips = censysparser.parser(self) - return ips.search_ipaddresses() + self.ips = ips.search_ipaddresses() + return self.ips except Exception as e: - print("Error occurred: " + str(e)) + print("Error occurred in the main Censys module - IP address search: " + str(e)) + diff --git a/theHarvester.py b/theHarvester.py index 28e5d811..aeb47079 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -412,11 +412,18 @@ def start(argv): from discovery import censys search = censys.search_censys(word) search.process() - all_ip = search.get_ipaddresses() - all_hosts = search.get_hostnames() + ips = search.get_ipaddresses() + setips = set(ips) + uniqueips = list(setips) #remove duplicates + all_ip.extend(uniqueips) + hosts = search.get_hostnames() + sethosts = set(hosts) + uniquehosts = list(sethosts) #remove duplicates + all_hosts.extend(uniquehosts) db = stash.stash_manager() - db.store_all(word, all_ip, 'ip', 'censys') - db.store_all(word, all_hosts, 'host', 'censys') + db.store_all(word,uniquehosts,'host','censys') + db.store_all(word,uniqueips,'ip','censys') + print("[-] Searching in CRTSH server..") search = crtsh.search_crtsh(word) @@ -485,7 +492,9 @@ def start(argv): search.process() emails = search.get_emails() hosts = search.get_hostnames() - all_hosts.extend(hosts) + sethosts = set(hosts) + uniquehosts = list(sethosts) #remove duplicates + all_hosts.extend(uniquehosts) db = stash.stash_manager() db.store_all(word, all_hosts, 'host', 'PGP') all_emails.extend(emails) @@ -544,6 +553,7 @@ def start(argv): else: print("\033[1;33;40m \n[+] IP addresses found in search engines:") print("------------------------------------") + print("Total IP addresses: "+ str(len(all_ip)) + "\n") for i in all_ip: print(i) print("\n\n[+] Emails found:") @@ -564,6 +574,7 @@ def start(argv): if all_emails == []: print("No emails found.") else: + print("Total emails: "+ str(len(all_emails)) + "\n") print(("\n".join(all_emails))) print("\033[1;33;40m \n[+] Hosts found in search engines:") @@ -574,6 +585,8 @@ def start(argv): total = len(all_hosts) print(("\nTotal hosts: " + str(total) + "\n")) all_hosts = sorted(set(all_hosts)) + for host in all_hosts: + print(host) print("\033[94m[-] Resolving hostnames IPs...\033[1;33;40m \n ") full_host = hostchecker.Checker(all_hosts) full = full_host.check()