From 505a71781d101e95207708a5f26476fba0cc4a95 Mon Sep 17 00:00:00 2001 From: jzold Date: Thu, 6 Dec 2018 16:32:25 +0000 Subject: [PATCH 1/2] fix: optimize search with minimum number of page queries --- discovery/censys.py | 19 ++++++++----------- theHarvester.py | 5 ++++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/discovery/censys.py b/discovery/censys.py index 855a172c..78604928 100644 --- a/discovery/censys.py +++ b/discovery/censys.py @@ -35,13 +35,17 @@ class search_censys: except Exception as e: print(e) - def process(self,morepage): - self.counter=1 - while self.counter <= morepage and self.counter <= 10: + def process(self): + self.url="https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=1" + self.do_search() + self.counter=2 + pages = censysparser.parser(self) + totalpages = pages.search_numberofpages() + while self.counter <= totalpages: try: self.page =str(self.counter) self.url="https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page) - print("\tSearching Censys results..") + print("\tSearching Censys results page " + self.page + "...") self.do_search() except Exception as e: print("Error occurred: " + str(e)) @@ -60,11 +64,4 @@ class search_censys: return ips.search_ipaddresses() except Exception as e: print("Error occurred: " + str(e)) - - def get_totalnumberofpages(self): - try: - pages = censysparser.parser(self) - return pages.search_numberofpages() - except Exception as e: - print("Error occurred: " + str(e)) diff --git a/theHarvester.py b/theHarvester.py index 47f6322f..6352dbec 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -311,10 +311,13 @@ def start(argv): #import locally or won't work search = censys.search_censys(word) search.process() - totalnumberofpages = search.get_totalnumberofpages() all_emails = [] all_ip = search.get_ipaddresses() all_hosts = search.get_hostnames() + db=stash.stash_manager() + db.store_all(word,all_hosts,'host','censys') + db.store_all(word,all_ip,'ip','censys') + elif engineitem == "trello": print("[-] Searching in Trello:") From 4cfd5b65cc2312bda11a5b04ed170c52966673f1 Mon Sep 17 00:00:00 2001 From: jzold Date: Thu, 6 Dec 2018 17:47:14 +0000 Subject: [PATCH 2/2] added "censys" to theHarvester usage help --- theHarvester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theHarvester.py b/theHarvester.py index 6352dbec..6c471bcb 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -46,7 +46,7 @@ def usage(): print("Usage: theharvester options \n") print(" -d: Domain to search or company name") - print(""" -b: data source: baidu, bing, bingapi, crtsh, dogpile, + print(""" -b: data source: baidu, bing, bingapi, cenysy, crtsh, dogpile, google, google-certificates, googleCSE, googleplus, google-profiles, hunterio, linkedin, netcraft, pgp, threatcrowd, twitter, vhost, virustotal, yahoo, all""")