From 3f1b9d7b132bb067505f30945247c11ec1921563 Mon Sep 17 00:00:00 2001 From: "H. Kamran" Date: Sat, 1 Dec 2018 19:38:08 -0800 Subject: [PATCH 1/5] Replace "Request" with "Requests", the actual module name --- theHarvester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theHarvester.py b/theHarvester.py index 47f6322f..1d058bde 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -10,7 +10,7 @@ import time try: import requests except: - print("Request library not found, please install it before proceeding\n") + print("Requests library not found, please install it before proceeding\n") sys.exit() try: From 505a71781d101e95207708a5f26476fba0cc4a95 Mon Sep 17 00:00:00 2001 From: jzold Date: Thu, 6 Dec 2018 16:32:25 +0000 Subject: [PATCH 2/5] fix: optimize search with minimum number of page queries --- discovery/censys.py | 19 ++++++++----------- theHarvester.py | 5 ++++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/discovery/censys.py b/discovery/censys.py index 855a172c..78604928 100644 --- a/discovery/censys.py +++ b/discovery/censys.py @@ -35,13 +35,17 @@ class search_censys: except Exception as e: print(e) - def process(self,morepage): - self.counter=1 - while self.counter <= morepage and self.counter <= 10: + def process(self): + self.url="https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=1" + self.do_search() + self.counter=2 + pages = censysparser.parser(self) + totalpages = pages.search_numberofpages() + while self.counter <= totalpages: try: self.page =str(self.counter) self.url="https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page) - print("\tSearching Censys results..") + print("\tSearching Censys results page " + self.page + "...") self.do_search() except Exception as e: print("Error occurred: " + str(e)) @@ -60,11 +64,4 @@ class search_censys: return ips.search_ipaddresses() except Exception as e: print("Error occurred: " + str(e)) - - def get_totalnumberofpages(self): - try: - pages = censysparser.parser(self) - return pages.search_numberofpages() - except Exception as e: - print("Error occurred: " + str(e)) diff --git a/theHarvester.py b/theHarvester.py index 47f6322f..6352dbec 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -311,10 +311,13 @@ def start(argv): #import locally or won't work search = censys.search_censys(word) search.process() - totalnumberofpages = search.get_totalnumberofpages() all_emails = [] all_ip = search.get_ipaddresses() all_hosts = search.get_hostnames() + db=stash.stash_manager() + db.store_all(word,all_hosts,'host','censys') + db.store_all(word,all_ip,'ip','censys') + elif engineitem == "trello": print("[-] Searching in Trello:") From 4cfd5b65cc2312bda11a5b04ed170c52966673f1 Mon Sep 17 00:00:00 2001 From: jzold Date: Thu, 6 Dec 2018 17:47:14 +0000 Subject: [PATCH 3/5] added "censys" to theHarvester usage help --- theHarvester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theHarvester.py b/theHarvester.py index 6352dbec..6c471bcb 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -46,7 +46,7 @@ def usage(): print("Usage: theharvester options \n") print(" -d: Domain to search or company name") - print(""" -b: data source: baidu, bing, bingapi, crtsh, dogpile, + print(""" -b: data source: baidu, bing, bingapi, cenysy, crtsh, dogpile, google, google-certificates, googleCSE, googleplus, google-profiles, hunterio, linkedin, netcraft, pgp, threatcrowd, twitter, vhost, virustotal, yahoo, all""") From 887776e37ad668527cb0ed5c50cd80a463fa9088 Mon Sep 17 00:00:00 2001 From: Laramies Date: Tue, 11 Dec 2018 22:07:48 +0100 Subject: [PATCH 4/5] Minor fixes XML save order --- README.md | 2 +- theHarvester.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d1c8fbef..832f4598 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ https://github.com/laramies/theHarvester Thanks: ------- * Matthew Brown @NotoriousRebel -* Janos Zold @Jzod +* Janos Zold @Jzold * John Matherly - SHODAN project * Lee Baird for suggestions and bugs reporting * Ahmed Aboul Ela - subdomain names dictionary (big and small) diff --git a/theHarvester.py b/theHarvester.py index 47f6322f..11f47c2a 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -150,6 +150,7 @@ def start(argv): db.store(word,x,'host','google') except Exception as e: print(e) + if engineitem == "netcraft": print("[-] Searching in Netcraft:") @@ -244,6 +245,10 @@ def start(argv): search.process() all_emails = search.get_emails() all_hosts = search.get_hostnames() + db=stash.stash_manager() + db.store_all(word,all_hosts,'host','yahoo') + db.store_all(word,all_emails,'emails','yahoo') + elif engineitem == "baidu": print("[-] Searching in Baidu..") @@ -251,6 +256,9 @@ def start(argv): search.process() all_emails = search.get_emails() all_hosts = search.get_hostnames() + db=stash.stash_manager() + db.store_all(word,all_hosts,'host','baidu') + db.store_all(word,all_emails,'emails','baidu') elif engineitem == "googleplus": print("[-] Searching in Google+ ..") @@ -652,13 +660,13 @@ def start(argv): for x in full: x = x.split(":") if len(x) == 2: - file.write('' + '' + x[0] + '' + x[1] + '' + '') + file.write('' + '' + x[1] + '' + x[0] + '' + '') else: file.write('' + x + '') for x in vhost: x = x.split(":") if len(x) == 2: - file.write('' + '' + x[0] + '' + x[1] + '' + '') + file.write('' + '' + x[1] + '' + x[0] + '' + '') else: file.write('' + x + '') From 681ed17233d6c1aaa4e11e97f9d85f41ebcc6123 Mon Sep 17 00:00:00 2001 From: Laramies Date: Tue, 11 Dec 2018 22:10:08 +0100 Subject: [PATCH 5/5] Vesion update --- .gitignore | 2 ++ README.md | 2 +- theHarvester.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 11e52209..6234e981 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ stash.sqlite *.sqlite .vscode venv +.xml +.html diff --git a/README.md b/README.md index 832f4598..a1607c1b 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ * | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | * * \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| * * * -* TheHarvester Ver. 3.0.2 * +* TheHarvester Ver. 3.0.4 * * Coded by Christian Martorella * * Edge-Security Research * * cmartorella@edge-security.com * diff --git a/theHarvester.py b/theHarvester.py index ac6c8371..6622ff2d 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -30,7 +30,7 @@ print("* | __| '_ \ / _ \ / /_/ / _` | '__\ \ / / _ \/ __| __/ _ \ '__| *") print("* | |_| | | | __/ / __ / (_| | | \ V / __/\__ \ || __/ | *") print("* \__|_| |_|\___| \/ /_/ \__,_|_| \_/ \___||___/\__\___|_| *") print("* *") -print("* theHarvester Ver. 3.0.3 *") +print("* theHarvester Ver. 3.0.4 *") print("* Coded by Christian Martorella *") print("* Edge-Security Research *") print("* cmartorella@edge-security.com *")