From 6b426a4c72d989a662593292d33460eeb58258f3 Mon Sep 17 00:00:00 2001 From: jzold Date: Sun, 9 Dec 2018 17:44:28 +0000 Subject: [PATCH 1/4] Add cymon search engine --- discovery/cymon.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 discovery/cymon.py diff --git a/discovery/cymon.py b/discovery/cymon.py new file mode 100644 index 00000000..503be0a6 --- /dev/null +++ b/discovery/cymon.py @@ -0,0 +1,46 @@ +import random +import requests +import cymonparser + +class search_cymon: + + def __init__(self, word): + self.word = word + self.url = "" + self.results = "" + self.server = "cymon.io" + self.userAgent = ["(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36" + ,("Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36"), + ("Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254"), + "Mozilla/5.0 (SMART-TV; X11; Linux armv7l) AppleWebKit/537.42 (KHTML, like Gecko) Chromium/25.0.1349.2 Chrome/25.0.1349.2 Safari/537.42" + ,"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 OPR/43.0.2442.991" + ,"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 OPR/48.0.2685.52" + ,"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko" + ,"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko" + ,"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)"] + + def do_search(self): + try: + headers = {'user-agent': random.choice(self.userAgent),'Accept':'*/*','Referer':self.url} + response = requests.get(self.url, headers=headers) + self.results = response.content + except Exception as e: + print(e) + + def process(self): + try: + self.url="https://" + self.server + "/domain/" + str(self.word) + print("\tSearching Cymon results...") + self.do_search() + except Exception as e: + print("Error occurred: " + str(e)) + + def get_ipaddresses(self): + try: + ips = cymonparser.parser(self) + return ips.search_ipaddresses() + except Exception as e: + print("Error occurred: " + str(e)) From b278889b56cf490d0841f548e4a2dcd4e4ba8c60 Mon Sep 17 00:00:00 2001 From: jzold Date: Sun, 9 Dec 2018 17:45:36 +0000 Subject: [PATCH 2/4] add cymon search engine --- cymonparser.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 cymonparser.py diff --git a/cymonparser.py b/cymonparser.py new file mode 100644 index 00000000..d83167c0 --- /dev/null +++ b/cymonparser.py @@ -0,0 +1,18 @@ +from bs4 import BeautifulSoup +import re + +class parser: + + def __init__(self, results): + self.results = results + self.ipaddresses = [] + self.soup = BeautifulSoup(results.results,features="html.parser") + + def search_ipaddresses(self): + try: + tags = self.soup.findAll('td') + allip = re.findall( r'[0-9]+(?:\.[0-9]+){3}',str(tags)) + self.ipaddresses = set(allip) + return self.ipaddresses + except Exception as e: + print("Error occurred: " + str(e)) From fef88abc1d845d2194f7c42d1ab8539428ce36f1 Mon Sep 17 00:00:00 2001 From: jzold Date: Sun, 9 Dec 2018 17:51:48 +0000 Subject: [PATCH 3/4] adding cymon --- theHarvester.py | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/theHarvester.py b/theHarvester.py index 6c471bcb..47c14b35 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -133,7 +133,7 @@ def start(argv): dnstld = True elif opt == '-b': engines = set(arg.split(',')) - supportedengines = set(["baidu","bing","crtsh","censys","bingapi","dogpile","google","googleCSE","virustotal","threatcrowd","googleplus","google-profiles",'google-certificates',"linkedin","pgp","twitter","trello","vhost","yahoo","netcraft","hunter","all"]) + supportedengines = set(["baidu","bing","crtsh","censys","cymon","bingapi","dogpile","google","googleCSE","virustotal","threatcrowd","googleplus","google-profiles",'google-certificates',"linkedin","pgp","twitter","trello","vhost","yahoo","netcraft","hunter","all"]) if set(engines).issubset(supportedengines): print("found supported engines") print(("[-] Starting harvesting process for domain: " + word + "\n")) @@ -317,7 +317,17 @@ def start(argv): db=stash.stash_manager() db.store_all(word,all_hosts,'host','censys') db.store_all(word,all_ip,'ip','censys') - + + elif engineitem == "cymon": + print("[-] Searching in Cymon:") + from discovery import cymon + #import locally or won't work + search = cymon.search_cymon(word) + search.process() + all_emails = [] + all_ip = search.get_ipaddresses() + db = stash.stash_manager() + db.store_all(word,all_ip,'ip','cymon') elif engineitem == "trello": print("[-] Searching in Trello:") @@ -426,17 +436,31 @@ def start(argv): search.process() domains = search.get_domains() all_hosts.extend(domains) + + print("[-] Searching in Cymon:") + from discovery import cymon + search = cymon.search_cymon(word) + search.process() + all_emails = [] + all_ip = search.get_ipaddresses() + db = stash.stash_manager() + db.store_all(word,all_ip,'ip','cymon') - - + print("[-] Searching in Censys:") + from discovery import censys + search = censys.search_censys(word) + search.process() + all_emails = [] + all_ip = search.get_ipaddresses() + all_hosts = search.get_hostnames() + db = stash.stash_manager() + db.store_all(word,all_ip,'ip','censys') + db.store_all(word,all_hosts,'host','censys') else: - #if engine not in ("baidu", "bing", "crtsh","bingapi","dogpile","google", "googleCSE","virustotal","threatcrowd", "googleplus", "google-profiles","linkedin", "pgp", "twitter", "vhost", "yahoo","netcraft","all"): usage() - print("Invalid search engine, try with: baidu, bing, bingapi, crtsh, dogpile, google, googleCSE, virustotal, netcraft, googleplus, google-profiles, linkedin, pgp, twitter, vhost, yahoo, hunter, all") + print("Invalid search engine, try with: baidu, bing, bingapi, crtsh, censys, cymon, dogpile, google, googleCSE, virustotal, netcraft, googleplus, google-profiles, linkedin, pgp, twitter, vhost, yahoo, hunter, all") sys.exit() - #else: - # pass #Results############################################################ print("\n\033[1;32;40mHarvesting results") From 703111b660e455f2ca9a49febff3ba6550b10d21 Mon Sep 17 00:00:00 2001 From: jzold Date: Sun, 9 Dec 2018 18:45:50 +0000 Subject: [PATCH 4/4] printing cymon search results --- theHarvester.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/theHarvester.py b/theHarvester.py index 47c14b35..4114c37f 100755 --- a/theHarvester.py +++ b/theHarvester.py @@ -325,6 +325,7 @@ def start(argv): search = cymon.search_cymon(word) search.process() all_emails = [] + all_hosts = [] all_ip = search.get_ipaddresses() db = stash.stash_manager() db.store_all(word,all_ip,'ip','cymon') @@ -464,6 +465,13 @@ def start(argv): #Results############################################################ print("\n\033[1;32;40mHarvesting results") + if (len(all_ip) == 0): + print("No IP addresses found") + else: + print("\033[1;33;40m \n[+] IP addresses found in search engines:") + print("------------------------------------") + for i in all_ip: + print(i) print("\n\n[+] Emails found:") print("------------------") @@ -479,7 +487,6 @@ def start(argv): print('No hosts found as all_hosts is not defined.') sys.exit() - if all_emails == []: print("No emails found") else: @@ -504,7 +511,7 @@ def start(argv): pass else: host_ip.append(ip.lower()) - + #DNS Brute force#################################################### dnsres = [] if dnsbrute == True: