Merge branch 'master' into report

cymonparser.py

@@ -0,0 +1,18 @@
+from bs4 import BeautifulSoup
+import re
+
+class parser:
+
+    def __init__(self, results):
+        self.results = results
+        self.ipaddresses = []
+        self.soup = BeautifulSoup(results.results,features="html.parser")
+
+    def search_ipaddresses(self):
+        try:
+            tags = self.soup.findAll('td')
+            allip = re.findall( r'[0-9]+(?:\.[0-9]+){3}',str(tags))
+            self.ipaddresses = set(allip)
+            return self.ipaddresses
+        except Exception as e:
+            print("Error occurred: " + str(e))

discovery/cymon.py

@@ -0,0 +1,46 @@
+import random
+import requests
+import cymonparser
+
+class search_cymon:
+
+    def __init__(self, word):
+        self.word = word
+        self.url = ""
+        self.results = ""
+        self.server = "cymon.io"
+        self.userAgent = ["(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"
+          ,("Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) " +
+          "AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36"),
+          ("Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) " +
+          "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254"),
+          "Mozilla/5.0 (SMART-TV; X11; Linux armv7l) AppleWebKit/537.42 (KHTML, like Gecko) Chromium/25.0.1349.2 Chrome/25.0.1349.2 Safari/537.42"
+          ,"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 OPR/43.0.2442.991"
+          ,"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 OPR/48.0.2685.52"
+          ,"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"
+          ,"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko"
+          ,"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)"]
+        
+    def do_search(self):
+        try:
+            headers = {'user-agent': random.choice(self.userAgent),'Accept':'*/*','Referer':self.url}
+            response = requests.get(self.url, headers=headers)
+            self.results = response.content
+        except Exception as e:
+            print(e)
+
+    def process(self):
+        try:
+            self.url="https://" + self.server + "/domain/" + str(self.word)
+            print("\tSearching Cymon results...")
+            self.do_search()
+        except Exception as e:
+            print("Error occurred: " + str(e))
+
+    def get_ipaddresses(self):
+        try:
+            ips = cymonparser.parser(self)
+            return ips.search_ipaddresses()
+        except Exception as e:
+            print("Error occurred: " + str(e))

theHarvester.py

@@ -133,7 +133,7 @@ def start(argv):
             dnstld = True
         elif opt == '-b':
             engines = set(arg.split(','))
-            supportedengines = set(["baidu","bing","crtsh","censys","bingapi","dogpile","google","googleCSE","virustotal","threatcrowd","googleplus","google-profiles",'google-certificates',"linkedin","pgp","twitter","trello","vhost","yahoo","netcraft","hunter","all"])
+            supportedengines = set(["baidu","bing","crtsh","censys","cymon","bingapi","dogpile","google","googleCSE","virustotal","threatcrowd","googleplus","google-profiles",'google-certificates',"linkedin","pgp","twitter","trello","vhost","yahoo","netcraft","hunter","all"])
             if set(engines).issubset(supportedengines):
                 print("found supported engines")
                 print(("[-] Starting harvesting process for domain: " + word +  "\n"))
@@ -334,6 +334,19 @@ def start(argv):
                         db=stash.stash_manager()
                         db.store_all(word,all_hosts,'host','censys')
                         db.store_all(word,all_ip,'ip','censys')
+                        
+                    elif engineitem == "cymon":
+                        print("[-] Searching in Cymon:")
+                        from discovery import cymon
+                        #import locally or won't work
+                        search = cymon.search_cymon(word)
+                        search.process()
+                        all_emails = []
+                        all_hosts = []
+                        all_ip = search.get_ipaddresses()
+                        db = stash.stash_manager()
+                        db.store_all(word,all_ip,'ip','cymon')
+
                     
                     elif engineitem == "trello":
                         print("[-] Searching in Trello:")
@@ -449,17 +462,32 @@ def start(argv):
                         all_hosts.extend(domains)
                         db = stash.stash_manager()
                         db.store_all(word, all_hosts, 'host', 'google-certificates')
+                     
+                        print("[-] Searching in Censys:")
+                        from discovery import censys
+                        search = censys.search_censys(word)
+                        search.process()
+                        all_emails = []
+                        all_ip = search.get_ipaddresses()
+                        all_hosts = search.get_hostnames()
+                        db = stash.stash_manager()
+                        db.store_all(word,all_ip,'ip','censys')
+                        db.store_all(word,all_hosts,'host','censys')
 
             else:
-            #if engine not in ("baidu", "bing", "crtsh","bingapi","dogpile","google", "googleCSE","virustotal","threatcrowd", "googleplus", "google-profiles","linkedin", "pgp", "twitter", "vhost", "yahoo","netcraft","all"):
                 usage()
-                print("Invalid search engine, try with: baidu, bing, bingapi, crtsh, dogpile, google, googleCSE, virustotal, netcraft, googleplus, google-profiles, linkedin, pgp, twitter, vhost, yahoo, hunter, all")
+                print("Invalid search engine, try with: baidu, bing, bingapi, crtsh, censys, cymon, dogpile, google, googleCSE, virustotal, netcraft, googleplus, google-profiles, linkedin, pgp, twitter, vhost, yahoo, hunter, all")
                 sys.exit()
-            #else:
-            #    pass
 
     #Results############################################################
     print("\n\033[1;32;40mHarvesting results")
+    if (len(all_ip) == 0):
+        print("No IP addresses found")
+    else:
+        print("\033[1;33;40m \n[+] IP addresses found in search engines:")
+        print("------------------------------------")
+        for i in all_ip:
+            print(i)
     print("\n\n[+] Emails found:")
     print("------------------")
 
@@ -475,7 +503,6 @@ def start(argv):
         print('No hosts found as all_hosts is not defined.')
         sys.exit()
 
-
     if all_emails == []:
         print("No emails found")
     else:
@@ -500,9 +527,10 @@ def start(argv):
                     pass
                 else:
                     host_ip.append(ip.lower())
+
         db=stash.stash_manager()
         db.store_all(word,host_ip,'ip','DNS-resolver')
-
+    
     #DNS Brute force####################################################
     dnsres = []
     if dnsbrute == True: