Merge pull request #112 from jzold/censys

Censys page search fix

discovery/censys.py

@@ -35,13 +35,17 @@ class search_censys:
         except Exception as e:
             print(e)
 
-    def process(self,morepage):
-        self.counter=1
-        while self.counter <= morepage and self.counter <= 10:
+    def process(self):
+        self.url="https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=1"
+        self.do_search()
+        self.counter=2
+        pages = censysparser.parser(self)
+        totalpages = pages.search_numberofpages()
+        while self.counter <= totalpages:
             try:
                 self.page =str(self.counter)
                 self.url="https://" + self.server + "/ipv4/_search?q=" + str(self.word) + "&page=" + str(self.page)                   
-                print("\tSearching Censys results..")
+                print("\tSearching Censys results page " + self.page + "...")
                 self.do_search()
             except Exception as e:
                 print("Error occurred: " + str(e))
@@ -60,11 +64,4 @@ class search_censys:
             return ips.search_ipaddresses()
         except Exception as e:
             print("Error occurred: " + str(e))
-    
-    def get_totalnumberofpages(self):
-        try:
-            pages = censysparser.parser(self)
-            return pages.search_numberofpages()
-        except Exception as e:
-            print("Error occurred: " + str(e))
 

theHarvester.py

@@ -46,7 +46,7 @@ def usage():
 
     print("Usage: theharvester options \n")
     print("       -d: Domain to search or company name")
-    print("""       -b: data source: baidu, bing, bingapi, crtsh, dogpile,
+    print("""       -b: data source: baidu, bing, bingapi, cenysy, crtsh, dogpile,
                         google, google-certificates, googleCSE, googleplus, google-profiles,
                         hunterio, linkedin, netcraft, pgp, threatcrowd,
                         twitter, vhost, virustotal, yahoo, all""")
@@ -311,10 +311,13 @@ def start(argv):
                         #import locally or won't work
                         search = censys.search_censys(word)
                         search.process()
-                        totalnumberofpages = search.get_totalnumberofpages()
                         all_emails = []
                         all_ip = search.get_ipaddresses()
                         all_hosts = search.get_hostnames()
+                        db=stash.stash_manager()
+                        db.store_all(word,all_hosts,'host','censys')
+                        db.store_all(word,all_ip,'ip','censys')
+
                     
                     elif engineitem == "trello":
                         print("[-] Searching in Trello:")