Merge pull request #12 from tatanus/master

added new search engines

discovery/__init__.py

@@ -12,4 +12,6 @@ __all__ = ["bingsearch",
            "jigsaw",
            "twittersearch",
            "dogpilesearch",
+           "baidusearch",
+           "yahoosearch",
            "googleCSE"]

discovery/baidusearch.py

@@ -0,0 +1,44 @@
+import httplib
+import myparser
+import time
+import sys
+
+
+class search_baidu:
+
+    def __init__(self, word, limit):
+        self.word = word
+        self.total_results = ""
+        self.server = "www.baidu.com"
+        self.hostname = "www.baidu.com"
+        self.userAgent = "(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6"
+        self.limit = limit
+        self.counter = 0
+
+    def do_search(self):
+        h = httplib.HTTP(self.server)
+
+        h.putrequest('GET', "/s?wd=%40" + self.word
+                     + "&pn=" + str(self.counter))
+        h.putheader('Host', self.hostname)
+        h.putheader('User-agent', self.userAgent)
+        h.endheaders()
+        returncode, returnmsg, headers = h.getreply()
+
+        self.total_results += h.getfile().read()
+
+    def process(self):
+        while self.counter <= self.limit and self.counter <= 1000:
+            self.do_search()
+            time.sleep(1)
+
+            print "\tSearching " + str(self.counter) + " results..."
+            self.counter += 10
+
+    def get_emails(self):
+        rawres = myparser.parser(self.total_results, self.word)
+        return rawres.emails()
+
+    def get_hostnames(self):
+        rawres = myparser.parser(self.total_results, self.word)
+        return rawres.hostnames()

discovery/yahoosearch.py

@@ -0,0 +1,44 @@
+import httplib
+import myparser
+import time
+import sys
+
+
+class search_yahoo:
+
+    def __init__(self, word, limit):
+        self.word = word
+        self.total_results = ""
+        self.server = "search.yahoo.com"
+        self.hostname = "search.yahoo.com"
+        self.userAgent = "(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6"
+        self.limit = limit
+        self.counter = 0
+
+    def do_search(self):
+        h = httplib.HTTP(self.server)
+
+        h.putrequest('GET', "/search?p=\"%40" + self.word
+                     + "\"&b=" + str(self.counter) + "&pz=10")
+        h.putheader('Host', self.hostname)
+        h.putheader('User-agent', self.userAgent)
+        h.endheaders()
+        returncode, returnmsg, headers = h.getreply()
+
+        self.total_results += h.getfile().read()
+
+    def process(self):
+        while self.counter <= self.limit and self.counter <= 1000:
+            self.do_search()
+            time.sleep(1)
+
+            print "\tSearching " + str(self.counter) + " results..."
+            self.counter += 10
+
+    def get_emails(self):
+        rawres = myparser.parser(self.total_results, self.word)
+        return rawres.emails()
+
+    def get_hostnames(self):
+        rawres = myparser.parser(self.total_results, self.word)
+        return rawres.hostnames()

theHarvester.py

@@ -46,6 +46,9 @@ def usage():
                         jigsaw
                         twitter
                         googleplus
+                        dogpilesearch
+                        yahoo
+                        baidu
                         all\n"""
     print "       -s: Start in result number X (default 0)"
     print "       -v: Verify host name via dns resolution and search for virtual hosts"
@@ -107,10 +110,10 @@ def start(argv):
             dnstld = True
         elif opt == '-b':
             engine = arg
-            if engine not in ("google","googleCSE" , "linkedin", "pgp", "all", "google-profiles", "bing", "bing_api", 
-                              "yandex", "people123", "jigsaw", "dogpilesearch","twitter","googleplus"):
+            if engine not in ("google","googleCSE" , "linkedin", "pgp", "all", "google-profiles", "bing", "bingapi", 
+                              "yandex", "people123", "jigsaw", "dogpilesearch", "twitter", "googleplus", "yahoo", "baidu"):
                 usage()
-                print "Invalid search engine, try with: bing, google, linkedin, pgp, jigsaw, bing_api, people123, google-profiles,dogpilesearch,twitter,googleplus"
+                print "Invalid search engine, try with: bing, google, linkedin, pgp, jigsaw, bingapi, people123, google-profiles, dogpilesearch, twitter, googleplus, yahoo, baidu"
                 sys.exit()
             else:
                 pass
@@ -183,6 +186,7 @@ def start(argv):
         for user in people:
             print user
         sys.exit()
+
     elif engine == "dogpilesearch":
         print "[-] Searching in Dogpilesearch.."
         search = dogpilesearch.search_dogpile(word, limit)
@@ -190,6 +194,20 @@ def start(argv):
         all_emails = search.get_emails()
         all_hosts = search.get_hostnames()
 
+    elif engine == "yahoo":
+        print "[-] Searching in Yahoo.."
+        search = yahoosearch.search_yahoo(word, limit)
+        search.process()
+        all_emails = search.get_emails()
+        all_hosts = search.get_hostnames()
+
+    elif engine == "baidu":
+        print "[-] Searching in Baidu.."
+        search = baidusearch.search_baidu(word, limit)
+        search.process()
+        all_emails = search.get_emails()
+        all_hosts = search.get_hostnames()
+
     elif engine == "googleplus":
         print "[-] Searching in Google+ .."
         search = googleplussearch.search_googleplus(word, limit)