From 9a6dce8e5b6d13fa9865147d48045fde84347087 Mon Sep 17 00:00:00 2001 From: Adam Compton Date: Wed, 25 Mar 2015 12:55:15 -0400 Subject: [PATCH] added new search engines --- discovery/__init__.py | 2 ++ discovery/baidusearch.py | 44 ++++++++++++++++++++++++++++++++++++++++ discovery/yahoosearch.py | 44 ++++++++++++++++++++++++++++++++++++++++ theHarvester.py | 24 +++++++++++++++++++--- 4 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 discovery/baidusearch.py create mode 100644 discovery/yahoosearch.py diff --git a/discovery/__init__.py b/discovery/__init__.py index 42a42444..4c6c920e 100644 --- a/discovery/__init__.py +++ b/discovery/__init__.py @@ -12,4 +12,6 @@ __all__ = ["bingsearch", "jigsaw", "twittersearch", "dogpilesearch", + "baidusearch", + "yahoosearch", "googleCSE"] diff --git a/discovery/baidusearch.py b/discovery/baidusearch.py new file mode 100644 index 00000000..9409800f --- /dev/null +++ b/discovery/baidusearch.py @@ -0,0 +1,44 @@ +import httplib +import myparser +import time +import sys + + +class search_baidu: + + def __init__(self, word, limit): + self.word = word + self.total_results = "" + self.server = "www.baidu.com" + self.hostname = "www.baidu.com" + self.userAgent = "(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6" + self.limit = limit + self.counter = 0 + + def do_search(self): + h = httplib.HTTP(self.server) + + h.putrequest('GET', "/s?wd=%40" + self.word + + "&pn=" + str(self.counter)) + h.putheader('Host', self.hostname) + h.putheader('User-agent', self.userAgent) + h.endheaders() + returncode, returnmsg, headers = h.getreply() + + self.total_results += h.getfile().read() + + def process(self): + while self.counter <= self.limit and self.counter <= 1000: + self.do_search() + time.sleep(1) + + print "\tSearching " + str(self.counter) + " results..." + self.counter += 10 + + def get_emails(self): + rawres = myparser.parser(self.total_results, self.word) + return rawres.emails() + + def get_hostnames(self): + rawres = myparser.parser(self.total_results, self.word) + return rawres.hostnames() diff --git a/discovery/yahoosearch.py b/discovery/yahoosearch.py new file mode 100644 index 00000000..e7b65055 --- /dev/null +++ b/discovery/yahoosearch.py @@ -0,0 +1,44 @@ +import httplib +import myparser +import time +import sys + + +class search_yahoo: + + def __init__(self, word, limit): + self.word = word + self.total_results = "" + self.server = "search.yahoo.com" + self.hostname = "search.yahoo.com" + self.userAgent = "(Mozilla/5.0 (Windows; U; Windows NT 6.0;en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6" + self.limit = limit + self.counter = 0 + + def do_search(self): + h = httplib.HTTP(self.server) + + h.putrequest('GET', "/search?p=\"%40" + self.word + + "\"&b=" + str(self.counter) + "&pz=10") + h.putheader('Host', self.hostname) + h.putheader('User-agent', self.userAgent) + h.endheaders() + returncode, returnmsg, headers = h.getreply() + + self.total_results += h.getfile().read() + + def process(self): + while self.counter <= self.limit and self.counter <= 1000: + self.do_search() + time.sleep(1) + + print "\tSearching " + str(self.counter) + " results..." + self.counter += 10 + + def get_emails(self): + rawres = myparser.parser(self.total_results, self.word) + return rawres.emails() + + def get_hostnames(self): + rawres = myparser.parser(self.total_results, self.word) + return rawres.hostnames() diff --git a/theHarvester.py b/theHarvester.py index 758cac1e..5ed017e5 100644 --- a/theHarvester.py +++ b/theHarvester.py @@ -46,6 +46,9 @@ def usage(): jigsaw twitter googleplus + dogpilesearch + yahoo + baidu all\n""" print " -s: Start in result number X (default 0)" print " -v: Verify host name via dns resolution and search for virtual hosts" @@ -107,10 +110,10 @@ def start(argv): dnstld = True elif opt == '-b': engine = arg - if engine not in ("google","googleCSE" , "linkedin", "pgp", "all", "google-profiles", "bing", "bing_api", - "yandex", "people123", "jigsaw", "dogpilesearch","twitter","googleplus"): + if engine not in ("google","googleCSE" , "linkedin", "pgp", "all", "google-profiles", "bing", "bingapi", + "yandex", "people123", "jigsaw", "dogpilesearch", "twitter", "googleplus", "yahoo", "baidu"): usage() - print "Invalid search engine, try with: bing, google, linkedin, pgp, jigsaw, bing_api, people123, google-profiles,dogpilesearch,twitter,googleplus" + print "Invalid search engine, try with: bing, google, linkedin, pgp, jigsaw, bingapi, people123, google-profiles, dogpilesearch, twitter, googleplus, yahoo, baidu" sys.exit() else: pass @@ -183,6 +186,7 @@ def start(argv): for user in people: print user sys.exit() + elif engine == "dogpilesearch": print "[-] Searching in Dogpilesearch.." search = dogpilesearch.search_dogpile(word, limit) @@ -190,6 +194,20 @@ def start(argv): all_emails = search.get_emails() all_hosts = search.get_hostnames() + elif engine == "yahoo": + print "[-] Searching in Yahoo.." + search = yahoosearch.search_yahoo(word, limit) + search.process() + all_emails = search.get_emails() + all_hosts = search.get_hostnames() + + elif engine == "baidu": + print "[-] Searching in Baidu.." + search = baidusearch.search_baidu(word, limit) + search.process() + all_emails = search.get_emails() + all_hosts = search.get_hostnames() + elif engine == "googleplus": print "[-] Searching in Google+ .." search = googleplussearch.search_googleplus(word, limit)