diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index 47893985..28883ed5 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -51,10 +51,6 @@ jobs: run: | python theHarvester.py -d yale.edu -b dnsdumpster - - name: Run theHarvester module Dogplie - run: | - python theHarvester.py -d yale.edu -b dogpile - - name: Run theHarvester module DuckDuckGo run: | python theHarvester.py -d yale.edu -b duckduckgo diff --git a/README.md b/README.md index 0da39ab3..58b3659f 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,6 @@ Passive: * dnsdumpster: DNSdumpster search engine - https://dnsdumpster.com -* dogpile: Dogpile search engine - www.dogpile.com - * duckduckgo: DuckDuckGo search engine - www.duckduckgo.com * exalead: a Meta search engine - www.exalead.com/search @@ -95,18 +93,17 @@ Documentation to setup API keys can be found at - https://github.com/laramies/th * bing * github -* hunter +* hunter - limited to 10 on the free plan so you will ned to do -l 10 switch * intelx * pentesttools * securityTrails * shodan -* spyse +* spyse - need to have a paid account be able to use the api now Install and dependencies: ------------------------- * Python 3.7+ -* python3 -m pip install pipenv * https://github.com/laramies/theHarvester/wiki/Installation diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 8c331bf7..13a71924 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -33,7 +33,7 @@ async def start(): parser.add_argument('-c', '--dns-brute', help='Perform a DNS brute force on the domain.', default=False, action='store_true') parser.add_argument('-f', '--filename', help='Save the results to an HTML and/or XML file.', default='', type=str) parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, bufferoverun, certspotter, crtsh, dnsdumpster, - dogpile, duckduckgo, exalead, github-code, google, + duckduckgo, exalead, github-code, google, hackertarget, hunter, intelx, linkedin, linkedin_links, netcraft, otx, pentesttools, qwant, rapiddns, securityTrails, spyse, sublist3r, suip, threatcrowd, threatminer, trello, twitter, urlscan, virustotal, yahoo''') @@ -212,14 +212,6 @@ async def start(): except Exception as e: print(f'\033[93m[!] An error occurred with dnsdumpster: {e} \033[0m') - elif engineitem == 'dogpile': - try: - from theHarvester.discovery import dogpilesearch - dogpile_search = dogpilesearch.SearchDogpile(word, limit) - stor_lst.append(store(dogpile_search, engineitem, store_host=True, store_emails=True)) - except Exception as e: - print(f'\033[93m[!] An error occurred with Dogpile: {e} \033[0m') - elif engineitem == 'duckduckgo': from theHarvester.discovery import duckduckgosearch duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(word, limit) diff --git a/theHarvester/discovery/dogpilesearch.py b/theHarvester/discovery/dogpilesearch.py deleted file mode 100644 index 144791a6..00000000 --- a/theHarvester/discovery/dogpilesearch.py +++ /dev/null @@ -1,37 +0,0 @@ -from theHarvester.lib.core import * -from theHarvester.parsers import myparser - - -class SearchDogpile: - - def __init__(self, word, limit): - self.word = word - self.total_results = "" - self.server = 'www.dogpile.com' - self.hostname = 'www.dogpile.com' - self.limit = limit - self.proxy = False - - async def do_search(self): - # Dogpile is hardcoded to return 10 results. - try: - headers = {'User-agent': Core.get_user_agent()} - base_url = f'https://{self.server}/search/web?qsi=xx&q=%40{self.word}' - urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 10) if num <= self.limit] - responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy) - for response in responses: - self.total_results += response - except Exception as e: - print(f'Error Occurred: {e}') - - async def process(self, proxy=False): - self.proxy = proxy - await self.do_search() - - async def get_emails(self): - rawres = myparser.Parser(self.total_results, self.word) - return await rawres.emails() - - async def get_hostnames(self): - rawres = myparser.Parser(self.total_results, self.word) - return await rawres.hostnames() diff --git a/theHarvester/discovery/huntersearch.py b/theHarvester/discovery/huntersearch.py index 47907fab..67b92f89 100644 --- a/theHarvester/discovery/huntersearch.py +++ b/theHarvester/discovery/huntersearch.py @@ -14,7 +14,7 @@ class SearchHunter: raise MissingKey(True) self.total_results = "" self.counter = start - self.database = f'https://api.hunter.io/v2/domain-search?domain={word}&api_key={self.key}&limit={self.limit}' + self.database = f'https://api.hunter.io/v2/domain-search?domain={self.word}&api_key={self.key}&limit={self.limit}' self.proxy = False async def do_search(self): diff --git a/theHarvester/discovery/qwantsearch.py b/theHarvester/discovery/qwantsearch.py index 58c6119c..81ec8198 100644 --- a/theHarvester/discovery/qwantsearch.py +++ b/theHarvester/discovery/qwantsearch.py @@ -81,4 +81,3 @@ class SearchQwant: async def process(self, proxy=False) -> None: self.proxy = proxy await self.do_search() - print(f'\tSearching {self.limit} results.') diff --git a/theHarvester/discovery/suip.py b/theHarvester/discovery/suip.py index ceaaefa0..87ffe4fd 100644 --- a/theHarvester/discovery/suip.py +++ b/theHarvester/discovery/suip.py @@ -33,6 +33,8 @@ class SearchSuip: async def do_search(self): try: results = await self.handler(url="https://suip.biz/") + from pprint import pprint + pprint(results) for num in range(len(results)): # iterate through results and parse out the urls result = results[num] diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 242eea96..ef714a6f 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -140,7 +140,6 @@ class Core: 'certspotter', 'crtsh', 'dnsdumpster', - 'dogpile', 'duckduckgo', 'exalead', 'github-code',