remove dogpile module as there is a capcha and update readme and tidy up on qwant

.github/workflows/theHarvester.yml

@@ -51,10 +51,6 @@ jobs:
       run: |
         python theHarvester.py -d yale.edu -b dnsdumpster
 
-    - name: Run theHarvester module Dogplie
-      run: |
-        python theHarvester.py -d yale.edu -b dogpile
-
     - name: Run theHarvester module DuckDuckGo
       run: |
         python theHarvester.py -d yale.edu -b duckduckgo

README.md

@@ -26,8 +26,6 @@ Passive:
 
 * dnsdumpster: DNSdumpster search engine - https://dnsdumpster.com
 
-* dogpile: Dogpile search engine - www.dogpile.com
-
 * duckduckgo: DuckDuckGo search engine - www.duckduckgo.com
 
 * exalead: a Meta search engine - www.exalead.com/search
@@ -95,18 +93,17 @@ Documentation to setup API keys can be found at - https://github.com/laramies/th
 
 * bing
 * github
-* hunter
+* hunter - limited to 10 on the free plan so you will ned to do -l 10 switch
 * intelx
 * pentesttools
 * securityTrails
 * shodan
-* spyse
+* spyse - need to have a paid account be able to use the api now
 
 
 Install and dependencies:
 -------------------------
 * Python 3.7+
-* python3 -m pip install pipenv
 * https://github.com/laramies/theHarvester/wiki/Installation
 
 

theHarvester/__main__.py

@@ -33,7 +33,7 @@ async def start():
     parser.add_argument('-c', '--dns-brute', help='Perform a DNS brute force on the domain.', default=False, action='store_true')
     parser.add_argument('-f', '--filename', help='Save the results to an HTML and/or XML file.', default='', type=str)
     parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, bufferoverun, certspotter, crtsh, dnsdumpster,
-                            dogpile, duckduckgo, exalead, github-code, google,
+                            duckduckgo, exalead, github-code, google,
                             hackertarget, hunter, intelx, linkedin, linkedin_links, netcraft, otx, pentesttools,
                             qwant, rapiddns, securityTrails, spyse, sublist3r, suip, threatcrowd, threatminer,
                             trello, twitter, urlscan, virustotal, yahoo''')
@@ -212,14 +212,6 @@ async def start():
                     except Exception as e:
                         print(f'\033[93m[!] An error occurred with dnsdumpster: {e} \033[0m')
 
-                elif engineitem == 'dogpile':
-                    try:
-                        from theHarvester.discovery import dogpilesearch
-                        dogpile_search = dogpilesearch.SearchDogpile(word, limit)
-                        stor_lst.append(store(dogpile_search, engineitem, store_host=True, store_emails=True))
-                    except Exception as e:
-                        print(f'\033[93m[!] An error occurred with Dogpile: {e} \033[0m')
-
                 elif engineitem == 'duckduckgo':
                     from theHarvester.discovery import duckduckgosearch
                     duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(word, limit)

theHarvester/discovery/dogpilesearch.py

@@ -1,37 +0,0 @@
-from theHarvester.lib.core import *
-from theHarvester.parsers import myparser
-
-
-class SearchDogpile:
-
-    def __init__(self, word, limit):
-        self.word = word
-        self.total_results = ""
-        self.server = 'www.dogpile.com'
-        self.hostname = 'www.dogpile.com'
-        self.limit = limit
-        self.proxy = False
-
-    async def do_search(self):
-        # Dogpile is hardcoded to return 10 results.
-        try:
-            headers = {'User-agent': Core.get_user_agent()}
-            base_url = f'https://{self.server}/search/web?qsi=xx&q=%40{self.word}'
-            urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 10) if num <= self.limit]
-            responses = await AsyncFetcher.fetch_all(urls, headers=headers, proxy=self.proxy)
-            for response in responses:
-                self.total_results += response
-        except Exception as e:
-            print(f'Error Occurred: {e}')
-
-    async def process(self, proxy=False):
-        self.proxy = proxy
-        await self.do_search()
-
-    async def get_emails(self):
-        rawres = myparser.Parser(self.total_results, self.word)
-        return await rawres.emails()
-
-    async def get_hostnames(self):
-        rawres = myparser.Parser(self.total_results, self.word)
-        return await rawres.hostnames()

theHarvester/discovery/huntersearch.py

@@ -14,7 +14,7 @@ class SearchHunter:
             raise MissingKey(True)
         self.total_results = ""
         self.counter = start
-        self.database = f'https://api.hunter.io/v2/domain-search?domain={word}&api_key={self.key}&limit={self.limit}'
+        self.database = f'https://api.hunter.io/v2/domain-search?domain={self.word}&api_key={self.key}&limit={self.limit}'
         self.proxy = False
 
     async def do_search(self):

theHarvester/discovery/qwantsearch.py

@@ -81,4 +81,3 @@ class SearchQwant:
     async def process(self, proxy=False) -> None:
         self.proxy = proxy
         await self.do_search()
-        print(f'\tSearching {self.limit} results.')

theHarvester/discovery/suip.py

@@ -33,6 +33,8 @@ class SearchSuip:
     async def do_search(self):
         try:
             results = await self.handler(url="https://suip.biz/")
+            from pprint import pprint
+            pprint(results)
             for num in range(len(results)):
                 # iterate through results and parse out the urls
                 result = results[num]

theHarvester/lib/core.py

@@ -140,7 +140,6 @@ class Core:
                             'certspotter',
                             'crtsh',
                             'dnsdumpster',
-                            'dogpile',
                             'duckduckgo',
                             'exalead',
                             'github-code',