Fix proxy handling and re-enable mypy type checking as tests fixed

Updated aiohttp calls to handle proxies correctly by converting them to strings when necessary. Removed an unnecessary blank line in core.py, and re-enabled static type checking with mypy in the GitHub workflow now that mypy tests are passing

.github/workflows/theHarvester.yml

@@ -102,7 +102,7 @@ jobs:
         run: |
           theHarvester -d yale.edu -c
 
-#      - name: Static type checking with mypy
-#        run: |
-#          mypy --pretty theHarvester/*/*.py
-#          mypy --pretty theHarvester/*/*/*.py
+      - name: Static type checking with mypy
+        run: |
+          mypy --pretty theHarvester/*/*.py
+          mypy --pretty theHarvester/*/*/*.py

theHarvester/discovery/dnsdumpster.py

@@ -28,7 +28,7 @@ class SearchDnsDumpster:
                     cookies = resp_cookies.split('csrftoken=')
                     csrftoken += cookies[1][: cookies[1].find(';')]
             else:
-                async with session.get(url, headers=headers, proxy=self.proxy) as resp:
+                async with session.get(url, headers=headers, proxy=str(self.proxy) if self.proxy else None) as resp:
                     resp_cookies = str(resp.cookies)
                     cookies = resp_cookies.split('csrftoken=')
                     csrftoken += cookies[1][: cookies[1].find(';')]
@@ -46,7 +46,7 @@ class SearchDnsDumpster:
                 async with session.post(url, headers=headers, data=data) as resp:
                     self.results = await resp.text()
             else:
-                async with session.post(url, headers=headers, data=data, proxy=self.proxy) as resp:
+                async with session.post(url, headers=headers, data=data, proxy=str(self.proxy) if self.proxy else None) as resp:
                     self.results = await resp.text()
             await session.close()
         except Exception as e:

theHarvester/lib/core.py

@@ -15,7 +15,6 @@ import ujson as json_loader
 import yaml
 
 from .version import version
-
 if TYPE_CHECKING:
     from collections.abc import Sized
 
@@ -287,12 +286,12 @@ class AsyncFetcher:
                 proxy = random.choice(cls().proxy_list)
                 if params != '':
                     async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
-                        async with session.get(url, params=params, proxy=proxy) as response:
+                        async with session.get(url, params=params, proxy=str(proxy) if proxy else None) as response:
                             await asyncio.sleep(5)
                             return await response.text() if json is False else await response.json()
                 else:
                     async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
-                        async with session.get(url, proxy=proxy) as response:
+                        async with session.get(url, proxy=str(proxy) if proxy else None) as response:
                             await asyncio.sleep(5)
                             return await response.text() if json is False else await response.json()
             elif params == '':

theHarvester/screenshot/screenshot.py

@@ -66,7 +66,7 @@ class ScreenShotter:
                 headers=headers,
                 connector=aiohttp.TCPConnector(ssl=sslcontext),
             ) as session:
-                async with session.get(url, verify_ssl=False) as resp:
+                async with session.get(url, ssl=False) as resp:
                     text = await resp.text('UTF-8')
                     return f'http://{url}' if not url.startswith('http') else url, text
         except Exception as e: