Merge pull request #40 from NotoriousRebel/dev

Update how final list gets displayed.

.github/workflows/theHarvester.yml

@@ -17,7 +17,7 @@ jobs:
         python-version: [3.7, 3.8]
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1.1.1
       with:
@@ -31,6 +31,10 @@ jobs:
       run: |
         python theHarvester.py -d yale.edu -b baidu
 
+    - name: Run theHarvester module bufferoverun
+      run: |
+        python theHarvester.py -d yale.edu -b bufferoverun
+
     - name: Run theHarvester module bing
       run: |
         python theHarvester.py -d yale.edu -b bing
@@ -41,7 +45,7 @@ jobs:
 
     - name: Run theHarvester module crtsh
       run: |
-        python theHarvester.py -d yale.edu -b crtsh
+        python theHarvester.py -d hcl.com -b crtsh
 
     - name: Run theHarvester module dnsdumpster
       run: |

.travis.yml

@@ -2,6 +2,8 @@ dist: bionic
 language: python
 cache: pip
 matrix:
+  allow_failures:
+    - python: 'nightly'
   include:
   - python: '3.7'
     env: TEST_SUITE=suite_3_7
@@ -15,7 +17,7 @@ before_install:
 install:
 - python setup.py test
 script:
-- python theHarvester.py -d apple.com -b baidu,bing,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,intelx,threatcrowd,trello,twitter,virustotal,yahoo
+- python theHarvester.py -d apple.com -b baidu,bing,bufferoverun,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,intelx,threatcrowd,trello,twitter,virustotal,yahoo
   -l 200
 - pytest
 - flake8 . --count --show-source --statistics

theHarvester/__main__.py

@@ -425,12 +425,11 @@ async def start():
         print('\n[*] Hosts found: ' + str(len(all_hosts)))
         print('---------------------')
         all_hosts = sorted(list(set(all_hosts)))
-        """full_host = hostchecker.Checker(all_hosts)
-        full, ips = await full_host.check()"""
         db = stash.StashManager()
+        full = [host if ':' in host and word in host else word in host.split(':')[0] and host for host in full]
+        full = [host for host in full if host]
         full.sort(key=lambda el: el.split(':')[0])
         for host in full:
-            host = str(host)
             print(host)
         host_ip = [netaddr_ip.format() for netaddr_ip in sorted([netaddr.IPAddress(ip) for ip in ips])]
         await db.store_all(word, host_ip, 'ip', 'DNS-resolver')

theHarvester/discovery/crtsh.py

@@ -1,5 +1,5 @@
 from theHarvester.lib.core import *
-from typing import Set, List, Dict
+from typing import Set
 
 
 class SearchCrtsh:
@@ -14,7 +14,7 @@ class SearchCrtsh:
         try:
             url = f'https://crt.sh/?q=%25.{self.word}&output=json'
             response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy)
-            response: List[Dict] = response[0]
+            response = response[0]
             data = set(
                 [dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value']
                  for dct in response])

theHarvester/lib/hostchecker.py

@@ -28,7 +28,7 @@ class Checker:
             else:
                 return f"{host}:{', '.join(map(str, addresses))}", addresses
         except Exception:
-            return f"{host}:", tuple()
+            return f"{host}", tuple()
 
     async def query_all(self, resolver) -> list:
         results = await asyncio.gather(*[asyncio.create_task(self.query(host, resolver))

theHarvester/parsers/myparser.py

@@ -10,10 +10,10 @@ class Parser:
 
     async def genericClean(self):
         self.results = self.results.replace('<em>', '').replace('<b>', '').replace('</b>', '').replace('</em>', '')\
-            .replace('%2f', '').replace('%3a', '').replace('<strong>', '').replace('</strong>', '')\
+            .replace('%3a', '').replace('<strong>', '').replace('</strong>', '')\
             .replace('<wbr>', '').replace('</wbr>', '')
 
-        for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '/', '\\'):
+        for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '%2f', '/', '\\'):
             self.results = self.results.replace(search, ' ')
 
     async def urlClean(self):