diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml
index 7c644ab0..c7d2964f 100644
--- a/.github/workflows/theHarvester.yml
+++ b/.github/workflows/theHarvester.yml
@@ -17,7 +17,7 @@ jobs:
python-version: [3.7, 3.8]
steps:
- - uses: actions/checkout@v1
+ - uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1.1.1
with:
@@ -31,6 +31,10 @@ jobs:
run: |
python theHarvester.py -d yale.edu -b baidu
+ - name: Run theHarvester module bufferoverun
+ run: |
+ python theHarvester.py -d yale.edu -b bufferoverun
+
- name: Run theHarvester module bing
run: |
python theHarvester.py -d yale.edu -b bing
@@ -41,7 +45,7 @@ jobs:
- name: Run theHarvester module crtsh
run: |
- python theHarvester.py -d yale.edu -b crtsh
+ python theHarvester.py -d hcl.com -b crtsh
- name: Run theHarvester module dnsdumpster
run: |
diff --git a/.travis.yml b/.travis.yml
index 1189fbd7..a2800431 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,8 @@ dist: bionic
language: python
cache: pip
matrix:
+ allow_failures:
+ - python: 'nightly'
include:
- python: '3.7'
env: TEST_SUITE=suite_3_7
@@ -15,7 +17,7 @@ before_install:
install:
- python setup.py test
script:
-- python theHarvester.py -d apple.com -b baidu,bing,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,intelx,threatcrowd,trello,twitter,virustotal,yahoo
+- python theHarvester.py -d apple.com -b baidu,bing,bufferoverun,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,intelx,threatcrowd,trello,twitter,virustotal,yahoo
-l 200
- pytest
- flake8 . --count --show-source --statistics
diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py
index acde95b3..7867fe54 100644
--- a/theHarvester/__main__.py
+++ b/theHarvester/__main__.py
@@ -425,12 +425,11 @@ async def start():
print('\n[*] Hosts found: ' + str(len(all_hosts)))
print('---------------------')
all_hosts = sorted(list(set(all_hosts)))
- """full_host = hostchecker.Checker(all_hosts)
- full, ips = await full_host.check()"""
db = stash.StashManager()
+ full = [host if ':' in host and word in host else word in host.split(':')[0] and host for host in full]
+ full = [host for host in full if host]
full.sort(key=lambda el: el.split(':')[0])
for host in full:
- host = str(host)
print(host)
host_ip = [netaddr_ip.format() for netaddr_ip in sorted([netaddr.IPAddress(ip) for ip in ips])]
await db.store_all(word, host_ip, 'ip', 'DNS-resolver')
diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py
index 1c773fe7..eb614dde 100644
--- a/theHarvester/discovery/crtsh.py
+++ b/theHarvester/discovery/crtsh.py
@@ -1,5 +1,5 @@
from theHarvester.lib.core import *
-from typing import Set, List, Dict
+from typing import Set
class SearchCrtsh:
@@ -14,7 +14,7 @@ class SearchCrtsh:
try:
url = f'https://crt.sh/?q=%25.{self.word}&output=json'
response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy)
- response: List[Dict] = response[0]
+ response = response[0]
data = set(
[dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value']
for dct in response])
diff --git a/theHarvester/lib/hostchecker.py b/theHarvester/lib/hostchecker.py
index 9d0ffa40..bc19a18d 100644
--- a/theHarvester/lib/hostchecker.py
+++ b/theHarvester/lib/hostchecker.py
@@ -28,7 +28,7 @@ class Checker:
else:
return f"{host}:{', '.join(map(str, addresses))}", addresses
except Exception:
- return f"{host}:", tuple()
+ return f"{host}", tuple()
async def query_all(self, resolver) -> list:
results = await asyncio.gather(*[asyncio.create_task(self.query(host, resolver))
diff --git a/theHarvester/parsers/myparser.py b/theHarvester/parsers/myparser.py
index e2cfb0ac..3b71a7d5 100644
--- a/theHarvester/parsers/myparser.py
+++ b/theHarvester/parsers/myparser.py
@@ -10,10 +10,10 @@ class Parser:
async def genericClean(self):
self.results = self.results.replace('', '').replace('', '').replace('', '').replace('', '')\
- .replace('%2f', '').replace('%3a', '').replace('', '').replace('', '')\
+ .replace('%3a', '').replace('', '').replace('', '')\
.replace('', '').replace('', '')
- for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '/', '\\'):
+ for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '%2f', '/', '\\'):
self.results = self.results.replace(search, ' ')
async def urlClean(self):