diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index 7c644ab0..c7d2964f 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -17,7 +17,7 @@ jobs: python-version: [3.7, 3.8] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1.1.1 with: @@ -31,6 +31,10 @@ jobs: run: | python theHarvester.py -d yale.edu -b baidu + - name: Run theHarvester module bufferoverun + run: | + python theHarvester.py -d yale.edu -b bufferoverun + - name: Run theHarvester module bing run: | python theHarvester.py -d yale.edu -b bing @@ -41,7 +45,7 @@ jobs: - name: Run theHarvester module crtsh run: | - python theHarvester.py -d yale.edu -b crtsh + python theHarvester.py -d hcl.com -b crtsh - name: Run theHarvester module dnsdumpster run: | diff --git a/.travis.yml b/.travis.yml index 1189fbd7..a2800431 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,8 @@ dist: bionic language: python cache: pip matrix: + allow_failures: + - python: 'nightly' include: - python: '3.7' env: TEST_SUITE=suite_3_7 @@ -15,7 +17,7 @@ before_install: install: - python setup.py test script: -- python theHarvester.py -d apple.com -b baidu,bing,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,intelx,threatcrowd,trello,twitter,virustotal,yahoo +- python theHarvester.py -d apple.com -b baidu,bing,bufferoverun,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,intelx,threatcrowd,trello,twitter,virustotal,yahoo -l 200 - pytest - flake8 . --count --show-source --statistics diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index acde95b3..7867fe54 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -425,12 +425,11 @@ async def start(): print('\n[*] Hosts found: ' + str(len(all_hosts))) print('---------------------') all_hosts = sorted(list(set(all_hosts))) - """full_host = hostchecker.Checker(all_hosts) - full, ips = await full_host.check()""" db = stash.StashManager() + full = [host if ':' in host and word in host else word in host.split(':')[0] and host for host in full] + full = [host for host in full if host] full.sort(key=lambda el: el.split(':')[0]) for host in full: - host = str(host) print(host) host_ip = [netaddr_ip.format() for netaddr_ip in sorted([netaddr.IPAddress(ip) for ip in ips])] await db.store_all(word, host_ip, 'ip', 'DNS-resolver') diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py index 1c773fe7..eb614dde 100644 --- a/theHarvester/discovery/crtsh.py +++ b/theHarvester/discovery/crtsh.py @@ -1,5 +1,5 @@ from theHarvester.lib.core import * -from typing import Set, List, Dict +from typing import Set class SearchCrtsh: @@ -14,7 +14,7 @@ class SearchCrtsh: try: url = f'https://crt.sh/?q=%25.{self.word}&output=json' response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy) - response: List[Dict] = response[0] + response = response[0] data = set( [dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in response]) diff --git a/theHarvester/lib/hostchecker.py b/theHarvester/lib/hostchecker.py index 9d0ffa40..bc19a18d 100644 --- a/theHarvester/lib/hostchecker.py +++ b/theHarvester/lib/hostchecker.py @@ -28,7 +28,7 @@ class Checker: else: return f"{host}:{', '.join(map(str, addresses))}", addresses except Exception: - return f"{host}:", tuple() + return f"{host}", tuple() async def query_all(self, resolver) -> list: results = await asyncio.gather(*[asyncio.create_task(self.query(host, resolver)) diff --git a/theHarvester/parsers/myparser.py b/theHarvester/parsers/myparser.py index e2cfb0ac..3b71a7d5 100644 --- a/theHarvester/parsers/myparser.py +++ b/theHarvester/parsers/myparser.py @@ -10,10 +10,10 @@ class Parser: async def genericClean(self): self.results = self.results.replace('', '').replace('', '').replace('', '').replace('', '')\ - .replace('%2f', '').replace('%3a', '').replace('', '').replace('', '')\ + .replace('%3a', '').replace('', '').replace('', '')\ .replace('', '').replace('', '') - for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '/', '\\'): + for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '%2f', '/', '\\'): self.results = self.results.replace(search, ' ') async def urlClean(self):