From ec4a724a8d566940d62b65fc50395a61120ed1d0 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sat, 15 Feb 2020 22:01:08 +0000 Subject: [PATCH 1/8] Add bufferoverun to GH actions --- .github/workflows/theHarvester.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index 7c644ab0..5d66969e 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -31,6 +31,10 @@ jobs: run: | python theHarvester.py -d yale.edu -b baidu + - name: Run theHarvester module bufferoverun + run: | + python theHarvester.py -d yale.edu -b bufferoverun + - name: Run theHarvester module bing run: | python theHarvester.py -d yale.edu -b bing @@ -41,7 +45,7 @@ jobs: - name: Run theHarvester module crtsh run: | - python theHarvester.py -d yale.edu -b crtsh + python theHarvester.py -d hcl.com -b crtsh - name: Run theHarvester module dnsdumpster run: | From 0b2a14315aa60f5731f692db0d3e64d679103592 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sat, 15 Feb 2020 22:25:48 +0000 Subject: [PATCH 2/8] Fix mypy error --- theHarvester/discovery/crtsh.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py index 1c773fe7..eb614dde 100644 --- a/theHarvester/discovery/crtsh.py +++ b/theHarvester/discovery/crtsh.py @@ -1,5 +1,5 @@ from theHarvester.lib.core import * -from typing import Set, List, Dict +from typing import Set class SearchCrtsh: @@ -14,7 +14,7 @@ class SearchCrtsh: try: url = f'https://crt.sh/?q=%25.{self.word}&output=json' response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy) - response: List[Dict] = response[0] + response = response[0] data = set( [dct['name_value'][2:] if '*.' == dct['name_value'][:2] else dct['name_value'] for dct in response]) From 4ac1cbc3fa85f683ce06855d3ce816c105ef115b Mon Sep 17 00:00:00 2001 From: VoltBoy <36332387+VoltBoy@users.noreply.github.com> Date: Sun, 16 Feb 2020 01:29:47 +0300 Subject: [PATCH 3/8] remove url scheme in genericClean Parser method (#385) * remove url scheme in genericClean Parser method * add :// to https and http tuple elements in genericClean parser method * remove %2f replacement in genericCliean parser method * returned a accidentally deleted row at Parser hostnames method * remove http and https tuple elements from generic clean, don't need this anymore * replace %2f to space in genericClean Parser method, just in case --- theHarvester/parsers/myparser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/theHarvester/parsers/myparser.py b/theHarvester/parsers/myparser.py index 16a74865..80c5a636 100644 --- a/theHarvester/parsers/myparser.py +++ b/theHarvester/parsers/myparser.py @@ -10,10 +10,10 @@ class Parser: def genericClean(self): self.results = self.results.replace('', '').replace('', '').replace('', '').replace('', '')\ - .replace('%2f', '').replace('%3a', '').replace('', '').replace('', '')\ + .replace('%3a', '').replace('', '').replace('', '')\ .replace('', '').replace('', '') - for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '/', '\\'): + for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '%2f', '/', '\\'): self.results = self.results.replace(search, ' ') def urlClean(self): From 84ce6cb1b3823645b4e02f607b46d1c5a38e4e4f Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sat, 15 Feb 2020 22:54:31 +0000 Subject: [PATCH 4/8] update GH actions to use v2 --- .github/workflows/theHarvester.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index 5d66969e..c7d2964f 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -17,7 +17,7 @@ jobs: python-version: [3.7, 3.8] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1.1.1 with: From 41f6b31e5152e5f0c431f1ff9f2c85d7863dc337 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sat, 15 Feb 2020 23:17:38 +0000 Subject: [PATCH 5/8] update travis to allow python nightly to allow failing --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1189fbd7..07f396a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,8 @@ dist: bionic language: python cache: pip matrix: + allow_failures: + - python: 'nightly' include: - python: '3.7' env: TEST_SUITE=suite_3_7 From 4124106f6f74ded1f2733ffe6396468f55852c64 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sat, 15 Feb 2020 23:20:18 +0000 Subject: [PATCH 6/8] update travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 07f396a7..a2800431 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ before_install: install: - python setup.py test script: -- python theHarvester.py -d apple.com -b baidu,bing,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,intelx,threatcrowd,trello,twitter,virustotal,yahoo +- python theHarvester.py -d apple.com -b baidu,bing,bufferoverun,certspotter,crtsh,dnsdumpster,dogpile,duckduckgo,exalead,linkedin,netcraft,intelx,threatcrowd,trello,twitter,virustotal,yahoo -l 200 - pytest - flake8 . --count --show-source --statistics From ae967ef84ec62c9a0f6549827b72c8e10e890806 Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Mon, 17 Feb 2020 00:33:33 -0500 Subject: [PATCH 7/8] If host did not resolve to ip strip colon from it. --- theHarvester/__main__.py | 5 +++-- theHarvester/lib/hostchecker.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index acde95b3..8384039b 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -428,9 +428,10 @@ async def start(): """full_host = hostchecker.Checker(all_hosts) full, ips = await full_host.check()""" db = stash.StashManager() - full.sort(key=lambda el: el.split(':')[0]) + full = [host if ':' in host else word in host.split(':')[0] and host for host in full] + full = [host for host in full if host] + full.sort(key=lambda el: el.split(':')[0] if el else ':' in el and el) for host in full: - host = str(host) print(host) host_ip = [netaddr_ip.format() for netaddr_ip in sorted([netaddr.IPAddress(ip) for ip in ips])] await db.store_all(word, host_ip, 'ip', 'DNS-resolver') diff --git a/theHarvester/lib/hostchecker.py b/theHarvester/lib/hostchecker.py index 9d0ffa40..bc19a18d 100644 --- a/theHarvester/lib/hostchecker.py +++ b/theHarvester/lib/hostchecker.py @@ -28,7 +28,7 @@ class Checker: else: return f"{host}:{', '.join(map(str, addresses))}", addresses except Exception: - return f"{host}:", tuple() + return f"{host}", tuple() async def query_all(self, resolver) -> list: results = await asyncio.gather(*[asyncio.create_task(self.query(host, resolver)) From 5a4b6cee78a05dcceb64b3464b6d71d25d2f3e2d Mon Sep 17 00:00:00 2001 From: NotoriousRebel Date: Mon, 17 Feb 2020 10:08:26 -0500 Subject: [PATCH 8/8] Updated how final list gets displayed. --- theHarvester/__main__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 8384039b..7867fe54 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -425,12 +425,10 @@ async def start(): print('\n[*] Hosts found: ' + str(len(all_hosts))) print('---------------------') all_hosts = sorted(list(set(all_hosts))) - """full_host = hostchecker.Checker(all_hosts) - full, ips = await full_host.check()""" db = stash.StashManager() - full = [host if ':' in host else word in host.split(':')[0] and host for host in full] + full = [host if ':' in host and word in host else word in host.split(':')[0] and host for host in full] full = [host for host in full if host] - full.sort(key=lambda el: el.split(':')[0] if el else ':' in el and el) + full.sort(key=lambda el: el.split(':')[0]) for host in full: print(host) host_ip = [netaddr_ip.format() for netaddr_ip in sorted([netaddr.IPAddress(ip) for ip in ips])]