From 442917aeaee1df88dab63e0848b4a26a8ab42b08 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2020 20:25:52 +0000 Subject: [PATCH 1/5] Bump pyyaml from 5.2 to 5.3 Bumps [pyyaml](https://github.com/yaml/pyyaml) from 5.2 to 5.3. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/5.2...5.3) Signed-off-by: dependabot-preview[bot] --- Pipfile | 2 +- Pipfile.lock | 53 +++++++++++++++++++--------------------------------- 2 files changed, 20 insertions(+), 35 deletions(-) diff --git a/Pipfile b/Pipfile index cb49fcde..d5c0f8c2 100644 --- a/Pipfile +++ b/Pipfile @@ -11,7 +11,7 @@ beautifulsoup4 = "==4.8.2" dnspython = "==1.16.0" netaddr = "==0.7.19" plotly = "==4.4.1" -PyYAML = "==5.2" +PyYAML = "==5.3" requests = "==2.22.0" retrying = "==1.3.3" shodan = "==1.21.1" diff --git a/Pipfile.lock b/Pipfile.lock index 785e2d78..e8a5d196 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7ff431996a0c8b7fd33cf0f9df02e9be5ae384b46c7fdf21ffa97bb2c8942a1a" + "sha256": "fd676707dc29b4a8e90c02a3374b89bdd0f4d2e4833034f3f87f114842adf651" }, "pipfile-spec": 6, "requires": {}, @@ -270,20 +270,20 @@ }, "pyyaml": { "hashes": [ - "sha256:0e7f69397d53155e55d10ff68fdfb2cf630a35e6daf65cf0bdeaf04f127c09dc", - "sha256:2e9f0b7c5914367b0916c3c104a024bb68f269a486b9d04a2e8ac6f6597b7803", - "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc", - "sha256:38a4f0d114101c58c0f3a88aeaa44d63efd588845c5a2df5290b73db8f246d15", - "sha256:483eb6a33b671408c8529106df3707270bfacb2447bf8ad856a4b4f57f6e3075", - "sha256:4b6be5edb9f6bb73680f5bf4ee08ff25416d1400fbd4535fe0069b2994da07cd", - "sha256:7f38e35c00e160db592091751d385cd7b3046d6d51f578b29943225178257b31", - "sha256:8100c896ecb361794d8bfdb9c11fce618c7cf83d624d73d5ab38aef3bc82d43f", - "sha256:c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c", - "sha256:e4c015484ff0ff197564917b4b4246ca03f411b9bd7f16e02a2f586eb48b6d04", - "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4" + "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", + "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf", + "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5", + "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e", + "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811", + "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e", + "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d", + "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20", + "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689", + "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994", + "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615" ], "index": "pypi", - "version": "==5.2" + "version": "==5.3" }, "requests": { "hashes": [ @@ -403,14 +403,6 @@ "index": "pypi", "version": "==3.7.9" }, - "importlib-metadata": { - "hashes": [ - "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45", - "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f" - ], - "markers": "python_version < '3.8'", - "version": "==1.3.0" - }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", @@ -455,10 +447,10 @@ }, "packaging": { "hashes": [ - "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", - "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb", + "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8" ], - "version": "==19.2" + "version": "==20.0" }, "pluggy": { "hashes": [ @@ -553,17 +545,10 @@ }, "wcwidth": { "hashes": [ - "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", - "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603", + "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8" ], - "version": "==0.1.7" - }, - "zipp": { - "hashes": [ - "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", - "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" - ], - "version": "==0.6.0" + "version": "==0.1.8" } } } From abc02c3aa5426a1acbdf0379d31e2ca6a8660fee Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 8 Jan 2020 21:12:27 +0000 Subject: [PATCH 2/5] Bump shodan from 1.21.1 to 1.21.2 Bumps [shodan](https://github.com/achillean/shodan-python) from 1.21.1 to 1.21.2. - [Release notes](https://github.com/achillean/shodan-python/releases) - [Changelog](https://github.com/achillean/shodan-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/achillean/shodan-python/compare/1.21.1...1.21.2) Signed-off-by: dependabot-preview[bot] --- Pipfile | 2 +- Pipfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Pipfile b/Pipfile index d5c0f8c2..198bcf3a 100644 --- a/Pipfile +++ b/Pipfile @@ -14,7 +14,7 @@ plotly = "==4.4.1" PyYAML = "==5.3" requests = "==2.22.0" retrying = "==1.3.3" -shodan = "==1.21.1" +shodan = "==1.21.2" texttable = "==1.6.2" lxml = "==4.4.2" uvloop = "==0.14.0" diff --git a/Pipfile.lock b/Pipfile.lock index e8a5d196..392c9d4c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "fd676707dc29b4a8e90c02a3374b89bdd0f4d2e4833034f3f87f114842adf651" + "sha256": "bb159dd51a2579b97c88c14ff2e6a9b04bb0202fda8a1a533afc369bbdfc830d" }, "pipfile-spec": 6, "requires": {}, @@ -302,10 +302,10 @@ }, "shodan": { "hashes": [ - "sha256:834dfd084fed290b2b445545b0d1cac7822f3c0ed6ba09707efb1716bb485ede" + "sha256:ed3c38c749a5d77c935b226b6a7761e972269bd0d55c5c08526af73896aa6edd" ], "index": "pypi", - "version": "==1.21.1" + "version": "==1.21.2" }, "six": { "hashes": [ From fd32110a2bd552f423fa732655d44e2337434c1e Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sat, 11 Jan 2020 19:57:23 +0000 Subject: [PATCH 3/5] Update deps and code style updates --- Pipfile.lock | 15 +++++++++++++++ requirements/base.txt | 4 ++-- theHarvester/__main__.py | 20 ++++++-------------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 392c9d4c..97fa1c0c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -403,6 +403,14 @@ "index": "pypi", "version": "==3.7.9" }, + "importlib-metadata": { + "hashes": [ + "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45", + "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f" + ], + "markers": "python_version < '3.8'", + "version": "==1.3.0" + }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", @@ -549,6 +557,13 @@ "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8" ], "version": "==0.1.8" + }, + "zipp": { + "hashes": [ + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" + ], + "version": "==0.6.0" } } } diff --git a/requirements/base.txt b/requirements/base.txt index b82a8c97..39cb8321 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,9 +6,9 @@ dnspython==1.16.0 lxml==4.4.2 netaddr==0.7.19 plotly==4.4.1 -PyYaml==5.2 +PyYaml==5.3 requests==2.22.0 -shodan==1.21.1 +shodan==1.21.2 texttable==1.6.2 retrying==1.3.3 uvloop==0.14.0 \ No newline at end of file diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 1e7dabd8..4952f50a 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -24,20 +24,14 @@ async def start(): parser.add_argument('-d', '--domain', help='company name or domain to search', required=True) parser.add_argument('-l', '--limit', help='limit the number of search results, default=500', default=500, type=int) parser.add_argument('-S', '--start', help='start with result number X, default=0', default=0, type=int) - parser.add_argument('-g', '--google-dork', help='use Google Dorks for Google search', default=False, - action='store_true') - parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080)', - default=False, action='store_true') - parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False, - action='store_true') - parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts', - action='store_const', const='basic', default=False) + parser.add_argument('-g', '--google-dork', help='use Google Dorks for Google search', default=False, action='store_true') + parser.add_argument('-p', '--port-scan', help='scan the detected hosts and check for Takeovers (21,22,80,443,8080)', default=False, action='store_true') + parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False, action='store_true') + parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts', action='store_const', const='basic', default=False) parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup') parser.add_argument('-t', '--dns-tld', help='perform a DNS TLD expansion discovery, default False', default=False) - parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False, - action='store_true') - parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, - action='store_true') + parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False, action='store_true') + parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true') parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, certspotter, crtsh, dnsdumpster, dogpile, duckduckgo, github-code, google, @@ -681,6 +675,4 @@ async def entry_point(): if __name__ == '__main__': - # import uvloop - # uvloop.install() asyncio.run(main=entry_point(), debug=True) From 6f7b4d0a93c61895ee98a18f5c66628dada9f206 Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sat, 11 Jan 2020 20:04:13 +0000 Subject: [PATCH 4/5] Fix merge conflict in otx hopefully --- theHarvester/discovery/otxsearch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/theHarvester/discovery/otxsearch.py b/theHarvester/discovery/otxsearch.py index dd8aae30..c9e00fbb 100644 --- a/theHarvester/discovery/otxsearch.py +++ b/theHarvester/discovery/otxsearch.py @@ -1,4 +1,6 @@ from theHarvester.lib.core import * +# import json +# import grequests import re From 2b1ce6c8681cdcd26c1aefbac372ebb43b08883c Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Sat, 11 Jan 2020 20:09:18 +0000 Subject: [PATCH 5/5] Fix merge conflict in otx hopefully take2 --- theHarvester/discovery/otxsearch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/theHarvester/discovery/otxsearch.py b/theHarvester/discovery/otxsearch.py index c9e00fbb..e9c9df65 100644 --- a/theHarvester/discovery/otxsearch.py +++ b/theHarvester/discovery/otxsearch.py @@ -1,6 +1,6 @@ from theHarvester.lib.core import * -# import json -# import grequests +import json +import grequests import re