diff --git a/.github/workflows/theHarvester.yml b/.github/workflows/theHarvester.yml index 674cbea8..113c9d8e 100644 --- a/.github/workflows/theHarvester.yml +++ b/.github/workflows/theHarvester.yml @@ -1,13 +1,14 @@ +--- name: TheHarvester Python CI on: push: branches: - - '*' + - '*' pull_request: branches: - - '*' + - '*' jobs: Python: @@ -15,141 +16,113 @@ jobs: strategy: max-parallel: 8 matrix: - os: [ubuntu-latest, macos-latest] - python-version: [3.8, 3.9, 3.10.0] + os: [ ubuntu-latest, macos-latest ] + python-version: [ 3.8, 3.9, 3.10.0 ] steps: - - uses: actions/checkout@v3 - - name: Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - pip install --upgrade pip - pip install wheel - pip install -r requirements/dev.txt + - uses: actions/checkout@v3 + - name: Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install --upgrade pip + pip install wheel + pip install -r requirements/dev.txt - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-line-length=127 --statistics + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest + - name: Test with pytest + run: | + pytest - - name: Static type checking with mypy - run: | - mypy --pretty theHarvester/*/*.py - mypy --pretty theHarvester/*/*/*.py + - name: Static type checking with mypy + run: | + mypy --pretty theHarvester/*/*.py + mypy --pretty theHarvester/*/*/*.py - - name: Run theHarvester module Anubis - run: | - python theHarvester.py -d apple.com -b anubis + - name: Run theHarvester module Anubis + run: | + python theHarvester.py -d apple.com -b anubis - - name: Run theHarvester module Baidu - run: | - python theHarvester.py -d yale.edu -b baidu + - name: Run theHarvester module Baidu + run: | + python theHarvester.py -d yale.edu -b baidu - - name: Run theHarvester module Bufferoverun - run: | - python theHarvester.py -d yale.edu -b bufferoverun + - name: Run theHarvester module Bufferoverun + run: | + python theHarvester.py -d yale.edu -b bufferoverun - - name: Run theHarvester module Bing - run: | - python theHarvester.py -d yale.edu -b bing + - name: Run theHarvester module Bing + run: | + python theHarvester.py -d yale.edu -b bing - - name: Run theHarvester module CertSpotter - run: | - python theHarvester.py -d yale.edu -b certspotter + - name: Run theHarvester module CertSpotter + run: | + python theHarvester.py -d yale.edu -b certspotter - - name: Run theHarvester module Crtsh - run: | - python theHarvester.py -d hcl.com -b crtsh + - name: Run theHarvester module Crtsh + run: | + python theHarvester.py -d hcl.com -b crtsh - - name: Run theHarvester module DnsDumpster - run: | - python theHarvester.py -d yale.edu -b dnsdumpster + - name: Run theHarvester module DnsDumpster + run: | + python theHarvester.py -d yale.edu -b dnsdumpster - - name: Run theHarvester module DuckDuckGo - run: | - python theHarvester.py -d yale.edu -b duckduckgo + - name: Run theHarvester module DuckDuckGo + run: | + python theHarvester.py -d yale.edu -b duckduckgo - - name: Run theHarvester module Google - run: | - python theHarvester.py -d yale.edu -b google + - name: Run theHarvester module HackerTarget + run: | + python theHarvester.py -d yale.edu -b hackertarget - - name: Run theHarvester module HackerTarget - run: | - python theHarvester.py -d yale.edu -b hackertarget + - name: Run theHarvester module Intelx + run: | + python theHarvester.py -d yale.edu -b intelx - - name: Run theHarvester module Intelx - run: | - python theHarvester.py -d yale.edu -b intelx + - name: Run theHarvester module Omnisint + run: | + python theHarvester.py -d yale.edu -b omnisint - - name: Run theHarvester module LinkedIn - run: | - python theHarvester.py -d yale.edu -b linkedin + - name: Run theHarvester module Otx + run: | + python theHarvester.py -d yale.edu -b otx - - name: Run theHarvester module Linkedin links - run: | - python theHarvester.py -d yale.edu -b linkedin_links + - name: Run theHarvester module Qwant + run: | + python theHarvester.py -d yale.edu -b qwant - - name: Run theHarvester module N45ht - run: | - python theHarvester.py -d yale.edu -b n45ht + - name: Run theHarvester module RapidDns + run: | + python theHarvester.py -d yale.edu -b rapiddns - - name: Run theHarvester module Omnisint - run: | - python theHarvester.py -d yale.edu -b omnisint + - name: Run theHarvester module Sublist3r + run: | + python theHarvester.py -d yale.edu -b sublist3r - - name: Run theHarvester module Otx - run: | - python theHarvester.py -d yale.edu -b otx + - name: Run theHarvester module Threatcrowd + run: | + python theHarvester.py -d yale.edu -b threatcrowd - - name: Run theHarvester module Qwant - run: | - python theHarvester.py -d yale.edu -b qwant + - name: Run theHarvester module Threatminer + run: | + python theHarvester.py -d yale.edu -b threatminer - - name: Run theHarvester module RapidDns - run: | - python theHarvester.py -d yale.edu -b rapiddns + - name: Run theHarvester module Urlscan + run: | + python theHarvester.py -d yale.edu -b urlscan - - name: Run theHarvester module Sublist3r - run: | - python theHarvester.py -d yale.edu -b sublist3r + - name: Run theHarvester module Yahoo + run: | + python theHarvester.py -d yale.edu -b yahoo - - name: Run theHarvester module Threatcrowd - run: | - python theHarvester.py -d yale.edu -b threatcrowd - - - name: Run theHarvester module Threatminer - run: | - python theHarvester.py -d yale.edu -b threatminer - - - name: Run theHarvester module Trello - run: | - python theHarvester.py -d yale.edu -b trello - - - name: Run theHarvester module Twitter - run: | - python theHarvester.py -d yale.edu -b twitter - - - name: Run theHarvester module Urlscan - run: | - python theHarvester.py -d yale.edu -b urlscan - - - name: Run theHarvester module Virustotal - run: | - python theHarvester.py -d yale.edu -b virustotal - - - name: Run theHarvester module Yahoo - run: | - python theHarvester.py -d yale.edu -b yahoo - - - name: Run theHarvester module DNS brute force - run: | - python theHarvester.py -d yale.edu -c + - name: Run theHarvester module DNS brute force + run: | + python theHarvester.py -d yale.edu -c diff --git a/Dockerfile b/Dockerfile index e9dfa37c..6b095107 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:impish +FROM ubuntu:kinetic LABEL maintainer="@jay_townsend1 & @NotoriousRebel1" RUN mkdir /app WORKDIR /app diff --git a/README.md b/README.md index e6c276f3..42ce7ca0 100644 --- a/README.md +++ b/README.md @@ -38,20 +38,12 @@ Passive: * github-code: GitHub code search engine (Requires a GitHub Personal Access Token, see below.) - www.github.com -* google: Google search engine (Optional Google dorking.) - www.google.com - * hackertarget: Online vulnerability scanners and network intelligence to help organizations - https://hackertarget.com * hunter: Hunter search engine (Requires an API key, see below.) - www.hunter.io * intelx: Intelx search engine (Requires an API key, see below.) - www.intelx.io -* linkedin: Google search engine, specific search for LinkedIn users - www.linkedin.com - -* linkedin_links: specific search for LinkedIn users for target domain (Uses Google search.) - -* n45ht: - https://n45ht.or.id - * omnisint: Project Crobat, A Centralised Searchable Open Source Project Sonar DNS Database - https://github.com/Cgboal/SonarSearch * otx: AlienVault Open Threat Exchange - https://otx.alienvault.com @@ -80,10 +72,6 @@ Passive: * threatminer: Data mining for threat intelligence - https://www.threatminer.org/ -* trello: Search trello boards (Uses Google search.) - -* twitter: Twitter accounts related to a specific domain (Uses Google search.) - * urlscan: A sandbox for the web that is a URL and website scanner - https://urlscan.io * vhost: Bing virtual hosts search @@ -109,7 +97,7 @@ Documentation to setup API keys can be found at - https://github.com/laramies/th * censys - API keys are required and can be retrieved from your [Censys account](https://search.censys.io/account/api). * fullhunt * github -* hunter - limited to 10 on the free plan so you will need to do -l 10 switch +* hunter - limited to 10 on the free plan, so you will need to do -l 10 switch * intelx * pentesttools - not free * projecdiscovery - invite only for now diff --git a/api-keys.yaml b/api-keys.yaml index 7ca5aa1c..f060fe1c 100644 --- a/api-keys.yaml +++ b/api-keys.yaml @@ -36,7 +36,7 @@ apikeys: shodan: key: - spyse: + virustotal: key: zoomeye: diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..a7109d12 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +minversion = 7.1.1 +testpaths = tests +asyncio_mode=auto \ No newline at end of file diff --git a/requirements/base.txt b/requirements/base.txt index e5d89875..abcf51e9 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -18,7 +18,5 @@ retrying==1.3.3 setuptools==63.3.0 shodan==1.28.0 slowapi==0.1.5 --e git+https://github.com/L1ghtn1ng/spyse-python@main#egg=spyse-python -starlette==0.19.1 uvicorn==0.18.2 uvloop==0.16.0; platform_system != "Windows" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..777ae408 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +ignore = E501, F405, F403, E402, F401 \ No newline at end of file diff --git a/tests/discovery/test_githubcode.py b/tests/discovery/test_githubcode.py index 248e3038..0e7d52d6 100644 --- a/tests/discovery/test_githubcode.py +++ b/tests/discovery/test_githubcode.py @@ -87,13 +87,13 @@ class TestSearchGithubCode: Core.github_key = MagicMock(return_value="lol") test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) test_result = githubcode.SuccessResult(list(), next_page=2, last_page=4) - assert(2 == await test_class_instance.next_page_or_end(test_result)) + assert (2 == await test_class_instance.next_page_or_end(test_result)) async def test_last_page(self): Core.github_key = MagicMock(return_value="lol") test_class_instance = githubcode.SearchGithubCode(word="test", limit=500) test_result = githubcode.SuccessResult(list(), None, None) - assert(None is await test_class_instance.next_page_or_end(test_result)) + assert (None is await test_class_instance.next_page_or_end(test_result)) if __name__ == '__main__': pytest.main() diff --git a/tests/discovery/test_linkedin_links.py b/tests/discovery/test_linkedin_links.py deleted file mode 100644 index a7de4d6f..00000000 --- a/tests/discovery/test_linkedin_links.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 -# coding=utf-8 -from theHarvester.discovery import linkedinsearch -from theHarvester.discovery.constants import splitter -import os -import re -import pytest - -pytestmark = pytest.mark.asyncio -github_ci = os.getenv('GITHUB_ACTIONS') # Github set this to be the following: true instead of True - - -class TestGetLinks(object): - - async def test_splitter(self): - results = [ - 'https://www.linkedin.com/in/don-draper-b1045618', - 'https://www.linkedin.com/in/don-draper-b59210a', - 'https://www.linkedin.com/in/don-draper-b5bb50b3', - 'https://www.linkedin.com/in/don-draper-b83ba26', - 'https://www.linkedin.com/in/don-draper-b854a51' - ] - filtered_results = await splitter(results) - assert len(filtered_results) == 1 - - async def test_get_links(self): - search = linkedinsearch.SearchLinkedin("facebook.com", '100') - await search.process() - links = await search.get_links() - assert isinstance(links, list) - - async def test_links_linkedin(self): - dir_path = os.path.dirname(os.path.realpath(__file__)) - mock_response = open(dir_path + "/test_linkedin_links.txt") - mock_response_content = mock_response.read() - mock_response.close() - reg_links = re.compile(r"url=https:\/\/www\.linkedin.com(.*?)&") - temp = reg_links.findall(mock_response_content) - resul = [] - for regex_item in temp: - stripped_url = regex_item.replace("url=", "") - resul.append("https://www.linkedin.com" + stripped_url) - assert set(resul) - - -if __name__ == '__main__': - pytest.main() diff --git a/tests/discovery/test_linkedin_links.txt b/tests/discovery/test_linkedin_links.txt deleted file mode 100644 index b8804830..00000000 --- a/tests/discovery/test_linkedin_links.txt +++ /dev/null @@ -1,5 +0,0 @@ -LinkedIn
https://www.linkedin.com/in/gm-tuhin-ialam-546526b8
Albany, New York Area - Facebook Advertising
Gm Tuhin.ialam. facebook.com at Facebook Advertising. Albany, New York Area.
-Marketing and Advertising. Facebook Advertising. 0 connections ...

NIKUL www.facebook.com/nikulact - Modeling - Self Modeling ...

https://in.linkedin.com/in/nikulact
‎
Ahmedabad Area, India - Self Modeling
View NIKUL www.facebook.com/nikulact's profile on LinkedIn, the world's largest
-professional community. NIKUL has 1 job listed on their profile. See the ...

Victor Scott - Metal Band facebook.com/alchemyoftime - Alchemy of ...

https://www.linkedin.com/in/victor-scott-9a967343
Albany, New York Area - Alchemy of Time
Victor Scott. Metal Band facebook.com/alchemyoftime at Alchemy of Time. Albany
-, New York Area. Music. Alchemy of Time. 1 connection ...

elkhorbat lkhorbat - http://www.facebook.com/pages/elkhorbat ...

https://www.linkedin.com/in/elkhorbat-lkhorbat-6028b33a
United States - http://www.facebook.com/pages/elkhorbat/302997479939
View elkhorbat lkhorbat's profile on LinkedIn, the world's largest professional
-community. elkhorbat has 1 job listed on their profile. See the complete profile on
diff --git a/tests/discovery/test_n45ht.py b/tests/discovery/test_n45ht.py deleted file mode 100644 index 10a87000..00000000 --- a/tests/discovery/test_n45ht.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -# coding=utf-8 -from theHarvester.lib.core import * -from theHarvester.discovery import n45htsearch -import os -import requests -import pytest - -pytestmark = pytest.mark.asyncio -github_ci = os.getenv('GITHUB_ACTIONS') # Github set this to be the following: true instead of True - - -class TestN45ht(object): - @staticmethod - def domain() -> str: - return 'uber.com' - - async def test_api(self): - base_url = f'https://api.n45ht.or.id/v1/subdomain-enumeration?domain={TestN45ht.domain()}' - headers = {'User-Agent': Core.get_user_agent()} - request = requests.get(base_url, headers=headers) - assert request.status_code == 200 - - async def test_do_search(self): - search = n45htsearch.SearchN45ht(TestN45ht.domain()) - await search.process() - assert isinstance(await search.get_hostnames(), set) - - -if __name__ == '__main__': - pytest.main() diff --git a/tests/discovery/test_otx.py b/tests/discovery/test_otx.py index 30b2d94d..acc41c4c 100644 --- a/tests/discovery/test_otx.py +++ b/tests/discovery/test_otx.py @@ -27,12 +27,6 @@ class TestOtx(object): assert isinstance(await search.get_hostnames(), set) assert isinstance(await search.get_ips(), set) - async def test_search_no_results(self): - search = otxsearch.SearchOtx('radiant.eu') - await search.process() - assert len(await search.get_hostnames()) == 0 - assert len(await search.get_ips()) == 0 - if __name__ == '__main__': pytest.main() diff --git a/tests/discovery/test_qwantsearch.py b/tests/discovery/test_qwantsearch.py index 8452653e..2fdcad4d 100644 --- a/tests/discovery/test_qwantsearch.py +++ b/tests/discovery/test_qwantsearch.py @@ -14,15 +14,15 @@ class TestSearchQwant(object): def domain() -> str: return 'example.com' - def test_get_start_offset_return_0(self): + async def test_get_start_offset_return_0(self): search = qwantsearch.SearchQwant(TestSearchQwant.domain(), 0, 200) assert search.get_start_offset() == 0 - def test_get_start_offset_return_50(self): + async def test_get_start_offset_return_50(self): search = qwantsearch.SearchQwant(TestSearchQwant.domain(), 55, 200) assert search.get_start_offset() == 50 - def test_get_start_offset_return_100(self): + async def test_get_start_offset_return_100(self): search = qwantsearch.SearchQwant(TestSearchQwant.domain(), 100, 200) assert search.get_start_offset() == 100 diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 68e9c7f4..ce74c757 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -22,23 +22,20 @@ async def start(rest_args=None): parser.add_argument('-d', '--domain', help='Company name or domain to search.', required=True) parser.add_argument('-l', '--limit', help='Limit the number of search results, default=500.', default=500, type=int) parser.add_argument('-S', '--start', help='Start with result number X, default=0.', default=0, type=int) - parser.add_argument('-g', '--google-dork', help='Use Google Dorks for Google search.', default=False, action='store_true') parser.add_argument('-p', '--proxies', help='Use proxies for requests, enter proxies in proxies.yaml.', default=False, action='store_true') parser.add_argument('-s', '--shodan', help='Use Shodan to query discovered hosts.', default=False, action='store_true') parser.add_argument('--screenshot', help='Take screenshots of resolved domains specify output directory: --screenshot output_directory', default="", type=str) parser.add_argument('-v', '--virtual-host', help='Verify host name via DNS resolution and search for virtual hosts.', action='store_const', const='basic', default=False) parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup.') - parser.add_argument('-t', '--dns-tld', help='Perform a DNS TLD expansion discovery, default False.', default=False) parser.add_argument('-r', '--take-over', help='Check for takeovers.', default=False, action='store_true') parser.add_argument('-n', '--dns-lookup', help='Enable DNS server lookup, default False.', default=False, action='store_true') parser.add_argument('-c', '--dns-brute', help='Perform a DNS brute force on the domain.', default=False, action='store_true') parser.add_argument('-f', '--filename', help='Save the results to an XML and JSON file.', default='', type=str) parser.add_argument('-b', '--source', help='''anubis, baidu, bing, binaryedge, bingapi, bufferoverun, censys, certspotter, crtsh, - dnsdumpster, duckduckgo, fullhunt, github-code, google, - hackertarget, hunter, intelx, linkedin, linkedin_links, n45ht, + dnsdumpster, duckduckgo, fullhunt, github-code, hackertarget, hunter, intelx, omnisint, otx, pentesttools, projectdiscovery, - qwant, rapiddns, rocketreach, securityTrails, spyse, sublist3r, threatcrowd, threatminer, - trello, twitter, urlscan, virustotal, yahoo, zoomeye''') + qwant, rapiddns, rocketreach, securityTrails, sublist3r, threatcrowd, threatminer, + urlscan, virustotal, yahoo, zoomeye''') # determines if filename is coming from rest api or user rest_filename = '' @@ -73,13 +70,11 @@ async def start(rest_args=None): all_ip: List = [] dnslookup = args.dns_lookup dnsserver = args.dns_server - dnstld = args.dns_tld engines: List = [] # If the user specifies full: List = [] ips: List = [] - google_dorking = args.google_dork host_ip: List = [] limit: int = args.limit shodan = args.shodan @@ -127,7 +122,7 @@ async def start(rest_args=None): search_engine.process(process_param, use_proxy) db_stash = stash.StashManager() if source: - print(f'\033[94m[*] Searching {source[0].upper() + source[1:]}. \033[0m') + print(f'\033[94m[*] Searching {source[0].upper() + source[1:]}. ') if store_host: host_names = [host for host in filter(await search_engine.get_hostnames()) if f'.{word}' in host] if source != 'hackertarget' and source != 'pentesttools' and source != 'rapiddns': @@ -158,10 +153,6 @@ async def start(rest_args=None): await db.store_all(word, all_emails, 'email', source) if store_people: people_list = await search_engine.get_people() - if source == 'twitter': - twitter_people_list_tracker.extend(people_list) - if source == 'linkedin': - linkedin_people_list_tracker.extend(people_list) await db_stash.store_all(word, people_list, 'people', source) if store_links: @@ -189,7 +180,7 @@ async def start(rest_args=None): engines = Core.get_supportedengines() # Iterate through search engines in order if set(engines).issubset(Core.get_supportedengines()): - print(f'\033[94m[*] Target: {word} \n \033[0m') + print(f'\n[*] Target: {word} \n') for engineitem in engines: if engineitem == 'anubis': @@ -264,7 +255,7 @@ async def start(rest_args=None): crtsh_search = crtsh.SearchCrtsh(word) stor_lst.append(store(crtsh_search, 'CRTsh', store_host=True)) except Exception as e: - print(f'\033[93m[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}\033[0m') + print(f'[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}') elif engineitem == 'dnsdumpster': try: @@ -272,7 +263,7 @@ async def start(rest_args=None): dns_dumpster_search = dnsdumpster.SearchDnsDumpster(word) stor_lst.append(store(dns_dumpster_search, engineitem, store_host=True, store_ip=True)) except Exception as e: - print(f'\033[93m[!] An error occurred with dnsdumpster: {e} \033[0m') + print(f'[!] An error occurred with dnsdumpster: {e}') elif engineitem == 'duckduckgo': from theHarvester.discovery import duckduckgosearch @@ -298,12 +289,6 @@ async def start(rest_args=None): else: pass - elif engineitem == 'google': - from theHarvester.discovery import googlesearch - google_search = googlesearch.SearchGoogle(word, limit, start) - stor_lst.append(store(google_search, engineitem, process_param=google_dorking, store_host=True, - store_emails=True)) - elif engineitem == 'hackertarget': from theHarvester.discovery import hackertarget hackertarget_search = hackertarget.SearchHackerTarget(word) @@ -333,24 +318,6 @@ async def start(rest_args=None): else: print(f'An exception has occurred in Intelx search: {e}') - elif engineitem == 'linkedin': - from theHarvester.discovery import linkedinsearch - linkedin_search = linkedinsearch.SearchLinkedin(word, limit) - stor_lst.append(store(linkedin_search, engineitem, store_people=True)) - - elif engineitem == 'linkedin_links': - from theHarvester.discovery import linkedinsearch - linkedin_links_search = linkedinsearch.SearchLinkedin(word, limit) - stor_lst.append(store(linkedin_links_search, 'linkedin', store_links=True)) - - elif engineitem == 'n45ht': - from theHarvester.discovery import n45htsearch - try: - n45ht_search = n45htsearch.SearchN45ht(word) - stor_lst.append(store(n45ht_search, engineitem, store_host=True)) - except Exception as e: - print(e) - elif engineitem == 'omnisint': from theHarvester.discovery import omnisint try: @@ -432,14 +399,6 @@ async def start(rest_args=None): except Exception as e: print(e) - elif engineitem == 'spyse': - from theHarvester.discovery import spyse - try: - spyse_search = spyse.SearchSpyse(word, limit) - stor_lst.append(store(spyse_search, engineitem, store_host=True, store_ip=True)) - except Exception as e: - print(e) - elif engineitem == 'threatcrowd': from theHarvester.discovery import threatcrowd try: @@ -456,17 +415,6 @@ async def start(rest_args=None): except Exception as e: print(e) - elif engineitem == 'trello': - from theHarvester.discovery import trello - # Import locally or won't work. - trello_search = trello.SearchTrello(word) - stor_lst.append(store(trello_search, engineitem, store_results=True)) - - elif engineitem == 'twitter': - from theHarvester.discovery import twittersearch - twitter_search = twittersearch.SearchTwitter(word, limit) - stor_lst.append(store(twitter_search, engineitem, store_people=True)) - elif engineitem == 'urlscan': from theHarvester.discovery import urlscan try: @@ -477,9 +425,15 @@ async def start(rest_args=None): print(e) elif engineitem == 'virustotal': - from theHarvester.discovery import virustotal - virustotal_search = virustotal.SearchVirustotal(word) - stor_lst.append(store(virustotal_search, engineitem, store_host=True)) + try: + from theHarvester.discovery import virustotal + virustotal_search = virustotal.SearchVirustotal(word) + stor_lst.append(store(virustotal_search, engineitem, store_host=True)) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + pass elif engineitem == 'yahoo': from theHarvester.discovery import yahoosearch @@ -502,7 +456,7 @@ async def start(rest_args=None): # Check if dns_brute is defined rest_args.dns_brute except Exception: - print('\033[93m[!] Invalid source.\n\n \033[0m') + print('\n[!] Invalid source.\n') sys.exit(1) async def worker(queue): @@ -540,7 +494,7 @@ async def start(rest_args=None): await handler(lst=stor_lst) return_ips: List = [] if rest_args is not None and len(rest_filename) == 0 and rest_args.dns_brute is False: - # Indicates user is using rest api but not wanting output to be saved to a file + # Indicates user is using REST api but not wanting output to be saved to a file full = [host if ':' in host and word in host else word in host.split(':')[0] and host for host in full] full = list({host for host in full if host}) full.sort() @@ -549,16 +503,16 @@ async def start(rest_args=None): # return list(set(all_emails)), return_ips, full, '', '' return total_asns, interesting_urls, twitter_people_list_tracker, linkedin_people_list_tracker, \ linkedin_links_tracker, all_urls, all_ip, all_emails, all_hosts - # Sanity check to see if all_emails and all_hosts are defined. + # Check to see if all_emails and all_hosts are defined. try: all_emails except NameError: - print('\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m') + print('\n\n[!] No emails found because all_emails is not defined.\n\n ') sys.exit(1) try: all_hosts except NameError: - print('\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m') + print('\n\n[!] No hosts found because all_hosts is not defined.\n\n ') sys.exit(1) # Results @@ -705,20 +659,6 @@ async def start(rest_args=None): for xh in dnsrev: print(xh) - # DNS TLD expansion - dnstldres = [] - if dnstld is True: - print('[*] Starting DNS TLD expansion.') - a = dnssearch.DnsTld(word, dnsserver, verbose=True) - res = a.process() - print('\n[*] Hosts found after DNS TLD expansion:') - print('----------------------------------------') - for y in res: - print(y) - dnstldres.append(y) - if y not in full: - full.append(y) - # Virtual hosts search if virtual == 'basic': print('\n[*] Virtual hosts:') @@ -746,7 +686,7 @@ async def start(rest_args=None): from theHarvester.screenshot.screenshot import ScreenShotter screen_shotter = ScreenShotter(args.screenshot) path_exists = screen_shotter.verify_path() - # Verify path exists if not create it or if user does not create it skip screenshot + # Verify path exists, if not create it or if user does not create it skips screenshot if path_exists: await screen_shotter.verify_installation() print(f'\nScreenshots can be found in: {screen_shotter.output}{screen_shotter.slash}') @@ -782,7 +722,7 @@ async def start(rest_args=None): shodanres = [] if shodan is True: import json - print('\033[94m[*] Searching Shodan. \033[0m') + print('\033[94m[*] Searching Shodan. ') try: for ip in host_ip: print(('\tSearching for ' + ip)) @@ -803,21 +743,7 @@ async def start(rest_args=None): print(json.dumps(shodandict[ip], indent=4, sort_keys=True)) print('\n') except Exception as e: - print(f'\033[93m[!] An error occurred with Shodan: {e} \033[0m') - else: - pass - - # Here we need to add explosion mode. - # We have to take out the TLDs to do this. - if args.dns_tld is not False: - counter = 0 - for word in vhost: - search_google = googlesearch.SearchGoogle(word, limit, counter) - await search_google.process(google_dorking) - emails = await search_google.get_emails() - hosts = await search_google.get_hostnames() - print(emails) - print(hosts) + print(f'[!] An error occurred with Shodan: {e} ') else: pass @@ -850,7 +776,7 @@ async def start(rest_args=None): file.write('') print('[*] XML File saved.') except Exception as error: - print(f'\033[93m[!] An error occurred while saving the XML file: {error} \033[0m') + print(f'[!] An error occurred while saving the XML file: {error}') try: # JSON REPORT SECTION @@ -858,7 +784,7 @@ async def start(rest_args=None): # create dict with values for json output json_dict: Dict = dict() # determine if variable exists - # it should but just a sanity check + # it should but just a validation check if 'ip_list' in locals(): if all_ip and len(all_ip) >= 1 and ip_list and len(ip_list) > 0: json_dict["ips"] = [str(ip) for ip in ip_list] @@ -897,7 +823,7 @@ async def start(rest_args=None): fp.write(ujson.dumps(json_dict, sort_keys=True)) print('[*] JSON File saved.') except Exception as er: - print(f'\033[93m[!] An error occurred while saving the JSON file: {er} \033[0m') + print(f'[!] An error occurred while saving the JSON file: {er} ') print('\n\n') sys.exit(0) @@ -907,7 +833,7 @@ async def entry_point(): Core.banner() await start() except KeyboardInterrupt: - print('\n\n\033[93m[!] ctrl+c detected from user, quitting.\n\n \033[0m') + print('\n\n[!] ctrl+c detected from user, quitting.\n\n ') except Exception as error_entry_point: print(error_entry_point) sys.exit(1) diff --git a/theHarvester/discovery/certspottersearch.py b/theHarvester/discovery/certspottersearch.py index bbb97791..d00fe9a1 100644 --- a/theHarvester/discovery/certspottersearch.py +++ b/theHarvester/discovery/certspottersearch.py @@ -19,7 +19,7 @@ class SearchCertspoter: if key == 'dns_names': self.totalhosts.update({name for name in value if name}) elif isinstance(response, dict): - self.totalhosts.update({response['dns_names'] if 'dns_names' in response.keys() else ''}) + self.totalhosts.update({response['dns_names'] if 'dns_names' in response.keys() else ''}) # type: ignore else: self.totalhosts.update({''}) except Exception as e: diff --git a/theHarvester/discovery/crtsh.py b/theHarvester/discovery/crtsh.py index 8c1696f5..4a759ddd 100644 --- a/theHarvester/discovery/crtsh.py +++ b/theHarvester/discovery/crtsh.py @@ -1,5 +1,5 @@ from theHarvester.lib.core import * -from typing import List +from typing import List, Set class SearchCrtsh: diff --git a/theHarvester/discovery/dnssearch.py b/theHarvester/discovery/dnssearch.py index 525ac3f7..a5a4e456 100644 --- a/theHarvester/discovery/dnssearch.py +++ b/theHarvester/discovery/dnssearch.py @@ -66,9 +66,7 @@ NETWORK_REGEX = r'\b({})(?:\:({}))?(?:\/({}))?\b'.format( NETMASK_REGEX) -def serialize_ip_range( - ip: str, - netmask: str = '24') -> str: +def serialize_ip_range(ip: str, netmask: str = '24') -> str: """ Serialize a network range in a constant format, 'x.x.x.x/y'. diff --git a/theHarvester/discovery/googlesearch.py b/theHarvester/discovery/googlesearch.py deleted file mode 100644 index 67dfed2a..00000000 --- a/theHarvester/discovery/googlesearch.py +++ /dev/null @@ -1,172 +0,0 @@ -from theHarvester.discovery.constants import * -from theHarvester.parsers import myparser -import asyncio - - -class SearchGoogle: - - def __init__(self, word, limit, start): - self.word = word - self.results = "" - self.totalresults = "" - self.server = 'www.google.com' - self.dorks = [] - self.links = [] - self.database = 'https://www.google.com/search?q=' - self.quantity = '100' - self.limit = limit - self.counter = start - self.proxy = False - - async def do_search(self): - # Do normal scraping. - urly = 'http://' + self.server + '/search?num=' + self.quantity + '&start=' + str( - self.counter) + '&hl=en&meta=&q=%40\"' + self.word + '\"' - try: - headers = {'User-Agent': googleUA} - resp = await AsyncFetcher.fetch_all([urly], headers=headers, proxy=self.proxy) - except Exception as e: - print(e) - self.results = resp[0] - searched = await search(self.results) - if searched: - try: - self.results = await google_workaround(urly) - if isinstance(self.results, bool): - print('Google is blocking your ip and the workaround, returning') - return - except Exception as e: - print(e) - # google blocked, no useful result - return - await asyncio.sleep(get_delay()) - self.totalresults += self.results - - async def do_search_profiles(self): - urly = 'http://' + self.server + '/search?num=' + self.quantity + '&start=' + str( - self.counter) + '&hl=en&meta=&q=site:www.google.com%20intitle:\"Google%20Profile\"%20\"Companies%20I%27ve%20worked%20for\"%20\"at%20' + self.word + '\"' - try: - headers = {'User-Agent': googleUA} - resp = await AsyncFetcher.fetch_all([urly], headers=headers, proxy=self.proxy) - except Exception as e: - print(e) - self.results = resp[0] - if await search(self.results): - try: - self.results = await google_workaround(urly) - if isinstance(self.results, bool): - print('Google is blocking your ip and the workaround, returning') - return - except Exception: - # google blocked, no useful result - return - await asyncio.sleep(get_delay()) - self.totalresults += self.results - - async def get_emails(self): - rawres = myparser.Parser(self.totalresults, self.word) - return await rawres.emails() - - async def get_hostnames(self): - rawres = myparser.Parser(self.totalresults, self.word) - return await rawres.hostnames() - - async def get_files(self): - rawres = myparser.Parser(self.totalresults, self.word) - return rawres.fileurls(self.files) - - async def get_profiles(self): - rawres = myparser.Parser(self.totalresults, self.word) - return rawres.profiles() - - async def process(self, google_dorking, proxy=False): - self.proxy = proxy - if google_dorking is False: - while self.counter <= self.limit and self.counter <= 1000: - await self.do_search() - print(f'\tSearching {self.counter} results.') - self.counter += 100 - else: # Google dorking is true. - self.counter = 0 # Reset counter. - print('\n') - print('[-] Searching with Google Dorks: ') - await self.googledork() # Call Google dorking method if user wanted it! - - async def process_profiles(self): - while self.counter < self.limit: - await self.do_search_profiles() - await asyncio.sleep(get_delay()) - self.counter += 100 - print(f'\tSearching {self.counter} results.') - - async def append_dorks(self): - # Wrap in try-except in case filepaths are messed up. - try: - with open('/etc/theHarvester/wordlists/dorks.txt', 'r') as fp: - self.dorks = [dork.strip() for dork in fp] - except FileNotFoundError: - try: - with open('/usr/local/etc/theHarvester/wordlists/dorks.txt', 'r') as fp: - self.dorks = [dork.strip() for dork in fp] - except FileNotFoundError: - try: - with open('wordlists/dorks.txt', 'r') as fp: - self.dorks = [dork.strip() for dork in fp] - except FileNotFoundError as error: - print(error) - - async def construct_dorks(self): - # Format is: site:targetwebsite.com + space + inurl:admindork - colon = '%3A' - plus = '%2B' - space = '+' - period = '%2E' - double_quote = '%22' - asterick = '%2A' - left_bracket = '%5B' - right_bracket = '%5D' - question_mark = '%3F' - slash = '%2F' - single_quote = '%27' - ampersand = '%26' - left_peren = '%28' - right_peren = '%29' - pipe = '%7C' - # Format is google.com/search?q=dork+space+self.word - self.links = tuple( - self.database + str(dork).replace(':', colon).replace('+', plus).replace('.', period).replace('"', - double_quote) - .replace('*', asterick).replace('[', left_bracket).replace(']', right_bracket) - .replace('?', question_mark).replace(' ', space).replace('/', slash).replace("'", single_quote) - .replace('&', ampersand).replace('(', left_peren).replace(')', right_peren).replace('|', - pipe) + space + self.word - for dork in self.dorks) - - async def googledork(self): - await self.append_dorks() # Call functions to create list. - await self.construct_dorks() - await self.send_dorks() - - async def send_dorks(self): # Helper function to minimize code reusability. - headers = {'User-Agent': googleUA} - # Get random user agent to try and prevent google from blocking IP. - for num in range(len(self.links)): - try: - if num % 10 == 0 and num > 0: - print(f'\tSearching through {num} results') - link = self.links[num] - req = await AsyncFetcher.fetch_all([link], headers=headers, proxy=self.proxy) - self.results = req[0] - if await search(self.results): - try: - self.results = await google_workaround(link) - if isinstance(self.results, bool): - print('Google is blocking your ip and the workaround, returning') - return - except Exception: - # google blocked, no useful result - return - await asyncio.sleep(get_delay()) - self.totalresults += self.results - except Exception as e: - print(f'\tException Occurred {e}') diff --git a/theHarvester/discovery/linkedinsearch.py b/theHarvester/discovery/linkedinsearch.py deleted file mode 100644 index 47ff715c..00000000 --- a/theHarvester/discovery/linkedinsearch.py +++ /dev/null @@ -1,55 +0,0 @@ -from theHarvester.discovery.constants import * -from theHarvester.lib.core import * -from theHarvester.parsers import myparser -import asyncio - - -class SearchLinkedin: - - def __init__(self, word, limit): - self.word = word.replace(' ', '%20') - self.results = "" - self.totalresults = "" - self.server = 'www.google.com' - self.quantity = '100' - self.limit = int(limit) - self.counter = 0 - self.proxy = False - - async def do_search(self): - urly = 'http://' + self.server + '/search?num=100&start=' + str(self.counter) + '&hl=en&meta=&q=site%3Alinkedin.com/in%20' + self.word - try: - headers = {'User-Agent': Core.get_user_agent()} - resp = await AsyncFetcher.fetch_all([urly], headers=headers, proxy=self.proxy) - self.results = resp[0] - if await search(self.results): - try: - self.results = await google_workaround(urly) - if isinstance(self.results, bool): - print('Google is blocking your ip and the workaround, returning') - return - except Exception: - # google blocked, no useful result - return - except Exception as e: - print(e) - await asyncio.sleep(get_delay()) - self.totalresults += self.results - - async def get_people(self): - rawres = myparser.Parser(self.totalresults, self.word) - temp = await rawres.people_linkedin() - return [person for person in temp - if person[0] != '.' and '...' not in person and len(person.split()) != 1] - - async def get_links(self): - links = myparser.Parser(self.totalresults, self.word) - return await splitter(await links.links_linkedin()) - - async def process(self, proxy=False): - self.proxy = proxy - while self.counter < self.limit: - await self.do_search() - await asyncio.sleep(get_delay()) - self.counter += 100 - print(f'\tSearching {self.counter} results.') diff --git a/theHarvester/discovery/n45htsearch.py b/theHarvester/discovery/n45htsearch.py deleted file mode 100644 index 5360115e..00000000 --- a/theHarvester/discovery/n45htsearch.py +++ /dev/null @@ -1,23 +0,0 @@ -from theHarvester.lib.core import * - - -class SearchN45ht: - - def __init__(self, word): - self.word = word - self.totalhosts = set() - self.proxy = False - - async def do_search(self): - url = f'https://api.n45ht.or.id/v1/subdomain-enumeration?domain={self.word}' - response = await AsyncFetcher.fetch_all([url], json=True, proxy=self.proxy) - responses = response[0] - dct = responses - self.totalhosts: set = {host for host in dct['subdomains']} - - async def get_hostnames(self) -> set: - return self.totalhosts - - async def process(self, proxy=False): - self.proxy = proxy - await self.do_search() diff --git a/theHarvester/discovery/spyse.py b/theHarvester/discovery/spyse.py deleted file mode 100644 index f68ad934..00000000 --- a/theHarvester/discovery/spyse.py +++ /dev/null @@ -1,81 +0,0 @@ -from theHarvester.discovery.constants import * -from theHarvester.lib.core import * -from spyse import Client, SearchQuery, QueryParam, DomainSearchParams, Operators - - -class SearchSpyse: - - def __init__(self, word, limit): - self.ips = set() - self.word = word - self.key = Core.spyse_key() - if self.key is None: - raise MissingKey('Spyse') - self.results = '' - self.hosts = set() - self.proxy = False - self.limit = limit - self.client = Client(self.key) - - async def do_search(self): - fetch_count = 0 - - query = SearchQuery() - query.append_param(QueryParam(DomainSearchParams.name, Operators.ends_with, self.word)) - - try: - total = self.client.count_domains(query) - if total == 0: - return - - # The default "Search" method returns only first 10 000 subdomains - # To obtain more than 10 000 subdomains the "Scroll" method should be using - # Note: The "Scroll" method is only available for "PRO" customers, so we need to check - # self.client.account.is_scroll_search_enabled param - if total > self.client.SEARCH_RESULTS_LIMIT and self.client.account.is_scroll_search_enabled: - scroll_id = None - while True: - scroll_results = self.client.scroll_domains(query, scroll_id) - - scroll_id = scroll_results.search_id - for domain in scroll_results.results: - self.hosts.add(domain.name) - - fetch_count += len(scroll_results.results) - - if len(scroll_results.results) == 0 or fetch_count >= self.limit: - break - else: - # Spyse allows to get up to 100 results per one request - max_limit = 100 - # Spyse "search" methods allows to fetch up to 10 000 first results - max_offset = 9900 - offset = 0 - - while True: - limit = max_limit if self.limit - fetch_count > max_limit else self.limit - fetch_count - if limit <= 0: - break - - results = self.client.search_domains(query, limit, offset) - - if len(results.results) == 0: - break - - for domain in results.results: - self.hosts.add(domain.name) - - offset += max_limit - fetch_count += len(results.results) - if offset > max_offset or fetch_count == total: - break - - except Exception as e: - print(f'An exception has occurred: {e}') - - async def get_hostnames(self): - return self.hosts - - async def process(self, proxy=False): - self.proxy = proxy - await self.do_search() diff --git a/theHarvester/discovery/trello.py b/theHarvester/discovery/trello.py deleted file mode 100644 index 5c4ea7ac..00000000 --- a/theHarvester/discovery/trello.py +++ /dev/null @@ -1,71 +0,0 @@ -from theHarvester.discovery.constants import * -from theHarvester.parsers import myparser -import random -import asyncio - - -class SearchTrello: - - def __init__(self, word): - self.word = word.replace(' ', '%20') - self.results = "" - self.totalresults = "" - self.server = 'www.google.com' - self.quantity = '100' - self.limit = 300 - self.trello_urls = [] - self.hostnames = [] - self.counter = 0 - self.proxy = False - - async def do_search(self): - base_url = f'https://{self.server}/search?num=300&start=xx&hl=en&q=site%3Atrello.com%20{self.word}' - urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 20) if num <= self.limit] - # limit is 20 as that is the most results google will show per num - headers = {'User-Agent': googleUA} - for url in urls: - try: - resp = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy) - self.results = resp[0] - if await search(self.results): - try: - self.results = await google_workaround(base_url) - if isinstance(self.results, bool): - print('Google is blocking your ip and the workaround, returning') - return - except Exception as e: - print(e) - self.totalresults += self.results - await asyncio.sleep(get_delay() - .5) - except Exception as e: - print(f'An exception has occurred in trello: {e}') - - async def get_emails(self): - rawres = myparser.Parser(self.totalresults, self.word) - return await rawres.emails() - - async def get_urls(self): - try: - rawres = myparser.Parser(self.totalresults, 'trello.com') - self.trello_urls = set(await rawres.urls()) - self.totalresults = '' - # reset what totalresults as before it was just google results now it is trello results - headers = {'User-Agent': random.choice(['curl/7.37.0', 'Wget/1.19.4'])} - # do not change the headers - responses = await AsyncFetcher.fetch_all(self.trello_urls, headers=headers, proxy=self.proxy) - for response in responses: - self.totalresults += response - - rawres = myparser.Parser(self.totalresults, self.word) - self.hostnames = await rawres.hostnames() - except Exception as e: - print(f'Error occurred: {e}') - - async def process(self, proxy=False): - self.proxy = proxy - await self.do_search() - await self.get_urls() - print(f'\tSearching {self.counter} results.') - - async def get_results(self) -> tuple: - return await self.get_emails(), self.hostnames, self.trello_urls diff --git a/theHarvester/discovery/twittersearch.py b/theHarvester/discovery/twittersearch.py deleted file mode 100644 index 150805cc..00000000 --- a/theHarvester/discovery/twittersearch.py +++ /dev/null @@ -1,54 +0,0 @@ -from theHarvester.discovery.constants import * -from theHarvester.lib.core import * -from theHarvester.parsers import myparser -import re - - -class SearchTwitter: - - def __init__(self, word, limit): - self.word = word.replace(' ', '%20') - self.results = "" - self.totalresults = "" - self.server = 'www.google.com' - self.quantity = '100' - self.limit = int(limit) - self.counter = 0 - self.proxy = False - - async def do_search(self): - base_url = f'https://{self.server}/search?num=100&start=xx&hl=en&meta=&q=site%3Atwitter.com%20intitle%3A%22on+Twitter%22%20{self.word}' - headers = {'User-Agent': Core.get_user_agent()} - try: - urls = [base_url.replace("xx", str(num)) for num in range(0, self.limit, 10) if num <= self.limit] - for url in urls: - response = await AsyncFetcher.fetch_all([url], headers=headers, proxy=self.proxy) - self.results = response[0] - if await search(self.results): - try: - self.results = await google_workaround(url) - if isinstance(self.results, bool): - print('Google is blocking your ip and the workaround, returning') - return - except Exception: - # google blocked, no useful result - return - self.totalresults += self.results - except Exception as error: - print(error) - - async def get_people(self, proxy=False): - self.proxy = proxy - rawres = myparser.Parser(self.totalresults, self.word) - to_parse = await rawres.people_twitter() - # fix invalid handles that look like @user other_output - handles = set() - for handle in to_parse: - result = re.search(r'^@?(\w){1,15}', handle) - if result: - handles.add(result.group(0)) - return handles - - async def process(self, proxy=False): - self.proxy = proxy - await self.do_search() diff --git a/theHarvester/discovery/virustotal.py b/theHarvester/discovery/virustotal.py index 55158f6f..7e42a81e 100644 --- a/theHarvester/discovery/virustotal.py +++ b/theHarvester/discovery/virustotal.py @@ -1,39 +1,29 @@ +from theHarvester.discovery.constants import * from theHarvester.lib.core import * -from theHarvester.parsers import myparser -import re +from pprint import pprint class SearchVirustotal: def __init__(self, word): self.word = word - self.results = "" - self.totalresults = "" - self.quantity = '100' - self.counter = 0 + self.key = Core.virustotal_key() + if self.key is None: + raise MissingKey('virustotal') + self.totalhosts = set self.proxy = False async def do_search(self): - base_url = f'https://www.virustotal.com/ui/domains/{self.word}/subdomains?relationships=resolutions&cursor=STMwCi4%3D&limit=40' - headers = {'User-Agent': Core.get_user_agent()} - responses = await AsyncFetcher.fetch_all([base_url], headers=headers, proxy=self.proxy) - self.results = responses[0] - self.totalresults += self.results + url = f'https://www.virustotal.com/api/v3/domains/{self.word}/subdomains?limit=40' + response = await AsyncFetcher.fetch_all([url], json=True, headers={'User-Agent': Core.get_user_agent(), + 'X-APIKEY': self.key}, + proxy=self.proxy) + entry = [host for host in response] + pprint(entry.items()) - async def get_hostnames(self): - rawres = myparser.Parser(self.results, self.word) - new_lst = [] - for host in await rawres.hostnames(): - host = str(host) - if host[0].isdigit(): - matches = re.match('.+([0-9])[^0-9]*$', host) - # Get last digit of string and shift hostname to remove ip in string - new_lst.append(host[matches.start(1) + 1:]) - else: - new_lst.append(host) - return new_lst + # async def get_hostnames(self) -> set: + # return self.total_results async def process(self, proxy=False): self.proxy = proxy - print('\tSearching results.') await self.do_search() diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index 289df348..d31fa7dc 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -1,6 +1,6 @@ # coding=utf-8 - -from typing import Set, Union, Any, Tuple, List +from __future__ import annotations +from typing import Union, Any, Tuple, List import yaml import asyncio import aiohttp @@ -12,7 +12,7 @@ import certifi class Core: @staticmethod def version() -> str: - return '4.0.3' + return '4.2.0dev' @staticmethod def api_keys() -> dict: @@ -76,14 +76,14 @@ class Core: def shodan_key() -> str: return Core.api_keys()['shodan']['key'] - @staticmethod - def spyse_key() -> str: - return Core.api_keys()['spyse']['key'] - @staticmethod def zoomeye_key() -> str: return Core.api_keys()['zoomeye']['key'] + @staticmethod + def virustotal_key() -> str: + return Core.api_keys()['virustotal']['key'] + @staticmethod def proxy_list() -> List: try: @@ -101,7 +101,7 @@ class Core: @staticmethod def banner() -> None: - print('\n\033[93m*******************************************************************') + print('*******************************************************************') print("* _ _ _ *") print(r"* | |_| |__ ___ /\ /\__ _ _ ____ _____ ___| |_ ___ _ __ *") print(r"* | __| _ \ / _ \ / /_/ / _` | '__\ \ / / _ \/ __| __/ _ \ '__| *") @@ -114,11 +114,11 @@ class Core: print('* Edge-Security Research *') print('* cmartorella@edge-security.com *') print('* *') - print('******************************************************************* \n\n\033[0m') + print('*******************************************************************') @staticmethod - def get_supportedengines() -> Set[Union[str, Any]]: - supportedengines = {'anubis', + def get_supportedengines() -> list[str | Any]: + supportedengines = ['anubis', 'baidu', 'binaryedge', 'bing', @@ -131,13 +131,9 @@ class Core: 'duckduckgo', 'fullhunt', 'github-code', - 'google', 'hackertarget', 'hunter', 'intelx', - 'linkedin', - 'linkedin_links', - 'n45ht', 'omnisint', 'otx', 'pentesttools', @@ -147,86 +143,89 @@ class Core: 'rocketreach', 'securityTrails', 'sublist3r', - 'spyse', 'threatcrowd', 'threatminer', - 'trello', - 'twitter', 'urlscan', 'virustotal', 'yahoo', 'zoomeye' - } + ] return supportedengines @staticmethod def get_user_agent() -> str: # User-Agents from https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ user_agents = [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Safari/605.1.15", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0", - "Mozilla/5.0 (X11; Linux x86_64; rv:93.0) Gecko/20100101 Firefox/93.0", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:94.0) Gecko/20100101 Firefox/94.0", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:93.0) Gecko/20100101 Firefox/93.0", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:93.0) Gecko/20100101 Firefox/93.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36 OPR/80.0.4170.63", - "Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 Edg/95.0.1020.44", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 Edg/95.0.1020.53", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36 Edg/95.0.1020.40", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36 Edg/95.0.1020.30", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", - "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36 Edg/94.0.992.50", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0", - "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36 OPR/80.0.4170.63 (Edition Yx 05)", - "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 YaBrowser/21.9.2.169 Yowser/2.5 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:92.0) Gecko/20100101 Firefox/92.0", - "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36", - "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:93.0) Gecko/20100101 Firefox/93.0", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36 OPR/80.0.4170.91", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:92.0) Gecko/20100101 Firefox/92.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:93.0) Gecko/20100101 Firefox/93.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:91.0) Gecko/20100101 Firefox/91.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36"] + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0', + 'Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:100.0) Gecko/20100101 Firefox/100.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.61 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.61 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:101.0) Gecko/20100101 Firefox/101.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.53', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.62 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.61 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.47', + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36 Edg/101.0.1210.39', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15', + 'Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Firefox/91.0', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36', + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36 Edg/101.0.1210.32', + 'Mozilla/5.0 (X11; Linux x86_64; rv:101.0) Gecko/20100101 Firefox/101.0', + 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 OPR/86.0.4363.59', + 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.141 YaBrowser/22.3.3.852 Yowser/2.5 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36 OPR/85.0.4341.75', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:99.0) Gecko/20100101 Firefox/99.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36', + 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Safari/605.1.15', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36 OPR/85.0.4341.71', + 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 OPR/86.0.4363.64', + 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:101.0) Gecko/20100101 Firefox/101.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.50', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36' + ] return random.choice(user_agents) @@ -238,7 +237,7 @@ class AsyncFetcher: if len(headers) == 0: headers = {'User-Agent': Core.get_user_agent()} timeout = aiohttp.ClientTimeout(total=720) - # by default timeout is 5 minutes, changed to 12 minutes + # By default, timeout is 5 minutes, changed to 12 minutes # results are well worth the wait try: if proxy: @@ -322,8 +321,8 @@ class AsyncFetcher: return url, '' @classmethod - async def fetch_all(cls, urls, headers='', params='', json=False, takeover=False, proxy=False) -> list: - # By default timeout is 5 minutes, 60 seconds should suffice + async def fetch_all(cls, urls, headers='', params='', json=False, takeover=False, proxy=False) -> tuple: + # By default, timeout is 5 minutes; 60 seconds should suffice timeout = aiohttp.ClientTimeout(total=60) if len(headers) == 0: headers = {'User-Agent': Core.get_user_agent()} diff --git a/theHarvester/parsers/myparser.py b/theHarvester/parsers/myparser.py index f2c76a9e..7a8eb4ac 100644 --- a/theHarvester/parsers/myparser.py +++ b/theHarvester/parsers/myparser.py @@ -9,8 +9,8 @@ class Parser: self.temp = [] async def genericClean(self): - self.results = self.results.replace('', '').replace('', '').replace('', '').replace('', '')\ - .replace('%3a', '').replace('', '').replace('', '')\ + self.results = self.results.replace('', '').replace('', '').replace('', '').replace('', '') \ + .replace('%3a', '').replace('', '').replace('', '') \ .replace('', '').replace('', '') for search in ('<', '>', ':', '=', ';', '&', '%3A', '%3D', '%3C', '%2f', '/', '\\'): @@ -55,22 +55,6 @@ class Parser: hostnames.extend(await self.unique()) return list(set(hostnames)) - async def people_googleplus(self): - self.results = re.sub('', '', self.results) - self.results = re.sub('', '', self.results) - reg_people = re.compile(r'>[a-zA-Z0-9._ ]* - Google\+') - self.temp = reg_people.findall(self.results) - resul = [] - for iteration in self.temp: - delete = iteration.replace(' | LinkedIn', '') - delete = delete.replace(' profiles ', '') - delete = delete.replace('LinkedIn', '') - delete = delete.replace('"', '') - delete = delete.replace('>', '') - if delete != " ": - resul.append(delete) - return resul - async def hostnames_all(self): reg_hosts = re.compile('(.*?)') temp = reg_hosts.findall(self.results) @@ -83,60 +67,8 @@ class Parser: hostnames = await self.unique() return hostnames - async def links_linkedin(self): - reg_links = re.compile(r"url=https:\/\/www\.linkedin.com(.*?)&") - self.temp = reg_links.findall(self.results) - resul = [] - for regex in self.temp: - final_url = regex.replace("url=", "") - resul.append("https://www.linkedin.com" + final_url) - return resul - - async def people_linkedin(self): - reg_people = re.compile(r'">[a-zA-Z0-9._ -]* -|\| LinkedIn') - self.temp = reg_people.findall(self.results) - resul = [] - for iteration in (self.temp): - delete = iteration.replace(' | LinkedIn', '') - delete = delete.replace(' - LinkedIn', '') - delete = delete.replace(' profiles ', '') - delete = delete.replace('LinkedIn', '') - delete = delete.replace('"', '') - delete = delete.replace('>', '') - delete = delete.strip("-") - if delete != " ": - resul.append(delete) - return resul - - async def people_twitter(self): - reg_people = re.compile(r'(@[a-zA-Z0-9._ -]*)') - self.temp = reg_people.findall(self.results) - users = await self.unique() - resul = [] - for iteration in users: - delete = iteration.replace(' | LinkedIn', '') - delete = delete.replace(' profiles ', '') - delete = delete.replace('LinkedIn', '') - delete = delete.replace('"', '') - delete = delete.replace('>', '') - if delete != " ": - resul.append(delete) - return resul - - async def profiles(self): - reg_people = re.compile(r'">[a-zA-Z0-9._ -]* - Google Profile') - self.temp = reg_people.findall(self.results) - resul = [] - for iteration in self.temp: - delete = iteration.replace(' Google Profile', '') - delete = delete.replace('-', '') - delete = delete.replace('">', '') - if delete != " ": - resul.append(delete) - return resul - async def set(self): - reg_sets = re.compile(r'>[a-zA-Z0-9]*') + reg_sets = re.compile(r'>[a-zA-Z\d]*') self.temp = reg_sets.findall(self.results) sets = [] for iteration in self.temp: @@ -146,7 +78,7 @@ class Parser: return sets async def urls(self): - found = re.finditer(r'(http|https)://(www\.)?trello.com/([a-zA-Z0-9\-_\.]+/?)*', self.results) + found = re.finditer(r'(http|https)://(www\.)?trello.com/([a-zA-Z\d\-_\.]+/?)*', self.results) urls = {match.group().strip() for match in found} return urls