From 4d48dcbb340faff2064aea787e0dc0b86f6b3c0a Mon Sep 17 00:00:00 2001 From: L1ghtn1ng Date: Tue, 11 Feb 2020 19:10:35 +0000 Subject: [PATCH] misc fixes and working on new module --- CONTRIBUTING.md => README/CONTRIBUTING.md | 0 COPYING => README/COPYING | 0 LICENSES => README/LICENSES | 2 +- tests/discovery/githubcode.py | 2 +- theHarvester/__main__.py | 22 +++++++++------ theHarvester/discovery/__init__.py | 1 + theHarvester/discovery/bufferoverun.py | 34 +++++++++++++++++++++++ theHarvester/discovery/constants.py | 2 -- theHarvester/discovery/dnsdumpster.py | 2 +- theHarvester/discovery/githubcode.py | 2 ++ theHarvester/lib/core.py | 3 +- theHarvester/lib/reportgraph.py | 1 - 12 files changed, 55 insertions(+), 16 deletions(-) rename CONTRIBUTING.md => README/CONTRIBUTING.md (100%) rename COPYING => README/COPYING (100%) rename LICENSES => README/LICENSES (90%) create mode 100644 theHarvester/discovery/bufferoverun.py diff --git a/CONTRIBUTING.md b/README/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to README/CONTRIBUTING.md diff --git a/COPYING b/README/COPYING similarity index 100% rename from COPYING rename to README/COPYING diff --git a/LICENSES b/README/LICENSES similarity index 90% rename from LICENSES rename to README/LICENSES index 0dab1406..279d496a 100644 --- a/LICENSES +++ b/README/LICENSES @@ -1,6 +1,6 @@ Released under the GPL v 2.0. -If you did not recieve a copy of the GPL, try http://www.gnu.org/. +If you did not receive a copy of the GPL, try http://www.gnu.org/. Copyright 2011 Christian Martorella diff --git a/tests/discovery/githubcode.py b/tests/discovery/githubcode.py index c61f557b..b8f358f9 100644 --- a/tests/discovery/githubcode.py +++ b/tests/discovery/githubcode.py @@ -2,7 +2,7 @@ from theHarvester.discovery import githubcode from theHarvester.discovery.githubcode import RetryResult, ErrorResult, SuccessResult from theHarvester.discovery.constants import MissingKey from theHarvester.lib.core import Core -from mock import MagicMock +from unittest.mock import MagicMock from requests import Response import pytest diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 9bc99413..e69c972c 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -24,21 +24,17 @@ async def start(): parser.add_argument('-d', '--domain', help='company name or domain to search', required=True) parser.add_argument('-l', '--limit', help='limit the number of search results, default=500', default=500, type=int) parser.add_argument('-S', '--start', help='start with result number X, default=0', default=0, type=int) - parser.add_argument('-g', '--google-dork', help='use Google Dorks for Google search', default=False, - action='store_true') - parser.add_argument('-p', '--proxies', help='use proxies for requests, enter proxies in proxies.yaml', - default=False, action='store_true') - parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False, - action='store_true') - parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts', - action='store_const', const='basic', default=False) + parser.add_argument('-g', '--google-dork', help='use Google Dorks for Google search', default=False, action='store_true') + parser.add_argument('-p', '--proxies', help='use proxies for requests, enter proxies in proxies.yaml', default=False, action='store_true') + parser.add_argument('-s', '--shodan', help='use Shodan to query discovered hosts', default=False, action='store_true') + parser.add_argument('-v', '--virtual-host', help='verify host name via DNS resolution and search for virtual hosts', action='store_const', const='basic', default=False) parser.add_argument('-e', '--dns-server', help='DNS server to use for lookup') parser.add_argument('-t', '--dns-tld', help='perform a DNS TLD expansion discovery, default False', default=False) parser.add_argument('-r', '--take-over', help='Check for takeovers', default=False, action='store_true') parser.add_argument('-n', '--dns-lookup', help='enable DNS server lookup, default False', default=False, action='store_true') parser.add_argument('-c', '--dns-brute', help='perform a DNS brute force on the domain', default=False, action='store_true') parser.add_argument('-f', '--filename', help='save the results to an HTML and/or XML file', default='', type=str) - parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, certspotter, crtsh, dnsdumpster, + parser.add_argument('-b', '--source', help='''baidu, bing, bingapi, bufferoverun, certspotter, crtsh, dnsdumpster, dogpile, duckduckgo, exalead, github-code, google, hunter, intelx, linkedin, linkedin_links, netcraft, otx, securityTrails, spyse, threatcrowd, @@ -186,6 +182,14 @@ async def start(): else: print(e) + elif engineitem == 'bufferoverun': + from theHarvester.discovery import bufferoverun + try: + bufferoverun_search = bufferoverun.SearchBufferover(word) + stor_lst.append(store(bufferoverun_search, engineitem, store_host=True, store_ip=True)) + except Exception as e: + print(e) + elif engineitem == 'certspotter': from theHarvester.discovery import certspottersearch try: diff --git a/theHarvester/discovery/__init__.py b/theHarvester/discovery/__init__.py index 35c2a131..539ab694 100644 --- a/theHarvester/discovery/__init__.py +++ b/theHarvester/discovery/__init__.py @@ -1,5 +1,6 @@ __all__ = ['baidusearch', 'bingsearch', + 'bufferoverun', 'crtsh', 'certspottersearch', 'dnssearch', diff --git a/theHarvester/discovery/bufferoverun.py b/theHarvester/discovery/bufferoverun.py new file mode 100644 index 00000000..4420285c --- /dev/null +++ b/theHarvester/discovery/bufferoverun.py @@ -0,0 +1,34 @@ +from theHarvester.lib.core import * +import re + + +class SearchBufferover: + + def __init__(self, word): + self.word = word + self.totalhosts = set() + self.totalips = set() + self.proxy = False + + async def do_search(self): + url = f'https://dns.bufferover.run/dns?q={self.word}' + headers = {'User-Agent': Core.get_user_agent()} + client = aiohttp.ClientSession(headers=headers, timeout=aiohttp.ClientTimeout(total=20)) + responses = await AsyncFetcher.fetch(client, url, json=True, proxy=self.proxy) + await client.close() + + dct = responses + self.totalhosts: set = {host for host in dct['FDNS_A']} + # filter out ips that are just called NXDOMAIN + self.totalips: set = {ip['address'] for ip in dct['FDNS_A'] + if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip['FDNS_A'])} + + async def get_hostnames(self) -> set: + return self.totalhosts + + async def get_ips(self) -> set: + return self.totalips + + async def process(self, proxy=False): + self.proxy = proxy + await self.do_search() diff --git a/theHarvester/discovery/constants.py b/theHarvester/discovery/constants.py index 2bb30d35..b22e1d4c 100644 --- a/theHarvester/discovery/constants.py +++ b/theHarvester/discovery/constants.py @@ -1,8 +1,6 @@ from theHarvester.lib.core import * from typing import Union import random -import aiohttp -import asyncio googleUA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 ' \ 'Safari/537.36 ' diff --git a/theHarvester/discovery/dnsdumpster.py b/theHarvester/discovery/dnsdumpster.py index 439385bb..d9202009 100644 --- a/theHarvester/discovery/dnsdumpster.py +++ b/theHarvester/discovery/dnsdumpster.py @@ -34,7 +34,7 @@ class SearchDnsDumpster: self.results = await resp.text() await session.close() except Exception as e: - print(f'An exception occured: {e}') + print(f'An exception occurred: {e}') self.totalresults += self.results async def get_hostnames(self): diff --git a/theHarvester/discovery/githubcode.py b/theHarvester/discovery/githubcode.py index cf21dae5..b9d3383c 100644 --- a/theHarvester/discovery/githubcode.py +++ b/theHarvester/discovery/githubcode.py @@ -7,6 +7,7 @@ import aiohttp import urllib.parse as urlparse import random + class RetryResult(NamedTuple): time: float @@ -96,6 +97,7 @@ class SearchGithubCode: else: async with sess.get(url, ) as resp: return await resp.text(), await resp.json(), resp.status, resp.links + @staticmethod async def next_page_or_end(result: SuccessResult) -> Optional[int]: if result.next_page is not None: diff --git a/theHarvester/lib/core.py b/theHarvester/lib/core.py index b51db7b5..95f2d1bb 100644 --- a/theHarvester/lib/core.py +++ b/theHarvester/lib/core.py @@ -10,7 +10,7 @@ import random class Core: @staticmethod def version() -> str: - return '3.1.1dev5' + return '3.2.0dev0' @staticmethod def bing_key() -> str: @@ -123,6 +123,7 @@ class Core: supportedengines = {'baidu', 'bing', 'bingapi', + 'bufferoverun', 'certspotter', 'crtsh', 'dnsdumpster', diff --git a/theHarvester/lib/reportgraph.py b/theHarvester/lib/reportgraph.py index 7d88ddb3..b05d0ad6 100644 --- a/theHarvester/lib/reportgraph.py +++ b/theHarvester/lib/reportgraph.py @@ -4,7 +4,6 @@ import plotly import plotly.graph_objs as go - class GraphGenerator: def __init__(self, domain):