From 7f973f9e3e6c3d95165fe5238f6d0b4667bfae2d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 17 Dec 2018 19:38:18 +0000 Subject: [PATCH] Refactor the ResourceAccessor so it can be used by layers. --- volatility/framework/layers/__init__.py | 175 +----------------- volatility/framework/layers/physical.py | 5 +- volatility/framework/layers/resources.py | 173 +++++++++++++++++ volatility/framework/layers/vmware.py | 25 ++- .../framework/plugins/windows/strings.py | 6 +- .../framework/plugins/windows/vadyarascan.py | 5 +- volatility/framework/plugins/yarascan.py | 5 +- volatility/framework/symbols/intermed.py | 6 +- 8 files changed, 208 insertions(+), 192 deletions(-) create mode 100644 volatility/framework/layers/resources.py diff --git a/volatility/framework/layers/__init__.py b/volatility/framework/layers/__init__.py index 01ac71d74..714b5cb77 100644 --- a/volatility/framework/layers/__init__.py +++ b/volatility/framework/layers/__init__.py @@ -1,174 +1 @@ -import bz2 -import contextlib -import gzip -import hashlib -import logging -import lzma -import os -import ssl -import urllib.parse -import urllib.request -import zipfile -from typing import List, Optional - -try: - import magic - - HAS_MAGIC = True -except ImportError: - HAS_MAGIC = False - -try: - import smb.SMBHandler -except ImportError: - pass - -from volatility import framework -from volatility.framework import constants, validity -from volatility.framework.layers import intel, lime, physical, segmented, vmware, crash - -vollog = logging.getLogger(__name__) - -# TODO: Type-annotating the ResourceAccessor.open method is difficult because HTTPResponse is not actually an IO[Any] type -# fix this - - -class ResourceAccessor(object): - """Object for openning URLs as files (downloading locally first if necessary)""" - - def __init__(self, - progress_callback: Optional[validity.ProgressCallback] = None, - context: Optional[ssl.SSLContext] = None) -> None: - """Creates a resource accessor - - Note: context is an SSL context, not a volatility context - """ - self._progress_callback = progress_callback - self._context = context - self._cached_files = [] # type: List[str] - self._handlers = list(framework.class_subclasses(urllib.request.BaseHandler)) - vollog.log(constants.LOGLEVEL_VVV, - "Available URL handlers: {}".format(", ".join([x.__name__ for x in self._handlers]))) - - def open(self, url, mode = "rb"): - """Returns a file-like object for a particular URL opened in mode""" - urllib.request.install_opener(urllib.request.build_opener(*self._handlers)) - - with contextlib.closing(urllib.request.urlopen(url, context = self._context)) as fp: - # Cache the file locally - parsed_url = urllib.parse.urlparse(url) - - if parsed_url.scheme == 'file': - # ZipExtFiles (files in zips) cannot seek, so must be cached in order to use and/or decompress - curfile = urllib.request.urlopen(url, context = self._context) - else: - # TODO: find a way to check if we already have this file (look at http headers?) - block_size = 1028 * 8 - temp_filename = os.path.join(constants.CACHE_PATH, - "data_" + hashlib.sha512(bytes(url, 'latin-1')).hexdigest()) - - if temp_filename not in self._cached_files or not os.path.exists(temp_filename): - vollog.info("Caching file at: {}".format(temp_filename)) - - try: - content_length = fp.info().get('Content-Length', -1) - except AttributeError: - # If our fp doesn't have an info member, carry on gracefully - content_length = -1 - cache_file = open(temp_filename, "wb") - - count = 0 - while True: - block = fp.read(block_size) - count += len(block) - if not block: - break - if self._progress_callback: - self._progress_callback(count / max(count, int(content_length)), - "Reading file {}".format(url)) - cache_file.write(block) - cache_file.close() - # Globally stash the file as cached this python session - self._cached_files += [temp_filename] - # Re-open the cache with a different mode - curfile = open(temp_filename, mode = "rb") - - # Determine whether the file is a particular type of file, and if so, open it as such - IMPORTED_MAGIC = False - if HAS_MAGIC: - while True: - detected = None - try: - # Detect the content - detected = magic.detect_from_fobj(curfile) - IMPORTED_MAGIC = True - # This is because python-magic and file provide a magic module - # Only file's python has magic.detect_from_fobj - except AttributeError: - pass - except: - pass - - if detected: - if detected.mime_type == 'application/x-xz': - curfile = lzma.LZMAFile(curfile, mode) - elif detected.mime_type == 'application/x-bzip2': - curfile = bz2.BZ2File(curfile, mode) - elif detected.mime_type == 'application/x-gzip': - curfile = gzip.GzipFile(fileobj = curfile, mode = mode) - else: - break - else: - break - - # Read and rewind to ensure we're inside any compressed file layers - curfile.read(1) - curfile.seek(0) - if not IMPORTED_MAGIC: - # Somewhat of a hack, but prevents a hard dependency on the magic module - url_path = parsed_url.path - while True: - if url_path.endswith(".xz"): - curfile = lzma.LZMAFile(curfile, mode) - elif url_path.endswith(".bz2"): - curfile = bz2.BZ2File(curfile, mode) - elif url_path.endswith(".gz"): - curfile = gzip.GzipFile(fileobj = curfile, mode = mode) - else: - break - url_path = ".".join(url_path.split(".")[:-1]) - - # Fallback in case the file doesn't exist - if curfile is None: - raise ValueError("URL does not reference an openable file") - return curfile - - -class JarHandler(urllib.request.BaseHandler): - """Handles the jar scheme for URIs - - Reference used for the schema syntax: - http://docs.netkernel.org/book/view/book:mod:reference/doc:layer1:schemes:jar - - Actual reference (found from https://www.w3.org/wiki/UriSchemes/jar) seemed not to return: - http://developer.java.sun.com/developer/onlineTraining/protocolhandlers/ - """ - - @staticmethod - def default_open(req): - """Handles the request if it's the jar scheme""" - if req.type == 'jar': - subscheme, remainder = req.full_url.split(":")[1], ":".join(req.full_url.split(":")[2:]) - if subscheme != 'file': - vollog.log(constants.LOGLEVEL_VVV, "Unsupported jar subscheme {}".format(subscheme)) - return None - - zipsplit = remainder.split("!") - if len(zipsplit) != 2: - vollog.log(constants.LOGLEVEL_VVV, - "Path did not contain exactly one fragment indicator: {}".format(remainder)) - return None - - zippath, filepath = zipsplit - return zipfile.ZipFile(zippath).open(filepath) - return None +from volatility.framework.layers import resources, intel, lime, physical, segmented, vmware, crash diff --git a/volatility/framework/layers/physical.py b/volatility/framework/layers/physical.py index 75a569f60..0574362fa 100644 --- a/volatility/framework/layers/physical.py +++ b/volatility/framework/layers/physical.py @@ -1,7 +1,8 @@ from typing import Any, Dict, IO, List, Optional -from volatility.framework import exceptions, interfaces, layers +from volatility.framework import exceptions, interfaces from volatility.framework.configuration import requirements +from volatility.framework.layers import resources class BufferDataLayer(interfaces.layers.DataLayerInterface): @@ -70,7 +71,7 @@ class FileLayer(interfaces.layers.DataLayerInterface): super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) self._location = self.config["location"] - self._accessor = layers.ResourceAccessor() + self._accessor = resources.ResourceAccessor() self._file_ = None # type: Optional[IO[Any]] self._size = None # type: Optional[int] # Instantiate the file to throw exceptions if the file doesn't open diff --git a/volatility/framework/layers/resources.py b/volatility/framework/layers/resources.py new file mode 100644 index 000000000..f8416bdb0 --- /dev/null +++ b/volatility/framework/layers/resources.py @@ -0,0 +1,173 @@ +import bz2 +import contextlib +import gzip +import hashlib +import logging +import lzma +import os +import ssl +import urllib.parse +import urllib.request +import zipfile +from typing import Optional + +from volatility import framework +from volatility.framework import validity, constants + +try: + import magic + + HAS_MAGIC = True +except ImportError: + HAS_MAGIC = False + +try: + import smb.SMBHandler +except ImportError: + pass + +vollog = logging.getLogger(__name__) + +# TODO: Type-annotating the ResourceAccessor.open method is difficult because HTTPResponse is not actually an IO[Any] type +# fix this + + +class ResourceAccessor(object): + """Object for openning URLs as files (downloading locally first if necessary)""" + + def __init__(self, + progress_callback: Optional[validity.ProgressCallback] = None, + context: Optional[ssl.SSLContext] = None) -> None: + """Creates a resource accessor + + Note: context is an SSL context, not a volatility context + """ + self._progress_callback = progress_callback + self._context = context + self._cached_files = [] # type: List[str] + self._handlers = list(framework.class_subclasses(urllib.request.BaseHandler)) + vollog.log(constants.LOGLEVEL_VVV, + "Available URL handlers: {}".format(", ".join([x.__name__ for x in self._handlers]))) + + def open(self, url, mode = "rb"): + """Returns a file-like object for a particular URL opened in mode""" + urllib.request.install_opener(urllib.request.build_opener(*self._handlers)) + + with contextlib.closing(urllib.request.urlopen(url, context = self._context)) as fp: + # Cache the file locally + parsed_url = urllib.parse.urlparse(url) + + if parsed_url.scheme == 'file': + # ZipExtFiles (files in zips) cannot seek, so must be cached in order to use and/or decompress + curfile = urllib.request.urlopen(url, context = self._context) + else: + # TODO: find a way to check if we already have this file (look at http headers?) + block_size = 1028 * 8 + temp_filename = os.path.join(constants.CACHE_PATH, + "data_" + hashlib.sha512(bytes(url, 'latin-1')).hexdigest()) + + if temp_filename not in self._cached_files or not os.path.exists(temp_filename): + vollog.info("Caching file at: {}".format(temp_filename)) + + try: + content_length = fp.info().get('Content-Length', -1) + except AttributeError: + # If our fp doesn't have an info member, carry on gracefully + content_length = -1 + cache_file = open(temp_filename, "wb") + + count = 0 + while True: + block = fp.read(block_size) + count += len(block) + if not block: + break + if self._progress_callback: + self._progress_callback(count / max(count, int(content_length)), + "Reading file {}".format(url)) + cache_file.write(block) + cache_file.close() + # Globally stash the file as cached this python session + self._cached_files += [temp_filename] + # Re-open the cache with a different mode + curfile = open(temp_filename, mode = "rb") + + # Determine whether the file is a particular type of file, and if so, open it as such + IMPORTED_MAGIC = False + if HAS_MAGIC: + while True: + detected = None + try: + # Detect the content + detected = magic.detect_from_fobj(curfile) + IMPORTED_MAGIC = True + # This is because python-magic and file provide a magic module + # Only file's python has magic.detect_from_fobj + except AttributeError: + pass + except: + pass + + if detected: + if detected.mime_type == 'application/x-xz': + curfile = lzma.LZMAFile(curfile, mode) + elif detected.mime_type == 'application/x-bzip2': + curfile = bz2.BZ2File(curfile, mode) + elif detected.mime_type == 'application/x-gzip': + curfile = gzip.GzipFile(fileobj = curfile, mode = mode) + else: + break + else: + break + + # Read and rewind to ensure we're inside any compressed file layers + curfile.read(1) + curfile.seek(0) + if not IMPORTED_MAGIC: + # Somewhat of a hack, but prevents a hard dependency on the magic module + url_path = parsed_url.path + while True: + if url_path.endswith(".xz"): + curfile = lzma.LZMAFile(curfile, mode) + elif url_path.endswith(".bz2"): + curfile = bz2.BZ2File(curfile, mode) + elif url_path.endswith(".gz"): + curfile = gzip.GzipFile(fileobj = curfile, mode = mode) + else: + break + url_path = ".".join(url_path.split(".")[:-1]) + + # Fallback in case the file doesn't exist + if curfile is None: + raise ValueError("URL does not reference an openable file") + return curfile + + +class JarHandler(urllib.request.BaseHandler): + """Handles the jar scheme for URIs + + Reference used for the schema syntax: + http://docs.netkernel.org/book/view/book:mod:reference/doc:layer1:schemes:jar + + Actual reference (found from https://www.w3.org/wiki/UriSchemes/jar) seemed not to return: + http://developer.java.sun.com/developer/onlineTraining/protocolhandlers/ + """ + + @staticmethod + def default_open(req): + """Handles the request if it's the jar scheme""" + if req.type == 'jar': + subscheme, remainder = req.full_url.split(":")[1], ":".join(req.full_url.split(":")[2:]) + if subscheme != 'file': + vollog.log(constants.LOGLEVEL_VVV, "Unsupported jar subscheme {}".format(subscheme)) + return None + + zipsplit = remainder.split("!") + if len(zipsplit) != 2: + vollog.log(constants.LOGLEVEL_VVV, + "Path did not contain exactly one fragment indicator: {}".format(remainder)) + return None + + zippath, filepath = zipsplit + return zipfile.ZipFile(zippath).open(filepath) + return None diff --git a/volatility/framework/layers/vmware.py b/volatility/framework/layers/vmware.py index 2258eede2..a40ab4484 100644 --- a/volatility/framework/layers/vmware.py +++ b/volatility/framework/layers/vmware.py @@ -1,10 +1,9 @@ -import os import struct from typing import Any, Dict, List, Optional from volatility.framework import interfaces, validity from volatility.framework.configuration import requirements -from volatility.framework.layers import physical, segmented +from volatility.framework.layers import physical, segmented, resources from volatility.framework.symbols import native @@ -123,13 +122,25 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface): current_layer_name = context.memory.free_layer_name("VmwareMetaLayer") current_config_path = interfaces.configuration.path_join("automagic", "layer_stacker", "stack", current_layer_name) - if os.path.exists(vmss): + + try: + _ = resources.ResourceAccessor().open(vmss).read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmss context.memory.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) - elif os.path.exists(vmsn): - context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmsn - context.memory.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) - else: + vmss_success = True + except IOError: + vmss_success = False + + if not vmss_success: + try: + _ = resources.ResourceAccessor().open(vmsn).read(10) + context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmsn + context.memory.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) + vmsn_success = True + except IOError: + vmsn_success = False + + if not vmss_success and not vmsn_success: return None new_layer_name = context.memory.free_layer_name("VmwareLayer") context.config[interfaces.configuration.path_join(current_config_path, "base_layer")] = layer_name diff --git a/volatility/framework/plugins/windows/strings.py b/volatility/framework/plugins/windows/strings.py index 234932b19..fb9aa3746 100644 --- a/volatility/framework/plugins/windows/strings.py +++ b/volatility/framework/plugins/windows/strings.py @@ -2,9 +2,9 @@ import logging import re from typing import Dict, Generator, List, Set, Tuple -from volatility.framework import interfaces, renderers, layers +from volatility.framework import interfaces, renderers from volatility.framework.configuration import requirements -from volatility.framework.layers import intel +from volatility.framework.layers import intel, resources from volatility.framework.renderers import format_hints from volatility.plugins.windows import pslist @@ -32,7 +32,7 @@ class Strings(interfaces.plugins.PluginInterface): """Generates results from a strings file""" revmap = self.generate_mapping(self.config['primary']) - accessor = layers.ResourceAccessor() + accessor = resources.ResourceAccessor() for line in accessor.open(self.config['strings_file'], "rb").readlines(): try: diff --git a/volatility/framework/plugins/windows/vadyarascan.py b/volatility/framework/plugins/windows/vadyarascan.py index d3b80a271..9509ec8f8 100644 --- a/volatility/framework/plugins/windows/vadyarascan.py +++ b/volatility/framework/plugins/windows/vadyarascan.py @@ -1,8 +1,9 @@ import logging from typing import Any, Iterable, List, Tuple -from volatility.framework import interfaces, layers, renderers +from volatility.framework import interfaces, renderers from volatility.framework.configuration import requirements +from volatility.framework.layers import resources from volatility.framework.renderers import format_hints from volatility.framework.symbols.windows import extensions from volatility.plugins import yarascan @@ -50,7 +51,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): rule += " wide ascii" rules = yara.compile(sources = {'n': 'rule r1 {{strings: $a = {} condition: $a}}'.format(rule)}) elif self.config.get('yara_file', None) is not None: - rules = yara.compile(file = layers.ResourceAccessor().open(self.config['yara_file'], "rb")) + rules = yara.compile(file = resources.ResourceAccessor().open(self.config['yara_file'], "rb")) else: vollog.error("No yara rules, nor yara rules file were specified") diff --git a/volatility/framework/plugins/yarascan.py b/volatility/framework/plugins/yarascan.py index ea392ec1e..ebc98d153 100644 --- a/volatility/framework/plugins/yarascan.py +++ b/volatility/framework/plugins/yarascan.py @@ -1,9 +1,10 @@ import logging from typing import Iterable, Tuple, List -from volatility.framework import interfaces, renderers, layers +from volatility.framework import interfaces, renderers from volatility.framework.configuration import requirements from volatility.framework.interfaces import plugins +from volatility.framework.layers import resources from volatility.framework.renderers import format_hints vollog = logging.getLogger(__name__) @@ -71,7 +72,7 @@ class YaraScan(plugins.PluginInterface): rule += " wide ascii" rules = yara.compile(sources = {'n': 'rule r1 {{strings: $a = {} condition: $a}}'.format(rule)}) elif self.config.get('yara_file', None) is not None: - rules = yara.compile(file = layers.ResourceAccessor().open(self.config['yara_file'], "rb")) + rules = yara.compile(file = resources.ResourceAccessor().open(self.config['yara_file'], "rb")) else: vollog.error("No yara rules, nor yara rules file were specified") diff --git a/volatility/framework/symbols/intermed.py b/volatility/framework/symbols/intermed.py index 42a21d00f..a59cf097e 100644 --- a/volatility/framework/symbols/intermed.py +++ b/volatility/framework/symbols/intermed.py @@ -10,8 +10,10 @@ from abc import ABCMeta from typing import Any, Dict, Generator, Iterable, List, Optional, Type, Tuple import volatility +import volatility.framework.layers.resources from volatility import schemas, symbols -from volatility.framework import class_subclasses, constants, exceptions, interfaces, objects, layers +from volatility.framework import class_subclasses, constants, exceptions, interfaces, objects +from volatility.framework.layers import physical from volatility.framework.configuration import requirements from volatility.framework.symbols import native, metadata @@ -79,7 +81,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): # Check there are no obvious errors # Open the file and test the version self._versions = dict([(x.version, x) for x in class_subclasses(ISFormatTable)]) - fp = layers.ResourceAccessor().open(isf_url) + fp = volatility.framework.layers.resources.ResourceAccessor().open(isf_url) reader = codecs.getreader("utf-8") json_object = json.load(reader(fp)) # type: ignore fp.close()