Refactor the ResourceAccessor so it can be used by layers.

volatility/framework/layers/__init__.py

@@ -1,174 +1 @@
-import bz2
-import contextlib
-import gzip
-import hashlib
-import logging
-import lzma
-import os
-import ssl
-import urllib.parse
-import urllib.request
-import zipfile
-from typing import List, Optional
-
-try:
-    import magic
-
-    HAS_MAGIC = True
-except ImportError:
-    HAS_MAGIC = False
-
-try:
-    import smb.SMBHandler
-except ImportError:
-    pass
-
-from volatility import framework
-from volatility.framework import constants, validity
-from volatility.framework.layers import intel, lime, physical, segmented, vmware, crash
-
-vollog = logging.getLogger(__name__)
-
-# TODO: Type-annotating the ResourceAccessor.open method is difficult because HTTPResponse is not actually an IO[Any] type
-#   fix this
-
-
-class ResourceAccessor(object):
-    """Object for openning URLs as files (downloading locally first if necessary)"""
-
-    def __init__(self,
-                 progress_callback: Optional[validity.ProgressCallback] = None,
-                 context: Optional[ssl.SSLContext] = None) -> None:
-        """Creates a resource accessor
-
-        Note: context is an SSL context, not a volatility context
-        """
-        self._progress_callback = progress_callback
-        self._context = context
-        self._cached_files = []  # type: List[str]
-        self._handlers = list(framework.class_subclasses(urllib.request.BaseHandler))
-        vollog.log(constants.LOGLEVEL_VVV,
-                   "Available URL handlers: {}".format(", ".join([x.__name__ for x in self._handlers])))
-
-    def open(self, url, mode = "rb"):
-        """Returns a file-like object for a particular URL opened in mode"""
-        urllib.request.install_opener(urllib.request.build_opener(*self._handlers))
-
-        with contextlib.closing(urllib.request.urlopen(url, context = self._context)) as fp:
-            # Cache the file locally
-            parsed_url = urllib.parse.urlparse(url)
-
-            if parsed_url.scheme == 'file':
-                # ZipExtFiles (files in zips) cannot seek, so must be cached in order to use and/or decompress
-                curfile = urllib.request.urlopen(url, context = self._context)
-            else:
-                # TODO: find a way to check if we already have this file (look at http headers?)
-                block_size = 1028 * 8
-                temp_filename = os.path.join(constants.CACHE_PATH,
-                                             "data_" + hashlib.sha512(bytes(url, 'latin-1')).hexdigest())
-
-                if temp_filename not in self._cached_files or not os.path.exists(temp_filename):
-                    vollog.info("Caching file at: {}".format(temp_filename))
-
-                    try:
-                        content_length = fp.info().get('Content-Length', -1)
-                    except AttributeError:
-                        # If our fp doesn't have an info member, carry on gracefully
-                        content_length = -1
-                    cache_file = open(temp_filename, "wb")
-
-                    count = 0
-                    while True:
-                        block = fp.read(block_size)
-                        count += len(block)
-                        if not block:
-                            break
-                        if self._progress_callback:
-                            self._progress_callback(count / max(count, int(content_length)),
-                                                    "Reading file {}".format(url))
-                        cache_file.write(block)
-                    cache_file.close()
-                    # Globally stash the file as cached this python session
-                    self._cached_files += [temp_filename]
-                # Re-open the cache with a different mode
-                curfile = open(temp_filename, mode = "rb")
-
-        # Determine whether the file is a particular type of file, and if so, open it as such
-        IMPORTED_MAGIC = False
-        if HAS_MAGIC:
-            while True:
-                detected = None
-                try:
-                    # Detect the content
-                    detected = magic.detect_from_fobj(curfile)
-                    IMPORTED_MAGIC = True
-                    # This is because python-magic and file provide a magic module
-                    # Only file's python has magic.detect_from_fobj
-                except AttributeError:
-                    pass
-                except:
-                    pass
-
-                if detected:
-                    if detected.mime_type == 'application/x-xz':
-                        curfile = lzma.LZMAFile(curfile, mode)
-                    elif detected.mime_type == 'application/x-bzip2':
-                        curfile = bz2.BZ2File(curfile, mode)
-                    elif detected.mime_type == 'application/x-gzip':
-                        curfile = gzip.GzipFile(fileobj = curfile, mode = mode)
-                    else:
-                        break
-                else:
-                    break
-
-                # Read and rewind to ensure we're inside any compressed file layers
-                curfile.read(1)
-                curfile.seek(0)
-        if not IMPORTED_MAGIC:
-            # Somewhat of a hack, but prevents a hard dependency on the magic module
-            url_path = parsed_url.path
-            while True:
-                if url_path.endswith(".xz"):
-                    curfile = lzma.LZMAFile(curfile, mode)
-                elif url_path.endswith(".bz2"):
-                    curfile = bz2.BZ2File(curfile, mode)
-                elif url_path.endswith(".gz"):
-                    curfile = gzip.GzipFile(fileobj = curfile, mode = mode)
-                else:
-                    break
-                url_path = ".".join(url_path.split(".")[:-1])
-
-        # Fallback in case the file doesn't exist
-        if curfile is None:
-            raise ValueError("URL does not reference an openable file")
-        return curfile
-
-
-class JarHandler(urllib.request.BaseHandler):
-    """Handles the jar scheme for URIs
-
-    Reference used for the schema syntax:
-    http://docs.netkernel.org/book/view/book:mod:reference/doc:layer1:schemes:jar
-
-    Actual reference (found from https://www.w3.org/wiki/UriSchemes/jar) seemed not to return:
-    http://developer.java.sun.com/developer/onlineTraining/protocolhandlers/
-    """
-
-    @staticmethod
-    def default_open(req):
-        """Handles the request if it's the jar scheme"""
-        if req.type == 'jar':
-            subscheme, remainder = req.full_url.split(":")[1], ":".join(req.full_url.split(":")[2:])
-            if subscheme != 'file':
-                vollog.log(constants.LOGLEVEL_VVV, "Unsupported jar subscheme {}".format(subscheme))
-                return None
-
-            zipsplit = remainder.split("!")
-            if len(zipsplit) != 2:
-                vollog.log(constants.LOGLEVEL_VVV,
-                           "Path did not contain exactly one fragment indicator: {}".format(remainder))
-                return None
-
-            zippath, filepath = zipsplit
-            return zipfile.ZipFile(zippath).open(filepath)
-        return None
+from volatility.framework.layers import resources, intel, lime, physical, segmented, vmware, crash

volatility/framework/layers/physical.py

@@ -1,7 +1,8 @@
 from typing import Any, Dict, IO, List, Optional
 
-from volatility.framework import exceptions, interfaces, layers
+from volatility.framework import exceptions, interfaces
 from volatility.framework.configuration import requirements
+from volatility.framework.layers import resources
 
 
 class BufferDataLayer(interfaces.layers.DataLayerInterface):
@@ -70,7 +71,7 @@ class FileLayer(interfaces.layers.DataLayerInterface):
         super().__init__(context = context, config_path = config_path, name = name, metadata = metadata)
 
         self._location = self.config["location"]
-        self._accessor = layers.ResourceAccessor()
+        self._accessor = resources.ResourceAccessor()
         self._file_ = None  # type: Optional[IO[Any]]
         self._size = None  # type: Optional[int]
         # Instantiate the file to throw exceptions if the file doesn't open

volatility/framework/layers/resources.py

@@ -0,0 +1,173 @@
+import bz2
+import contextlib
+import gzip
+import hashlib
+import logging
+import lzma
+import os
+import ssl
+import urllib.parse
+import urllib.request
+import zipfile
+from typing import Optional
+
+from volatility import framework
+from volatility.framework import validity, constants
+
+try:
+    import magic
+
+    HAS_MAGIC = True
+except ImportError:
+    HAS_MAGIC = False
+
+try:
+    import smb.SMBHandler
+except ImportError:
+    pass
+
+vollog = logging.getLogger(__name__)
+
+# TODO: Type-annotating the ResourceAccessor.open method is difficult because HTTPResponse is not actually an IO[Any] type
+#   fix this
+
+
+class ResourceAccessor(object):
+    """Object for openning URLs as files (downloading locally first if necessary)"""
+
+    def __init__(self,
+                 progress_callback: Optional[validity.ProgressCallback] = None,
+                 context: Optional[ssl.SSLContext] = None) -> None:
+        """Creates a resource accessor
+
+        Note: context is an SSL context, not a volatility context
+        """
+        self._progress_callback = progress_callback
+        self._context = context
+        self._cached_files = []  # type: List[str]
+        self._handlers = list(framework.class_subclasses(urllib.request.BaseHandler))
+        vollog.log(constants.LOGLEVEL_VVV,
+                   "Available URL handlers: {}".format(", ".join([x.__name__ for x in self._handlers])))
+
+    def open(self, url, mode = "rb"):
+        """Returns a file-like object for a particular URL opened in mode"""
+        urllib.request.install_opener(urllib.request.build_opener(*self._handlers))
+
+        with contextlib.closing(urllib.request.urlopen(url, context = self._context)) as fp:
+            # Cache the file locally
+            parsed_url = urllib.parse.urlparse(url)
+
+            if parsed_url.scheme == 'file':
+                # ZipExtFiles (files in zips) cannot seek, so must be cached in order to use and/or decompress
+                curfile = urllib.request.urlopen(url, context = self._context)
+            else:
+                # TODO: find a way to check if we already have this file (look at http headers?)
+                block_size = 1028 * 8
+                temp_filename = os.path.join(constants.CACHE_PATH,
+                                             "data_" + hashlib.sha512(bytes(url, 'latin-1')).hexdigest())
+
+                if temp_filename not in self._cached_files or not os.path.exists(temp_filename):
+                    vollog.info("Caching file at: {}".format(temp_filename))
+
+                    try:
+                        content_length = fp.info().get('Content-Length', -1)
+                    except AttributeError:
+                        # If our fp doesn't have an info member, carry on gracefully
+                        content_length = -1
+                    cache_file = open(temp_filename, "wb")
+
+                    count = 0
+                    while True:
+                        block = fp.read(block_size)
+                        count += len(block)
+                        if not block:
+                            break
+                        if self._progress_callback:
+                            self._progress_callback(count / max(count, int(content_length)),
+                                                    "Reading file {}".format(url))
+                        cache_file.write(block)
+                    cache_file.close()
+                    # Globally stash the file as cached this python session
+                    self._cached_files += [temp_filename]
+                # Re-open the cache with a different mode
+                curfile = open(temp_filename, mode = "rb")
+
+        # Determine whether the file is a particular type of file, and if so, open it as such
+        IMPORTED_MAGIC = False
+        if HAS_MAGIC:
+            while True:
+                detected = None
+                try:
+                    # Detect the content
+                    detected = magic.detect_from_fobj(curfile)
+                    IMPORTED_MAGIC = True
+                    # This is because python-magic and file provide a magic module
+                    # Only file's python has magic.detect_from_fobj
+                except AttributeError:
+                    pass
+                except:
+                    pass
+
+                if detected:
+                    if detected.mime_type == 'application/x-xz':
+                        curfile = lzma.LZMAFile(curfile, mode)
+                    elif detected.mime_type == 'application/x-bzip2':
+                        curfile = bz2.BZ2File(curfile, mode)
+                    elif detected.mime_type == 'application/x-gzip':
+                        curfile = gzip.GzipFile(fileobj = curfile, mode = mode)
+                    else:
+                        break
+                else:
+                    break
+
+                # Read and rewind to ensure we're inside any compressed file layers
+                curfile.read(1)
+                curfile.seek(0)
+        if not IMPORTED_MAGIC:
+            # Somewhat of a hack, but prevents a hard dependency on the magic module
+            url_path = parsed_url.path
+            while True:
+                if url_path.endswith(".xz"):
+                    curfile = lzma.LZMAFile(curfile, mode)
+                elif url_path.endswith(".bz2"):
+                    curfile = bz2.BZ2File(curfile, mode)
+                elif url_path.endswith(".gz"):
+                    curfile = gzip.GzipFile(fileobj = curfile, mode = mode)
+                else:
+                    break
+                url_path = ".".join(url_path.split(".")[:-1])
+
+        # Fallback in case the file doesn't exist
+        if curfile is None:
+            raise ValueError("URL does not reference an openable file")
+        return curfile
+
+
+class JarHandler(urllib.request.BaseHandler):
+    """Handles the jar scheme for URIs
+
+    Reference used for the schema syntax:
+    http://docs.netkernel.org/book/view/book:mod:reference/doc:layer1:schemes:jar
+
+    Actual reference (found from https://www.w3.org/wiki/UriSchemes/jar) seemed not to return:
+    http://developer.java.sun.com/developer/onlineTraining/protocolhandlers/
+    """
+
+    @staticmethod
+    def default_open(req):
+        """Handles the request if it's the jar scheme"""
+        if req.type == 'jar':
+            subscheme, remainder = req.full_url.split(":")[1], ":".join(req.full_url.split(":")[2:])
+            if subscheme != 'file':
+                vollog.log(constants.LOGLEVEL_VVV, "Unsupported jar subscheme {}".format(subscheme))
+                return None
+
+            zipsplit = remainder.split("!")
+            if len(zipsplit) != 2:
+                vollog.log(constants.LOGLEVEL_VVV,
+                           "Path did not contain exactly one fragment indicator: {}".format(remainder))
+                return None
+
+            zippath, filepath = zipsplit
+            return zipfile.ZipFile(zippath).open(filepath)
+        return None

volatility/framework/layers/vmware.py

@@ -1,10 +1,9 @@
-import os
 import struct
 from typing import Any, Dict, List, Optional
 
 from volatility.framework import interfaces, validity
 from volatility.framework.configuration import requirements
-from volatility.framework.layers import physical, segmented
+from volatility.framework.layers import physical, segmented, resources
 from volatility.framework.symbols import native
 
 
@@ -123,13 +122,25 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface):
             current_layer_name = context.memory.free_layer_name("VmwareMetaLayer")
             current_config_path = interfaces.configuration.path_join("automagic", "layer_stacker", "stack",
                                                                      current_layer_name)
-            if os.path.exists(vmss):
+
+            try:
+                _ = resources.ResourceAccessor().open(vmss).read(10)
                 context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmss
                 context.memory.add_layer(physical.FileLayer(context, current_config_path, current_layer_name))
-            elif os.path.exists(vmsn):
-                context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmsn
-                context.memory.add_layer(physical.FileLayer(context, current_config_path, current_layer_name))
-            else:
+                vmss_success = True
+            except IOError:
+                vmss_success = False
+
+            if not vmss_success:
+                try:
+                    _ = resources.ResourceAccessor().open(vmsn).read(10)
+                    context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmsn
+                    context.memory.add_layer(physical.FileLayer(context, current_config_path, current_layer_name))
+                    vmsn_success = True
+                except IOError:
+                    vmsn_success = False
+
+            if not vmss_success and not vmsn_success:
                 return None
             new_layer_name = context.memory.free_layer_name("VmwareLayer")
             context.config[interfaces.configuration.path_join(current_config_path, "base_layer")] = layer_name

volatility/framework/plugins/windows/strings.py

@@ -2,9 +2,9 @@ import logging
 import re
 from typing import Dict, Generator, List, Set, Tuple
 
-from volatility.framework import interfaces, renderers, layers
+from volatility.framework import interfaces, renderers
 from volatility.framework.configuration import requirements
-from volatility.framework.layers import intel
+from volatility.framework.layers import intel, resources
 from volatility.framework.renderers import format_hints
 from volatility.plugins.windows import pslist
 
@@ -32,7 +32,7 @@ class Strings(interfaces.plugins.PluginInterface):
         """Generates results from a strings file"""
         revmap = self.generate_mapping(self.config['primary'])
 
-        accessor = layers.ResourceAccessor()
+        accessor = resources.ResourceAccessor()
 
         for line in accessor.open(self.config['strings_file'], "rb").readlines():
             try:

volatility/framework/plugins/windows/vadyarascan.py

@@ -1,8 +1,9 @@
 import logging
 from typing import Any, Iterable, List, Tuple
 
-from volatility.framework import interfaces, layers, renderers
+from volatility.framework import interfaces, renderers
 from volatility.framework.configuration import requirements
+from volatility.framework.layers import resources
 from volatility.framework.renderers import format_hints
 from volatility.framework.symbols.windows import extensions
 from volatility.plugins import yarascan
@@ -50,7 +51,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface):
                 rule += " wide ascii"
             rules = yara.compile(sources = {'n': 'rule r1 {{strings: $a = {} condition: $a}}'.format(rule)})
         elif self.config.get('yara_file', None) is not None:
-            rules = yara.compile(file = layers.ResourceAccessor().open(self.config['yara_file'], "rb"))
+            rules = yara.compile(file = resources.ResourceAccessor().open(self.config['yara_file'], "rb"))
         else:
             vollog.error("No yara rules, nor yara rules file were specified")
 

volatility/framework/plugins/yarascan.py

@@ -1,9 +1,10 @@
 import logging
 from typing import Iterable, Tuple, List
 
-from volatility.framework import interfaces, renderers, layers
+from volatility.framework import interfaces, renderers
 from volatility.framework.configuration import requirements
 from volatility.framework.interfaces import plugins
+from volatility.framework.layers import resources
 from volatility.framework.renderers import format_hints
 
 vollog = logging.getLogger(__name__)
@@ -71,7 +72,7 @@ class YaraScan(plugins.PluginInterface):
                 rule += " wide ascii"
             rules = yara.compile(sources = {'n': 'rule r1 {{strings: $a = {} condition: $a}}'.format(rule)})
         elif self.config.get('yara_file', None) is not None:
-            rules = yara.compile(file = layers.ResourceAccessor().open(self.config['yara_file'], "rb"))
+            rules = yara.compile(file = resources.ResourceAccessor().open(self.config['yara_file'], "rb"))
         else:
             vollog.error("No yara rules, nor yara rules file were specified")
 

volatility/framework/symbols/intermed.py

@@ -10,8 +10,10 @@ from abc import ABCMeta
 from typing import Any, Dict, Generator, Iterable, List, Optional, Type, Tuple
 
 import volatility
+import volatility.framework.layers.resources
 from volatility import schemas, symbols
-from volatility.framework import class_subclasses, constants, exceptions, interfaces, objects, layers
+from volatility.framework import class_subclasses, constants, exceptions, interfaces, objects
+from volatility.framework.layers import physical
 from volatility.framework.configuration import requirements
 from volatility.framework.symbols import native, metadata
 
@@ -79,7 +81,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface):
         # Check there are no obvious errors
         # Open the file and test the version
         self._versions = dict([(x.version, x) for x in class_subclasses(ISFormatTable)])
-        fp = layers.ResourceAccessor().open(isf_url)
+        fp = volatility.framework.layers.resources.ResourceAccessor().open(isf_url)
         reader = codecs.getreader("utf-8")
         json_object = json.load(reader(fp))  # type: ignore
         fp.close()