From aee4349e3cc8a021cd909b41a44856c506368e66 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 28 Dec 2016 03:20:44 +0000 Subject: [PATCH] More documentation updates (and minor code cleanups). --- volatility/framework/automagic/pdbscan.py | 53 ++++++++++-- volatility/framework/automagic/stacker.py | 32 ++++++-- volatility/framework/automagic/windows.py | 99 +++++++++++++++++++---- 3 files changed, 157 insertions(+), 27 deletions(-) diff --git a/volatility/framework/automagic/pdbscan.py b/volatility/framework/automagic/pdbscan.py index 8ce154177..46cc713a2 100644 --- a/volatility/framework/automagic/pdbscan.py +++ b/volatility/framework/automagic/pdbscan.py @@ -63,8 +63,8 @@ class PdbSigantureScanner(interfaces.layers.ScannerInterface): def scan(ctx, layer_name, progress_callback = None, start = None, end = None): - """Scans through `layer_name` at `ctx` and returns the tuple - (GUID, age, pdb_name, signature_offset, mz_offset) + """Scans through `layer_name` at `ctx` looking for RSDS headers that indicate one of four common pdb kernel names + (as listed in `self.pdb_names`) and returns the tuple (GUID, age, pdb_name, signature_offset, mz_offset) .. note:: This is automagical and therefore not guaranteed to provide correct results. @@ -110,6 +110,12 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): """An Automagic object that looks for all Intel translation layers and scans each of them for a pdb signature. When found, a search for a corresponding Intermediate Format data file is carried out and if found an appropriate symbol space is automatically loaded. + + Once a specific kernel PDB signature has been found, a virtual address for the loaded kernel is determined + by one of two methods. The first method assumes a specific mapping from the kernel's physical address to its + virtual address (typically the kernel is loaded at its physical location plus a specific offset). The second method + searches for a particular structure that lists the kernel module's virtual address, its size (not checked) and the + module's name. This value is then used if one was not found using the previous method. """ priority = 30 @@ -153,7 +159,18 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): return results def recurse_symbol_requirements(self, context, config_path, requirement): - """Determines if there is actually an unfulfilled symbol requirement waiting""" + """Determines if there is actually an unfulfilled symbol requirement waiting + + This ensures we do not carry out an expensive search when there is no requirement for a particular symbol table. + + :param context: Context on which to operate + :type context: ~volatility.framework.interfaces.context.ContextInterface + :param config_path: Configuration path of the top-level requirement + :type config_path: str + :param requirement: Top-level requirement whose subrequirements will all be searched + :type requirement: ~volatility.framework.interfaces.configuration.RequirementInterface + :return: A list of tuples containing the config_path, sub_config_path and requirement identifying the SymbolRequirements + """ sub_config_path = interfaces.configuration.path_join(config_path, requirement.name) results = [] if isinstance(requirement, interfaces.configuration.SymbolRequirement): @@ -166,9 +183,12 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): return results def recurse_symbol_fulfiller(self, context): - """Traverses the requirement tree looking to populate Symbol requirements based on the result of the pdb_finder + """Fulfills the SymbolRequirements in `self._symbol_requirements` found by the `recurse_symbol_requirements`. - This pass will construct any requirements that may need it + This pass will construct any requirements that may need it in the context it was passed + + :param context: Context on which to operate + :type context: ~volatility.framework.interfaces.context.ContextInterface """ for config_path, sub_config_path, requirement in self._symbol_requirements: # TODO: Potentially think about multiple symbol requirements in both the same and different levels of the requirement tree @@ -203,7 +223,12 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vollog.debug("No suitable kernel pdb signature found") def set_kernel_virtual_offset(self, context): - """Traverses the requirement tree, looking for kernel_virtual_offset values that may need setting""" + """Traverses the requirement tree, looking for kernel_virtual_offset values that may need setting and sets + it based on the previously identified `valid_kernels`. + + :param context: Context on which to operate and provide the kernel virtual offset + :type context: ~volatility.framework.interfaces.context.ContextInterface + """ for virtual_layer in self.valid_kernels: # Sit the virtual offset under the TranslationLayer it applies to kvo_path = interfaces.configuration.path_join(context.memory[virtual_layer].config_path, @@ -213,7 +238,21 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vollog.debug("Setting kernel_virtual_offset to {}".format(hex(kvo))) def determine_valid_kernels(self, context, potential_kernels, progress_callback = None): - """Runs through the identified potential kernels and verifies their suitability""" + """Runs through the identified potential kernels and verifies their suitability + + This carries out a scan using the pdb_signature scanner on a physical layer. It uses the + results of the scan to determine the virtual offset of the kernel. On early windows implementations + there is a fixed mapping between the physical and virtual addresses of the kernel. On more recent versions + a search is conducted for a structure that will identify the kernel's virtual offset. + + :param context: Context on which to operate + :type context: ~volatility.framework.interfaces.context.ContextInterface + :param potential_kernels: Dictionary containing `GUID`, `age`, `pdb_name` and `mz_offset` keys + :type potential_kernels: dict + :param progress_callback: Function taking a percentage and optional description to be called during expensive computations to indicate progress + :type progress_callback: function + :return: A dictionary of valid kernels + """ valid_kernels = {} for virtual_layer_name in potential_kernels: kernels = potential_kernels[virtual_layer_name] diff --git a/volatility/framework/automagic/stacker.py b/volatility/framework/automagic/stacker.py index e758152dc..304a4d3ba 100644 --- a/volatility/framework/automagic/stacker.py +++ b/volatility/framework/automagic/stacker.py @@ -1,6 +1,14 @@ -from urllib import parse +"""This module attempts to automatically stack layers. + +This automagic module fulfills :class:`~volatility.framework.interfaces.configuration.TranslationLayerRequirement` that are not already fulfilled, by attempting to +stack as many layers on top of each other as possible. The base/lowest layer is derived from the +"automagic.general.single_location" configuration path. Layers are then attempting in likely height order, and +once a layer successfully stacks on top of the existing layers, it is removed from the possible choices list +(so no layer type can exist twice in the layer stack). +""" import logging +from urllib import parse import volatility from volatility.framework import interfaces @@ -11,7 +19,16 @@ vollog = logging.getLogger(__name__) class LayerStacker(interfaces.automagic.AutomagicInterface): - """Class that attempts to build up """ + """Class that attempts to build up layers in a single stack + + This class mimics the volatility 2 style of stacking address spaces. It builds up various layers based on + separate :class:`~volatility.framework.interfaces.automagic.StackerLayerInterface` classes. These classes are + built up based on a `stack_order` class variable each has. + + This has a high priority to provide other automagic modules as complete a context/configuration tree as possible. + Upon completion it will re-call the :class:`~volatility.framework.automagic.construct_layers.ConstructionMagic`, + so that any stacked layers are actually constructed and added to the context. + """ # Most important automagic, must happen first! priority = 10 page_map_offset = None @@ -85,14 +102,19 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): constructor(context, config_path, requirement) def find_suitable_requirements(self, stacked_layers, requirement, context, config_path): + """Looks for translation layer requirements and attempts to apply the stacked layers to it. If it succeeds + it returns the configuration path and layer name where the stacked nodes were spliced into the tree. + + :return: A tuple of a configuration path and layer name for the top of the stacked layers + :rtype: (str, str)""" child_config_path = interfaces.configuration.path_join(config_path, requirement.name) if isinstance(requirement, interfaces.configuration.TranslationLayerRequirement): if not requirement.validate(context, config_path): original_setting = context.config.get(child_config_path, None) - for layer in stacked_layers: - context.config[child_config_path] = layer + for layer_name in stacked_layers: + context.config[child_config_path] = layer_name if requirement.validate(context, config_path): - return child_config_path, layer + return child_config_path, layer_name else: # Clean-up to restore the config if original_setting: diff --git a/volatility/framework/automagic/windows.py b/volatility/framework/automagic/windows.py index 53759e5a2..ef5a80680 100644 --- a/volatility/framework/automagic/windows.py +++ b/volatility/framework/automagic/windows.py @@ -1,9 +1,27 @@ -if __name__ == "__main__": - import os - import sys +"""Module to identify the Directory Table Base and architecture of windows memory images - sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))) +This module contains a PageMapScanner that scans a physical layer to identify self-referential pointers. +All windows versions include a self-referential pointer in their Directory Table Base's top table, in order to +have a single offset that will allow manipulation of the page tables themselves. +In older windows version the self-referential pointer was at a specific fixed index within the table, +which was different for each architecture. In very recent Windows versions, the self-referential pointer +index has been randomized, so a different heuristic must be used. In these versions of windows it was found +that the physical offset for the DTB was always within the range of 0x1a0000 to 0x1b0000. As such, a search +for any self-referential pointer within these pages gives a high probability of being an accurate DTB. + +The self-referential indices for older versions of windows are listed below: + + +--------------+-------+ + | Architecture | Index | + +==============+=======+ + | x86 | 0x300 | + +--------------+-------+ + | PAE | 0x3 | + +--------------+-------+ + | x64 | 0x1ED | + +--------------+-------+ +""" import logging import struct @@ -16,7 +34,11 @@ PAGE_SIZE = 0x1000 class DtbTest(validity.ValidityRoutines): - super_bit = 2 + """This class generically contains the tests for a page based on a set of class parameters + + When constructed it contains all the information necessary to extract a specific index from a page + and determine whether it points back to that page's offset. + """ def __init__(self, layer_type = None, ptr_struct = None, ptr_reference = None, mask = None): self.layer_type = self._check_class(layer_type, interfaces.layers.TranslationLayerInterface) @@ -25,13 +47,24 @@ class DtbTest(validity.ValidityRoutines): self.ptr_reference = self._check_type(ptr_reference, int) self.mask = self._check_type(mask, int) - def unpack(self, value): + def _unpack(self, value): return struct.unpack("<" + self.ptr_struct, value)[0] def __call__(self, data, data_offset, page_offset): + """Tests a specific page in a chunk of data to see if it contains a self-referential pointer. + + :param data: The chunk of data that contains the page to be scanned + :type data: bytes + :param data_offset: Where, within the layer, the chunk of data lives + :type data_offset: int + :param page_offset: Where, within the data, the page to be scanned starts + :type page_offset: int + :return: A valid DTB within this page + :rtype: int or None + """ value = data[page_offset + (self.ptr_reference * self.ptr_size):page_offset + ( (self.ptr_reference + 1) * self.ptr_size)] - ptr = self.unpack(value) + ptr = self._unpack(value) # The value *must* be present (bit 0) since it's a mapped page # It's almost always writable (bit 1) # It's occasionally Super, but not reliably so, haven't checked when/why not @@ -42,10 +75,19 @@ class DtbTest(validity.ValidityRoutines): return self.second_pass(dtb, data, data_offset) def second_pass(self, dtb, data, data_offset): + """Re-reads over the whole page to validate other records based on the number of pages marked user vs super + + :param dtb: The identified dtb that needs validating + :type dtb: int + :param data: The chunk of data that contains the dtb to be validated + :type data: bytes + :param data_offset: Where, within the layer, the chunk of data lives + :type data_offset: int + """ page = data[dtb - data_offset:dtb - data_offset + PAGE_SIZE] usr_count, sup_count = 0, 0 for i in range(0, PAGE_SIZE, self.ptr_size): - val = self.unpack(page[i:i + self.ptr_size]) + val = self._unpack(page[i:i + self.ptr_size]) if val & 0x1: sup_count += 0 if (val & 0x4) else 1 usr_count += 1 if (val & 0x4) else 0 @@ -80,16 +122,33 @@ class DtbTestPae(DtbTest): mask = 0x3FFFFFFFFFF000) def second_pass(self, dtb, data, data_offset): + """PAE top level directory tables contains four entries and the self-referential pointer occurs in the second + level of tables (so as not to use up a full quarter of the space). This is very high in the space, and occurs + in the fourht (last quarter) second-level table. The second-level tables appear always to come sequentially + directly after the real dtb. The value for the real DTB is therefore four page earlier (and the fourth entry + should point back to the `dtb` parameter this function was originally passed. + + :param dtb: The identified self-referential pointer that needs validating + :type dtb: int + :param data: The chunk of data that contains the dtb to be validated + :type data: bytes + :param data_offset: Where, within the layer, the chunk of data lives + :type data_offset: int + :return: Returns the actual DTB of the PAE space + :rtype: int + """ dtb -= 0x4000 # If we're not in something that the overlap would pick up if dtb - data_offset >= 0: pointers = data[dtb - data_offset + (3 * self.ptr_size): dtb - data_offset + (4 * self.ptr_size)] - val = self.unpack(pointers) + val = self._unpack(pointers) if (val & self.mask == dtb + 0x4000) and (val & 0xFFF == 0x001): return dtb class DtbSelfReferential(DtbTest): + """A generic DTB test which looks for a self-referential pointer at *any* index within the page.""" + def __init__(self, layer_type, ptr_struct, ptr_reference, mask): super().__init__(layer_type = layer_type, ptr_struct = ptr_struct, @@ -125,9 +184,11 @@ class DtbSelfRef64bit(DtbSelfReferential): class PageMapScanner(interfaces.layers.ScannerInterface): + """Scans through all pages using DTB tests to determine a dtb offset and architecture""" overlap = 0x4000 thread_safe = True tests = [DtbTest32bit, DtbTest64bit, DtbTestPae] + """The default tests to run when searching for DTBs""" def __init__(self, tests): super().__init__() @@ -148,15 +209,16 @@ class PageMapScanner(interfaces.layers.ScannerInterface): class WintelHelper(interfaces.automagic.AutomagicInterface, interfaces.automagic.StackerLayerInterface): + """This class if both an :class:`~volatility.framework.interfaces.automagic.AutomagicInterface` and a + :class:`~volatility.framework.interfaces.automagic.StackerLayerInterface` class. + + It will both scan for existing TranslationLayers that do not have a DTB and scan for them using + the :class:`PageMapScanner`, and also act as a stacker when a + :class:`~volatility.framework.interfaces.configuration.TranslationLayerRequirement` has not been fulfilled""" priority = 20 stack_order = 90 tests = [DtbTest32bit(), DtbTest64bit(), DtbTestPae()] - def branch_leave(self, node, config_path): - """Ensure we're called on internal nodes as well as external""" - self(node, config_path) - return True - def __call__(self, context, config_path, requirement, progress_callback = None): useful = [] sub_config_path = interfaces.configuration.path_join(config_path, requirement.name) @@ -185,7 +247,14 @@ class WintelHelper(interfaces.automagic.AutomagicInterface, interfaces.automagic @classmethod def stack(cls, context, layer_name, progress_callback = None): - """Attempts to determine and stack an intel layer on a physical layer where possible""" + """Attempts to determine and stack an intel layer on a physical layer where possible + + Where the DTB scan fails, it attempts a heuristic of checking for the DTB within a specific range. + New versions of windows, with randomized self-referential pointers, appear to always load their dtb within + a small specific range (`0x1a0000` and `0x1b0000`), so instead we scan for all self-referential pointers in + that range, and ignore any that contain multiple self-references (since the DTB is very unlikely to point to + itself more than once). + """ hits = context.memory[layer_name].scan(context, PageMapScanner(cls.tests)) layer = None for test, dtb in hits: