More documentation updates (and minor code cleanups).

volatility/framework/automagic/pdbscan.py

@@ -63,8 +63,8 @@ class PdbSigantureScanner(interfaces.layers.ScannerInterface):
 
 
 def scan(ctx, layer_name, progress_callback = None, start = None, end = None):
-    """Scans through `layer_name` at `ctx` and returns the tuple
-       (GUID, age, pdb_name, signature_offset, mz_offset)
+    """Scans through `layer_name` at `ctx` looking for RSDS headers that indicate one of four common pdb kernel names
+       (as listed in `self.pdb_names`) and returns the tuple (GUID, age, pdb_name, signature_offset, mz_offset)
 
        .. note:: This is automagical and therefore not guaranteed to provide correct results.
 
@@ -110,6 +110,12 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
     """An Automagic object that looks for all Intel translation layers and scans each of them for a pdb signature.
     When found, a search for a corresponding Intermediate Format data file is carried out and if found an appropriate
     symbol space is automatically loaded.
+
+    Once a specific kernel PDB signature has been found, a virtual address for the loaded kernel is determined
+    by one of two methods.  The first method assumes a specific mapping from the kernel's physical address to its
+    virtual address (typically the kernel is loaded at its physical location plus a specific offset).  The second method
+    searches for a particular structure that lists the kernel module's virtual address, its size (not checked) and the
+    module's name.  This value is then used if one was not found using the previous method.
     """
     priority = 30
 
@@ -153,7 +159,18 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
         return results
 
     def recurse_symbol_requirements(self, context, config_path, requirement):
-        """Determines if there is actually an unfulfilled symbol requirement waiting"""
+        """Determines if there is actually an unfulfilled symbol requirement waiting
+
+        This ensures we do not carry out an expensive search when there is no requirement for a particular symbol table.
+
+        :param context: Context on which to operate
+        :type context: ~volatility.framework.interfaces.context.ContextInterface
+        :param config_path: Configuration path of the top-level requirement
+        :type config_path: str
+        :param requirement: Top-level requirement whose subrequirements will all be searched
+        :type requirement: ~volatility.framework.interfaces.configuration.RequirementInterface
+        :return: A list of tuples containing the config_path, sub_config_path and requirement identifying the SymbolRequirements
+        """
         sub_config_path = interfaces.configuration.path_join(config_path, requirement.name)
         results = []
         if isinstance(requirement, interfaces.configuration.SymbolRequirement):
@@ -166,9 +183,12 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
         return results
 
     def recurse_symbol_fulfiller(self, context):
-        """Traverses the requirement tree looking to populate Symbol requirements based on the result of the pdb_finder
+        """Fulfills the SymbolRequirements in `self._symbol_requirements` found by the `recurse_symbol_requirements`.
 
-           This pass will construct any requirements that may need it
+        This pass will construct any requirements that may need it in the context it was passed
+
+        :param context: Context on which to operate
+        :type context: ~volatility.framework.interfaces.context.ContextInterface
         """
         for config_path, sub_config_path, requirement in self._symbol_requirements:
             # TODO: Potentially think about multiple symbol requirements in both the same and different levels of the requirement tree
@@ -203,7 +223,12 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
                     vollog.debug("No suitable kernel pdb signature found")
 
     def set_kernel_virtual_offset(self, context):
-        """Traverses the requirement tree, looking for kernel_virtual_offset values that may need setting"""
+        """Traverses the requirement tree, looking for kernel_virtual_offset values that may need setting and sets
+        it based on the previously identified `valid_kernels`.
+
+        :param context: Context on which to operate and provide the kernel virtual offset
+        :type context: ~volatility.framework.interfaces.context.ContextInterface
+        """
         for virtual_layer in self.valid_kernels:
             # Sit the virtual offset under the TranslationLayer it applies to
             kvo_path = interfaces.configuration.path_join(context.memory[virtual_layer].config_path,
@@ -213,7 +238,21 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
             vollog.debug("Setting kernel_virtual_offset to {}".format(hex(kvo)))
 
     def determine_valid_kernels(self, context, potential_kernels, progress_callback = None):
-        """Runs through the identified potential kernels and verifies their suitability"""
+        """Runs through the identified potential kernels and verifies their suitability
+
+        This carries out a scan using the pdb_signature scanner on a physical layer.  It uses the
+        results of the scan to determine the virtual offset of the kernel.  On early windows implementations
+        there is a fixed mapping between the physical and virtual addresses of the kernel.  On more recent versions
+        a search is conducted for a structure that will identify the kernel's virtual offset.
+
+        :param context: Context on which to operate
+        :type context: ~volatility.framework.interfaces.context.ContextInterface
+        :param potential_kernels: Dictionary containing `GUID`, `age`, `pdb_name` and `mz_offset` keys
+        :type potential_kernels: dict
+        :param progress_callback: Function taking a percentage and optional description to be called during expensive computations to indicate progress
+        :type progress_callback: function
+        :return: A dictionary of valid kernels
+        """
         valid_kernels = {}
         for virtual_layer_name in potential_kernels:
             kernels = potential_kernels[virtual_layer_name]

volatility/framework/automagic/stacker.py

@@ -1,6 +1,14 @@
-from urllib import parse
+"""This module attempts to automatically stack layers.
+
+This automagic module fulfills :class:`~volatility.framework.interfaces.configuration.TranslationLayerRequirement` that are not already fulfilled, by attempting to
+stack as many layers on top of each other as possible.  The base/lowest layer is derived from the
+"automagic.general.single_location" configuration path.  Layers are then attempting in likely height order, and
+once a layer successfully stacks on top of the existing layers, it is removed from the possible choices list
+(so no layer type can exist twice in the layer stack).
+"""
 
 import logging
+from urllib import parse
 
 import volatility
 from volatility.framework import interfaces
@@ -11,7 +19,16 @@ vollog = logging.getLogger(__name__)
 
 
 class LayerStacker(interfaces.automagic.AutomagicInterface):
-    """Class that attempts to build up """
+    """Class that attempts to build up layers in a single stack
+
+    This class mimics the volatility 2 style of stacking address spaces.  It builds up various layers based on
+    separate :class:`~volatility.framework.interfaces.automagic.StackerLayerInterface` classes.  These classes are
+    built up based on a `stack_order` class variable each has.
+
+    This has a high priority to provide other automagic modules as complete a context/configuration tree as possible.
+    Upon completion it will re-call the :class:`~volatility.framework.automagic.construct_layers.ConstructionMagic`,
+    so that any stacked layers are actually constructed and added to the context.
+    """
     # Most important automagic, must happen first!
     priority = 10
     page_map_offset = None
@@ -85,14 +102,19 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
             constructor(context, config_path, requirement)
 
     def find_suitable_requirements(self, stacked_layers, requirement, context, config_path):
+        """Looks for translation layer requirements and attempts to apply the stacked layers to it.  If it succeeds
+        it returns the configuration path and layer name where the stacked nodes were spliced into the tree.
+
+        :return: A tuple of a configuration path and layer name for the top of the stacked layers
+        :rtype: (str, str)"""
         child_config_path = interfaces.configuration.path_join(config_path, requirement.name)
         if isinstance(requirement, interfaces.configuration.TranslationLayerRequirement):
             if not requirement.validate(context, config_path):
                 original_setting = context.config.get(child_config_path, None)
-                for layer in stacked_layers:
-                    context.config[child_config_path] = layer
+                for layer_name in stacked_layers:
+                    context.config[child_config_path] = layer_name
                     if requirement.validate(context, config_path):
-                        return child_config_path, layer
+                        return child_config_path, layer_name
                 else:
                     # Clean-up to restore the config
                     if original_setting:

volatility/framework/automagic/windows.py

@@ -1,9 +1,27 @@
-if __name__ == "__main__":
-    import os
-    import sys
+"""Module to identify the Directory Table Base and architecture of windows memory images
 
-    sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')))
+This module contains a PageMapScanner that scans a physical layer to identify self-referential pointers.
+All windows versions include a self-referential pointer in their Directory Table Base's top table, in order to
+have a single offset that will allow manipulation of the page tables themselves.
 
+In older windows version the self-referential pointer was at a specific fixed index within the table,
+which was different for each architecture.  In very recent Windows versions, the self-referential pointer
+index has been randomized, so a different heuristic must be used.  In these versions of windows it was found
+that the physical offset for the DTB was always within the range of 0x1a0000 to 0x1b0000.  As such, a search
+for any self-referential pointer within these pages gives a high probability of being an accurate DTB.
+
+The self-referential indices for older versions of windows are listed below:
+
+    +--------------+-------+
+    | Architecture | Index |
+    +==============+=======+
+    | x86          | 0x300 |
+    +--------------+-------+
+    | PAE          | 0x3   |
+    +--------------+-------+
+    | x64          | 0x1ED |
+    +--------------+-------+
+"""
 import logging
 import struct
 
@@ -16,7 +34,11 @@ PAGE_SIZE = 0x1000
 
 
 class DtbTest(validity.ValidityRoutines):
-    super_bit = 2
+    """This class generically contains the tests for a page based on a set of class parameters
+
+    When constructed it contains all the information necessary to extract a specific index from a page
+    and determine whether it points back to that page's offset.
+    """
 
     def __init__(self, layer_type = None, ptr_struct = None, ptr_reference = None, mask = None):
         self.layer_type = self._check_class(layer_type, interfaces.layers.TranslationLayerInterface)
@@ -25,13 +47,24 @@ class DtbTest(validity.ValidityRoutines):
         self.ptr_reference = self._check_type(ptr_reference, int)
         self.mask = self._check_type(mask, int)
 
-    def unpack(self, value):
+    def _unpack(self, value):
         return struct.unpack("<" + self.ptr_struct, value)[0]
 
     def __call__(self, data, data_offset, page_offset):
+        """Tests a specific page in a chunk of data to see if it contains a self-referential pointer.
+
+        :param data: The chunk of data that contains the page to be scanned
+        :type data: bytes
+        :param data_offset: Where, within the layer, the chunk of data lives
+        :type data_offset: int
+        :param page_offset: Where, within the data, the page to be scanned starts
+        :type page_offset: int
+        :return: A valid DTB within this page
+        :rtype: int or None
+        """
         value = data[page_offset + (self.ptr_reference * self.ptr_size):page_offset + (
             (self.ptr_reference + 1) * self.ptr_size)]
-        ptr = self.unpack(value)
+        ptr = self._unpack(value)
         # The value *must* be present (bit 0) since it's a mapped page
         # It's almost always writable (bit 1)
         # It's occasionally Super, but not reliably so, haven't checked when/why not
@@ -42,10 +75,19 @@ class DtbTest(validity.ValidityRoutines):
             return self.second_pass(dtb, data, data_offset)
 
     def second_pass(self, dtb, data, data_offset):
+        """Re-reads over the whole page to validate other records based on the number of pages marked user vs super
+
+        :param dtb: The identified dtb that needs validating
+        :type dtb: int
+        :param data: The chunk of data that contains the dtb to be validated
+        :type data: bytes
+        :param data_offset: Where, within the layer, the chunk of data lives
+        :type data_offset: int
+        """
         page = data[dtb - data_offset:dtb - data_offset + PAGE_SIZE]
         usr_count, sup_count = 0, 0
         for i in range(0, PAGE_SIZE, self.ptr_size):
-            val = self.unpack(page[i:i + self.ptr_size])
+            val = self._unpack(page[i:i + self.ptr_size])
             if val & 0x1:
                 sup_count += 0 if (val & 0x4) else 1
                 usr_count += 1 if (val & 0x4) else 0
@@ -80,16 +122,33 @@ class DtbTestPae(DtbTest):
                          mask = 0x3FFFFFFFFFF000)
 
     def second_pass(self, dtb, data, data_offset):
+        """PAE top level directory tables contains four entries and the self-referential pointer occurs in the second
+        level of tables (so as not to use up a full quarter of the space).  This is very high in the space, and occurs
+        in the fourht (last quarter) second-level table.  The second-level tables appear always to come sequentially
+        directly after the real dtb.  The value for the real DTB is therefore four page earlier (and the fourth entry
+        should point back to the `dtb` parameter this function was originally passed.
+
+        :param dtb: The identified self-referential pointer that needs validating
+        :type dtb: int
+        :param data: The chunk of data that contains the dtb to be validated
+        :type data: bytes
+        :param data_offset: Where, within the layer, the chunk of data lives
+        :type data_offset: int
+        :return: Returns the actual DTB of the PAE space
+        :rtype: int
+        """
         dtb -= 0x4000
         # If we're not in something that the overlap would pick up
         if dtb - data_offset >= 0:
             pointers = data[dtb - data_offset + (3 * self.ptr_size): dtb - data_offset + (4 * self.ptr_size)]
-            val = self.unpack(pointers)
+            val = self._unpack(pointers)
             if (val & self.mask == dtb + 0x4000) and (val & 0xFFF == 0x001):
                 return dtb
 
 
 class DtbSelfReferential(DtbTest):
+    """A generic DTB test which looks for a self-referential pointer at *any* index within the page."""
+
     def __init__(self, layer_type, ptr_struct, ptr_reference, mask):
         super().__init__(layer_type = layer_type,
                          ptr_struct = ptr_struct,
@@ -125,9 +184,11 @@ class DtbSelfRef64bit(DtbSelfReferential):
 
 
 class PageMapScanner(interfaces.layers.ScannerInterface):
+    """Scans through all pages using DTB tests to determine a dtb offset and architecture"""
     overlap = 0x4000
     thread_safe = True
     tests = [DtbTest32bit, DtbTest64bit, DtbTestPae]
+    """The default tests to run when searching for DTBs"""
 
     def __init__(self, tests):
         super().__init__()
@@ -148,15 +209,16 @@ class PageMapScanner(interfaces.layers.ScannerInterface):
 
 
 class WintelHelper(interfaces.automagic.AutomagicInterface, interfaces.automagic.StackerLayerInterface):
+    """This class if both an :class:`~volatility.framework.interfaces.automagic.AutomagicInterface` and a
+    :class:`~volatility.framework.interfaces.automagic.StackerLayerInterface` class.
+
+    It will both scan for existing TranslationLayers that do not have a DTB and scan for them using
+    the :class:`PageMapScanner`, and also act as a stacker when a
+    :class:`~volatility.framework.interfaces.configuration.TranslationLayerRequirement` has not been fulfilled"""
     priority = 20
     stack_order = 90
     tests = [DtbTest32bit(), DtbTest64bit(), DtbTestPae()]
 
-    def branch_leave(self, node, config_path):
-        """Ensure we're called on internal nodes as well as external"""
-        self(node, config_path)
-        return True
-
     def __call__(self, context, config_path, requirement, progress_callback = None):
         useful = []
         sub_config_path = interfaces.configuration.path_join(config_path, requirement.name)
@@ -185,7 +247,14 @@ class WintelHelper(interfaces.automagic.AutomagicInterface, interfaces.automagic
 
     @classmethod
     def stack(cls, context, layer_name, progress_callback = None):
-        """Attempts to determine and stack an intel layer on a physical layer where possible"""
+        """Attempts to determine and stack an intel layer on a physical layer where possible
+
+        Where the DTB scan fails, it attempts a heuristic of checking for the DTB within a specific range.
+        New versions of windows, with randomized self-referential pointers, appear to always load their dtb within
+        a small specific range (`0x1a0000` and `0x1b0000`), so instead we scan for all self-referential pointers in
+        that range, and ignore any that contain multiple self-references (since the DTB is very unlikely to point to
+        itself more than once).
+        """
         hits = context.memory[layer_name].scan(context, PageMapScanner(cls.tests))
         layer = None
         for test, dtb in hits: