Refactor the progress_callback so it can be used in automagics.

Refactored the location of the progress_callback so that the UI can
specify it and ensure it is used by the automagics process as well as
the plugins themselves.

The progress_callback function has also had an optional description parameter
added, so that it can differentiated in case two scans happen
simultaneously.

volatility/cli/__init__.py

@@ -90,7 +90,7 @@ class CommandLine(object):
         ###
         # Clever magic figures out how to fulfill each requirement that might not be fulfilled
         automagics = automagic.available()
-        automagic.run(automagics, ctx, plugin, "plugins")
+        automagic.run(automagics, ctx, plugin, "plugins", progress_callback = progress_callback)
 
         # Check all the requirements and/or go back to the automagic step
         if not plugin.validate(ctx, config_path):
@@ -107,5 +107,16 @@ class CommandLine(object):
         TextRenderer().render(constructed.run())
 
 
+def progress_callback(progress, description = None):
+    """ A sinmple function for providing text-based feedback
+
+    .. warning:: Only for development use.
+
+    :param progress: Percentage of progress of the current procedure
+    :type progress: int or float
+    """
+    print("\rProgress: ", round(progress, 2), "\t\t", description or '', end = '\n')
+
+
 def main():
     CommandLine().run()

volatility/framework/automagic/__init__.py

@@ -15,8 +15,16 @@ def available():
                   key = lambda x: x.priority)
 
 
-def run(automagics, context, configurable, config_path = ""):
-    """Runs through the list of automagics in order, allowing them to make changes to the context
+def run(automagics, context, configurable, config_path = "", progress_callback = None):
+    """Runs through the list of `automagics` in order, allowing them to make changes to the context
+
+    :param automagics: A list of :class:`~volatility.framework.interfaces.automagic.AutomagicInterface` objects
+    :param context: The context (that inherits from :class:`~volatility.framework.interfaces.context.ContextInterface`) for modification
+    :param configurable: An object that inherits from :class:`~volatility.framework.interfaces.configuration.ConfigurableInterface`
+    :param config_path: The path within the `context.config` for options required by the `configurable`
+    :param progress_callback: A function that takes a percentage (and an optional description) that will be called periodically
+
+    This is where any automagic is allowed to run, and alter the context in order to satisfy/improve all requirements
 
        This is where any automagic is allowed to run, and alter the context in order to satisfy/improve all requirements
     """
@@ -39,4 +47,4 @@ def run(automagics, context, configurable, config_path = ""):
 
     for automagic in automagics:
         vollog.info("Running automagic: {}".format(automagic.__class__.__name__))
-        automagic(context, config_path, requirement)
+        automagic(context, config_path, requirement, progress_callback)

volatility/framework/automagic/construct_layers.py

@@ -13,7 +13,7 @@ class ConstructionMagic(interfaces.automagic.AutomagicInterface):
     """
     priority = 0
 
-    def __call__(self, context, config_path, requirement, optional = False):
+    def __call__(self, context, config_path, requirement, progress_callback = None, optional = False):
         if not requirement.validate(context, config_path):
             # Having called validate at the top level tells us both that we need to dig deeper
             # but also ensures that TranslationLayerRequirements have got the correct subrequirements if their class is populated

volatility/framework/automagic/pdbscan.py

@@ -50,8 +50,8 @@ class PdbSigantureScanner(interfaces.layers.ScannerInterface):
             sig = data.find(b"RSDS", sig + 1)
 
 
-def scan(ctx, layer_name, start = None, end = None):
-    """Scans through layer_name at context and returns the tuple
+def scan(ctx, layer_name, progress_callback = None, start = None, end = None):
+    """Scans through `layer_name` at `ctx` and returns the tuple
        (GUID, age, pdb_name, signature_offset, mz_offset)
 
        Note that this is automagical and therefore not guaranteed to provide
@@ -95,10 +95,6 @@ def scan(ctx, layer_name, start = None, end = None):
     return results
 
 
-def progress_callback(progress):
-    print("\rProgress: ", progress, "     ", end = '')
-
-
 class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
     """Looks for all Intel address spaces and attempts to identify the PDB guid required for the space"""
     priority = 30
@@ -112,8 +108,10 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
         super().__init__()
         self.valid_kernels = []
 
-    def recurse_pdb_finder(self, context, config_path, requirement):
-        """Traverses the requirement tree looking for virtual layers that might contain a windows PDB
+    def recurse_pdb_finder(self, context, config_path, requirement, progress_callback = None):
+        """Traverses the requirement tree, rooted at `requirement` looking for virtual layers that might contain a windows PDB.
+
+        Returns a list of possible kernel locations in the physical memory
 
            Returns a list of possible kernel locations in the physical memory
         """
@@ -126,7 +124,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
             virtual_layer_name = context.config.get(sub_config_path, None)
             layer_name = context.config.get(interfaces.configuration.path_join(sub_config_path, "memory_layer"), None)
             if layer_name:
-                results = {virtual_layer_name: scan(context, layer_name)}
+                results = {virtual_layer_name: scan(context, layer_name, progress_callback = progress_callback)}
         else:
             for subreq in requirement.requirements.values():
                 results.update(self.recurse_pdb_finder(context, sub_config_path, subreq))
@@ -192,7 +190,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
             context.config[kvo_path] = kvo
             vollog.debug("Setting kernel_virtual_offset to {}".format(hex(kvo)))
 
-    def determine_valid_kernels(self, context, potential_kernels):
+    def determine_valid_kernels(self, context, potential_kernels, progress_callback = None):
         """Runs through the identified potential kernels and verifies their suitability"""
         valid_kernels = {}
         for virtual_layer_name in potential_kernels:
@@ -234,7 +232,8 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
                     # If we're here, chances are high we're in a Win10 x64 image with kernel base randomization
                     physical_layer = context.memory[physical_layer_name]
                     # TODO:  On older windows, this might be \WINDOWS\system32\nt rather than \SystemRoot\system32\nt
-                    results = physical_layer.scan(context, scanners.BytesScanner(b"\\SystemRoot\\system32\\nt"))
+                    results = physical_layer.scan(context, scanners.BytesScanner(b"\\SystemRoot\\system32\\nt"),
+                                                  progress_callback = progress_callback)
                     seen = set()
                     for result in results:
                         # TODO: Identify the specific structure we're finding and document this a bit better
@@ -247,7 +246,8 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
                         try:
                             potential_mz = vlayer.read(offset = address, length = 2)
                             if potential_mz == b"MZ":
-                                subscan = scan(context, virtual_layer_name, start = address, end = address + (1 << 26))
+                                subscan = scan(context, virtual_layer_name, start = address, end = address + (1 << 26),
+                                               progress_callback = progress_callback)
                                 for result in subscan:
                                     valid_kernels[virtual_layer_name] = (address, result)
                                     break
@@ -260,14 +260,14 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
             vollog.warning("No suitable kernels found during pdbscan")
         return valid_kernels
 
-    def __call__(self, context, config_path, requirement):
+    def __call__(self, context, config_path, requirement, progress_callback = None):
         # TODO: Check if we really need to search for pdbs
         if "pdbscan" not in context.symbol_space:
             context.symbol_space.append(native.NativeTable("pdbscan", native.std_ctypes))
         self._symbol_requirements = self.recurse_symbol_requirements(context, config_path, requirement)
         if self._symbol_requirements:
-            potential_kernels = self.recurse_pdb_finder(context, config_path, requirement)
-            self.valid_kernels = self.determine_valid_kernels(context, potential_kernels)
+            potential_kernels = self.recurse_pdb_finder(context, config_path, requirement, progress_callback)
+            self.valid_kernels = self.determine_valid_kernels(context, potential_kernels, progress_callback)
             if self.valid_kernels:
                 self.recurse_symbol_fulfiller(context)
                 self.set_kernel_virtual_offset(context)

volatility/framework/automagic/stacker.py

@@ -17,7 +17,7 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
     page_map_offset = None
     location = None
 
-    def __call__(self, context, config_path, requirement):
+    def __call__(self, context, config_path, requirement, progress_callback = None):
         """Runs the automagic over the configurable"""
 
         # Quick exit if we're not needed
@@ -57,7 +57,7 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
             for stacker_cls in stack_set:
                 stacker = stacker_cls()
                 try:
-                    new_layer = stacker.stack(new_context, current_layer_name)
+                    new_layer = stacker.stack(new_context, current_layer_name, progress_callback)
                     if new_layer:
                         new_context.memory.add_layer(new_layer)
                         break

volatility/framework/automagic/windows.py

@@ -157,7 +157,7 @@ class WintelHelper(interfaces.automagic.AutomagicInterface, interfaces.automagic
         self(node, config_path)
         return True
 
-    def __call__(self, context, config_path, requirement):
+    def __call__(self, context, config_path, requirement, progress_callback = None):
         useful = []
         sub_config_path = interfaces.configuration.path_join(config_path, requirement.name)
         if (isinstance(requirement, requirements.TranslationLayerRequirement) and
@@ -174,7 +174,7 @@ class WintelHelper(interfaces.automagic.AutomagicInterface, interfaces.automagic
             if ("memory_layer" in requirement.requirements and
                     requirement.requirements["memory_layer"].validate(context, sub_config_path)):
                 physical_layer = requirement.requirements["memory_layer"].config_value(context, sub_config_path)
-                hits = context.memory[physical_layer].scan(context, PageMapScanner(useful))
+                hits = context.memory[physical_layer].scan(context, PageMapScanner(useful), progress_callback)
                 for test, dtb in hits:
                     context.config[interfaces.configuration.path_join(sub_config_path, "page_map_offset")] = dtb
                     requirement.construct(context, config_path)
@@ -184,7 +184,7 @@ class WintelHelper(interfaces.automagic.AutomagicInterface, interfaces.automagic
                 self(context, sub_config_path, subreq)
 
     @classmethod
-    def stack(cls, context, layer_name):
+    def stack(cls, context, layer_name, progress_callback = None):
         """Attempts to determine and stack an intel layer on a physical layer where possible"""
         hits = context.memory[layer_name].scan(context, PageMapScanner(cls.tests))
         layer = None
@@ -202,7 +202,7 @@ class WintelHelper(interfaces.automagic.AutomagicInterface, interfaces.automagic
             # There is a very high chance that the DTB will live in this narrow segment, assuming we couldn't find it previously
             # TODO: This scan takes time, it might be worth adding a progress callback to it
             hits = context.memory[layer_name].scan(context, PageMapScanner([DtbSelfRef64bit()]), min_address = 0x1a0000,
-                                                   max_address = 0x1f0000)
+                                                   max_address = 0x1f0000, progress_callback = progress_callback)
             # Flatten the generator
             hits = list(hits)
             if hits:

volatility/framework/interfaces/automagic.py

@@ -9,7 +9,7 @@ class AutomagicInterface(validity.ValidityRoutines, metaclass = ABCMeta):
     priority = 10
 
     @abstractmethod
-    def __call__(self, context, config_path, configurable):
+    def __call__(self, context, config_path, configurable, progress_callback = None):
         """Runs the automagic over the configurable"""
 
 
@@ -24,7 +24,7 @@ class StackerLayerInterface(validity.ValidityRoutines, metaclass = ABCMeta):
 
     @classmethod
     @abstractmethod
-    def stack(self, context, layer_name):
+    def stack(self, context, layer_name, progress_callback = None):
         """Method to determine whether this builder can operate on the named layer,
            If so, modify the context appropriately.
 

volatility/framework/interfaces/layers.py

@@ -172,7 +172,7 @@ class DataLayerInterface(configuration.ConfigurableInterface, validity.ValidityR
                     while not result.ready():
                         if progress_callback:
                             # Run the progress_callback
-                            progress_callback(scan_metric(progress.value))
+                            progress_callback(scan_metric(progress.value), "Scanning {}".format(self.name))
                         # Ensures we don't burn CPU cycles going round in a ready waiting loop
                         # without delaying the user too long between progress updates/results
                         result.wait(0.1)
@@ -196,7 +196,7 @@ class DataLayerInterface(configuration.ConfigurableInterface, validity.ValidityR
         return list(scanner(chunk, iterator_value))
 
     def _scan_metric(self, _scanner, min_address, max_address, value):
-        return (value * 100) / (max_address - min_address)
+        return max(0, (value * 100) / (max_address - min_address))
 
     def build_configuration(self):
         config = super().build_configuration()

volatility/framework/layers/intel.py

@@ -185,7 +185,7 @@ class Intel(interfaces.layers.TranslationLayerInterface):
         return list(scanner(data, chunk_end - len(data_to_scan)))
 
     def _scan_metric(self, _scanner, min_address, max_address, value):
-        return ((value - min_address) * 100) / (max_address - min_address)
+        return max(0, ((value - min_address) * 100) / (max_address - min_address))
 
 
 class IntelPAE(Intel):

volatility/framework/layers/lime.py

@@ -75,7 +75,7 @@ class LimeStacker(interfaces.automagic.StackerLayerInterface):
     stack_order = 10
 
     @classmethod
-    def stack(cls, context, layer_name):
+    def stack(cls, context, layer_name, progress_callback = None):
         try:
             LimeLayer._check_header(context.memory[layer_name])
         except LimeFormatException:

volatility/framework/layers/vmware.py

@@ -103,7 +103,7 @@ class VmwareLayer(segmented.SegmentedLayer):
 
 class VmwareStacker(interfaces.automagic.StackerLayerInterface):
     @classmethod
-    def stack(cls, context, layer_name):
+    def stack(cls, context, layer_name, progress_callback = None):
         """Attempt to stack this based on the starting information"""
         if not isinstance(context.memory[layer_name], physical.FileLayer):
             return