Merge branch 'volatilityfoundation:develop' into linux_sockstats_plugin

.readthedocs.yml

@@ -16,7 +16,4 @@ formats: all
 python:
   version: 3.7
   install:
-    - method: pip
-      path: .
-      extra_requirements:
-        - doc
+    - requirements: doc/requirements.txt

MANIFEST.in

@@ -1,6 +1,6 @@
 prune development
 include * .*
-include doc/make.bat doc/Makefile
+include doc/make.bat doc/Makefile doc/requirements.txt
 recursive-include doc/source *
 recursive-include volatility3 *.json
 recursive-exclude doc/source volatility3.*.rst

README.md

@@ -14,7 +14,9 @@ technical and performance challenges associated with the original
 code base that became apparent over the previous 10 years. Another benefit
 of the rewrite is that Volatility 3 could be released under a custom
 license that was more aligned with the goals of the Volatility community,
-the Volatility Software License (VSL). See the [LICENSE](LICENSE.txt) file for more details.
+the Volatility Software License (VSL). See the 
+[LICENSE](https://www.volatilityfoundation.org/license/vsl-v1.0) file for 
+more details.
 
 ## Requirements
 
@@ -39,7 +41,7 @@ pip3 install -r requirements.txt
 
 ## Downloading Volatility
 
-The latest stable version of Volatility will always be the master branch of the GitHub repository. You can get the latest version of the code using the following command:
+The latest stable version of Volatility will always be the stable branch of the GitHub repository. You can get the latest version of the code using the following command:
 
 ```shell
 git clone https://github.com/volatilityfoundation/volatility3.git
@@ -102,7 +104,7 @@ The latest generated copy of the documentation can be found at: <https://volatil
 
 ## Licensing and Copyright
 
-Copyright (C) 2007-2021 Volatility Foundation
+Copyright (C) 2007-2022 Volatility Foundation
 
 All Rights Reserved
 

development/stock-linux-json.py

@@ -87,7 +87,7 @@ class Downloader:
         output_filename = 'unknown-kernel.json'
         for named_file in named_files:
             prefix = '--system-map'
-            if not 'System' in named_files[named_file]:
+            if 'System' not in named_files[named_file]:
                 prefix = '--elf'
                 output_filename = './' + '-'.join((named_file.split('/')[-1]).split('-')[2:])[:-4] + '.json.xz'
             args += [prefix, named_files[named_file]]

doc/requirements.txt

@@ -1,4 +1,4 @@
 # These packages are required for building the documentation.
-sphinx>=1.8.2
+sphinx>=4.0.0
 sphinx_autodoc_typehints>=1.4.0
-sphinx-rtd-theme>=0.4.3
+sphinx-rtd-theme>=0.4.3

doc/source/conf.py

@@ -1,4 +1,4 @@
-# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0
+# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0
 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
 #
 #
@@ -84,6 +84,15 @@ def setup(app):
                     for line in submodule_lines:
                         contents.write(line.replace(b'volatility3.framework.plugins', b'volatility3.plugins'))
 
+    # Clear up the framework.plugins page
+    with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "rb") as contents:
+        real_lines = contents.readlines()
+
+    with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "wb") as contents:
+        for line in real_lines:
+            if b'volatility3.framework.plugins.' not in line:
+                contents.write(line)
+
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -126,7 +135,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'Volatility 3'
-copyright = '2012-2019, Volatility Foundation'
+copyright = '2012-2022, Volatility Foundation'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

doc/source/development.rst

@@ -0,0 +1,8 @@
+Writing Plugins
+===============
+
+.. toctree::
+
+    simple-plugin
+    complex-plugin
+    using-as-a-library

doc/source/index.rst

@@ -12,11 +12,9 @@ Here are some guidelines for using Volatility 3 effectively:
 .. toctree::
 
     basics
-    simple-plugin
-    vol2to3
-    complex-plugin
-    using-as-a-library
+    development
     symbol-tables
+    vol2to3
     volshell
     glossary
 

doc/source/symbol-tables.rst

@@ -76,3 +76,21 @@ The banners available for volatility to use can be found using the `isfinfo` plu
 long time to run depending on the number of JSON files available.  This will list all the JSON (ISF) files that
 volatility3 is aware of, and for linux/mac systems what banner string they search for.  For volatility to use the JSON
 file, the banners must match exactly (down to the compilation date).
+
+.. note::
+
+  Steps for constructing a new kernel ISF JSON file:
+
+  * Run the `banners` plugin on the image to determine the necessary kernel
+  * Locate a copy of the debug kernel that matches the identified banner
+
+    * Clone or update the dwarf2json repo: :code:`git clone https://github.com/volatilityfoundation/dwarf2json`
+    * Run :code:`go build` in the directory if the source has changed
+
+  * Run :code:`dwarf2json linux --elf [path to debug kernel] > [kernel name].json`
+
+    * For Mac change `linux` to `mac`
+
+  * Copy the `.json` file to the symbols directory into `[symbols directory]/linux`
+
+    * For Mac change `linux` to `mac`

doc/source/using-as-a-library.rst

@@ -131,7 +131,8 @@ A suitable list of automagics for a particular plugin (based on operating system
     automagics = automagic.choose_automagic(available_automagics, plugin)
 
 This will take the plugin module, extract the operating system (first level of the hierarchy) and then return just
-the automagics which apply to the operating system.
+the automagics which apply to the operating system.  Each automagic can exclude itself from being used for specific
+operating systems, so that an automagic designed for linux is not used for windows or mac plugins.
 
 These automagics can then be run by providing the list, the context, the plugin to be run, the hierarchy name that
 the plugin will be constructed on ('plugins' by default) and a progress_callback.  This is a callable which takes

doc/source/vol2to3.rst

@@ -62,6 +62,10 @@ automagic processes are clearly defined and can be enabled or disabled as necess
 included a stacker automagic to emulate the most common feature of Volatility 2, automatically stacking address spaces
 (now translation layers) on top of each other.
 
+By default the automagic chosen to be run are determined based on the plugin requested, so that linux plugins get linux
+specific automagic and windows plugins get windows specific automagic.  This should reduce unnecessarily searching for
+linux kernels in a windows image, for example.  At the moment this is not user configurableS.
+
 Searching and Scanning
 ----------------------
 Scanning is very similar to scanning in Volatility 2, a scanner object (such as a

doc/source/volshell.rst

@@ -22,7 +22,7 @@ be run.
 When volshell starts, it will show the version of volshell, a brief message indicating how to get more help, the current
 operating system mode for volshell, and the current layer available for use.
 
-.. code-block:: python
+::
 
     Volshell (Volatility 3 Framework) 1.0.1
     Readline imported successfully	PDB scanning finished
@@ -53,7 +53,7 @@ run our examples against.
 We'll start by creating a process variable, and putting the first result from `ps()` in it.  Since the shell is a
 python environment, we can do the following:
 
-.. code-block:: python
+::
 
     (primary) >>> proc = ps()[0]
     (primary) >>> proc
@@ -68,7 +68,7 @@ built-in mechanism for providing more information about a structure, called `dis
 either a type name (which if not prefixed with symbol table name, will use the kernel symbol table identified by the
 automagic).
 
-.. code-block:: python
+::
 
     (primary) >>> dt('_EPROCESS')
     nt_symbols1!_EPROCESS (2624 bytes)
@@ -80,7 +80,7 @@ automagic).
 
 It can also be provided with an object and will interpret the data for each in the process:
 
-.. code-block:: python
+::
 
     (primary) >>> dt(proc)
     nt_symbols1!_EPROCESS (2624 bytes)
@@ -92,7 +92,7 @@ It can also be provided with an object and will interpret the data for each in t
 
 These values can be accessed directory as attributes
 
-.. code-block:: python
+::
 
     (primary) >>> proc.UniqueProcessId
     356
@@ -100,7 +100,7 @@ These values can be accessed directory as attributes
 Pointer structures contain the value they point to, but attributes accessed are forwarded to the object they point to.
 This means that pointers do not need to be explicitly dereferenced to access underling objects.
 
-.. code-block:: python
+::
 
     (primary) >>> proc.Pcb.DirectoryTableBase
     4355817472
@@ -112,7 +112,7 @@ It's possible to run any plugin by importing it appropriately and passing it to
 method.  In the following example we'll provide no additional parameters.  Volatility will show us which parameters
 were required:
 
-.. code-block:: python
+::
 
     (primary) >>> from volatility3.plugins.windows import pslist
     (primary) >>> display_plugin_output(pslist.PsList)
@@ -124,14 +124,14 @@ was fulfilled.
 We can see all the options that the plugin can accept by access the `get_requirements()` method of the plugin.
 This is a classmethod, so can be called on an uninstantiated copy of the plugin.
 
-.. code-block:: python
+::
 
     (primary) >>> pslist.PsList.get_requirements()
     [<TranslationLayerRequirement: primary>, <SymbolTableRequirement: nt_symbols>, <BooleanRequirement: physical>, <ListRequirement: pid>, <BooleanRequirement: dump>]
 
 We can provide arguments via the `dpo` method call:
 
-.. code-block:: python
+::
 
     (primary) >>> display_plugin_output(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols'])
 
@@ -149,7 +149,7 @@ by the `dpo` method is always `context`.
 Instead of print the results directly to screen, they can be gathered into a TreeGrid objects for direct access by
 using the `generate_treegrid` or `gt` command.
 
-.. code-block:: python
+::
 
     (primary) >>> treegrid = gt(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols'])
     (primary) >>> treegrid.populate()

volatility3/cli/__init__.py

@@ -543,7 +543,7 @@ class CommandLine:
                 self._file = io.open(fd, mode = 'w+b')
                 CLIFileHandler.__init__(self, filename)
                 for item in dir(self._file):
-                    if not item.startswith('_') and not item in ['closed', 'close', 'mode', 'name']:
+                    if not item.startswith('_') and item not in ('closed', 'close', 'mode', 'name'):
                         setattr(self, item, getattr(self._file, item))
 
             def __getattr__(self, item):

volatility3/cli/text_renderer.py

@@ -182,6 +182,17 @@ class QuickTextRenderer(CLIRenderer):
         outfd.write("\n")
 
 
+class NoneRenderer(CLIRenderer):
+    """Outputs no results"""
+    name = "none"
+
+    def get_render_options(self):
+        pass
+
+    def render(self, grid: interfaces.renderers.TreeGrid) -> None:
+        if not grid.populated:
+            grid.populate(lambda x, y: True, True)
+
 class CSVRenderer(CLIRenderer):
     _type_renderers = {
         format_hints.Bin: quoted_optional(lambda x: f"0b{x:b}"),
@@ -272,9 +283,10 @@ class PrettyTextRenderer(CLIRenderer):
                 column = grid.columns[column_index]
                 renderer = self._type_renderers.get(column.type, self._type_renderers['default'])
                 data = renderer(node.values[column_index])
+                field_width = max([len(self.tab_stop(x)) for x in f"{data}".split("\n")])
                 max_column_widths[column.name] = max(max_column_widths.get(column.name, len(column.name)),
-                                                     len(f"{data}"))
-                line[column] = data
+                                                     field_width)
+                line[column] = data.split("\n")
             accumulator.append((node.path_depth, line))
             return accumulator
 
@@ -296,7 +308,25 @@ class PrettyTextRenderer(CLIRenderer):
         column_titles = [""] + [column.name for column in grid.columns]
         outfd.write(format_string.format(*column_titles))
         for (depth, line) in final_output:
-            outfd.write(format_string.format("*" * depth, *[line[column] for column in grid.columns]))
+            nums_line = max([len(line[column]) for column in line])
+            for column in line:
+                line[column] = line[column] + ([""] * (nums_line - len(line[column])))
+            for index in range(nums_line):
+                if index == 0:
+                    outfd.write(format_string.format("*" * depth, *[self.tab_stop(line[column][index]) for column in grid.columns]))
+                else:
+                    outfd.write(format_string.format(" " * depth, *[self.tab_stop(line[column][index]) for column in grid.columns]))
+
+    def tab_stop(self, line: str) -> str:
+        tab_width = 8
+        while line.find('\t') >= 0:
+            i = line.find('\t')
+            if (tab_width > 0):
+                pad = " " * (tab_width - (i % tab_width))
+            else:
+                pad = ""
+            line = line.replace("\t", pad, 1)
+        return line
 
 
 class JsonRenderer(CLIRenderer):

volatility3/cli/volshell/__init__.py

@@ -7,6 +7,7 @@ import json
 import logging
 import os
 import sys
+import glob
 
 import volatility3.plugins
 import volatility3.symbols
@@ -137,8 +138,7 @@ class VolShell(cli.CommandLine):
             console.setLevel(10 - (partial_args.verbosity - 2))
 
         if partial_args.clear_cache:
-            for cache_filename in glob.glob(os.path.join(constants.CACHE_PATH, '*.cache')):
-                os.unlink(cache_filename)
+            framework.clear_cache()
 
         # Do the initialization
         ctx = contexts.Context()  # Construct a blank context
@@ -238,9 +238,14 @@ class VolShell(cli.CommandLine):
                 vollog.debug("Writing out configuration data to config.json")
                 with open("config.json", "w") as f:
                     json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2)
+        except exceptions.UnsatisfiedException as excp:
+            self.process_unsatisfied_exceptions(excp)
+            parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n")
 
+        try:
             # Construct and run the plugin
-            constructed.run()
+            if constructed:
+                constructed.run()
         except exceptions.VolatilityException as excp:
             self.process_exceptions(excp)
             parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n")

volatility3/cli/volshell/linux.py

@@ -17,7 +17,7 @@ class Volshell(generic.Volshell):
     def get_requirements(cls):
         return (super().get_requirements() + [
             requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"),
-            requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)),
+            requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)),
             requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True)
         ])
 

volatility3/framework/automagic/__init__.py

@@ -21,14 +21,6 @@ from volatility3.framework.configuration import requirements
 
 vollog = logging.getLogger(__name__)
 
-windows_automagic = [
-    'ConstructionMagic', 'LayerStacker', 'KernelPDBScanner', 'WinSwapLayers', 'KernelModule'
-]
-
-linux_automagic = ['ConstructionMagic', 'LayerStacker', 'LinuxBannerCache', 'LinuxSymbolFinder', 'KernelModule']
-
-mac_automagic = ['ConstructionMagic', 'LayerStacker', 'MacBannerCache', 'MacSymbolFinder', 'KernelModule']
-
 
 def available(context: interfaces.context.ContextInterface) -> List[interfaces.automagic.AutomagicInterface]:
     """Returns an ordered list of all subclasses of
@@ -58,10 +50,7 @@ def choose_automagic(
     plugin_category = "None"
     plugin_categories = plugin.__module__.split('.')
     lowest_index = len(plugin_categories)
-
-    automagic_categories = {'windows': windows_automagic, 'linux': linux_automagic, 'mac': mac_automagic}
-
-    for os in automagic_categories:
+    for os in constants.OS_CATEGORIES:
         try:
             if plugin_categories.index(os) < lowest_index:
                 lowest_index = plugin_categories.index(os)
@@ -70,14 +59,16 @@ def choose_automagic(
             # The value wasn't found, try the next one
             pass
 
-    if plugin_category not in automagic_categories:
+    if plugin_category not in constants.OS_CATEGORIES:
         vollog.info("No plugin category detected")
         return automagics
-
     vollog.info(f"Detected a {plugin_category} category plugin")
+
     output = []
     for amagic in automagics:
-        if amagic.__class__.__name__ in automagic_categories[plugin_category]:
+        if plugin_category not in amagic.exclusion_list:
+            # Only include uncategorized automagic, or platform specific automagic
+            # (This allows user defined/uncategorized automagic to be included)
             output += [amagic]
     return output
 

volatility3/framework/automagic/linux.py

@@ -147,6 +147,7 @@ class LinuxBannerCache(symbol_cache.SymbolBannerCache):
     os = "linux"
     symbol_name = "linux_banner"
     banner_path = constants.LINUX_BANNERS_PATH
+    exclusion_list = ['mac', 'windows']
 
 
 class LinuxSymbolFinder(symbol_finder.SymbolFinder):
@@ -156,3 +157,4 @@ class LinuxSymbolFinder(symbol_finder.SymbolFinder):
     banner_cache = LinuxBannerCache
     symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols"
     find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1]
+    exclusion_list = ['mac', 'windows']

volatility3/framework/automagic/mac.py

@@ -202,6 +202,7 @@ class MacBannerCache(symbol_cache.SymbolBannerCache):
     os = "mac"
     symbol_name = "version"
     banner_path = constants.MAC_BANNERS_PATH
+    exclusion_list = ['windows', 'linux']
 
 
 class MacSymbolFinder(symbol_finder.SymbolFinder):
@@ -211,3 +212,4 @@ class MacSymbolFinder(symbol_finder.SymbolFinder):
     banner_cache = MacBannerCache
     find_aslr = MacIntelStacker.find_aslr
     symbol_class = "volatility3.framework.symbols.mac.MacKernelIntermedSymbols"
+    exclusion_list = ['windows', 'linux']

volatility3/framework/automagic/pdbscan.py

@@ -44,6 +44,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
     """
     priority = 30
     max_pdb_size = 0x400000
+    exclusion_list = ['linux', 'mac']
 
     def find_virtual_layers_from_req(self, context: interfaces.context.ContextInterface, config_path: str,
                                      requirement: interfaces.configuration.RequirementInterface) -> List[str]:

volatility3/framework/automagic/symbol_finder.py

@@ -5,7 +5,7 @@
 import logging
 from typing import Any, Iterable, List, Tuple, Type, Optional, Callable
 
-from volatility3.framework import interfaces, constants
+from volatility3.framework import interfaces, constants, layers
 from volatility3.framework.automagic import symbol_cache
 from volatility3.framework.configuration import requirements
 from volatility3.framework.layers import scanners

volatility3/framework/automagic/windows.py

@@ -238,6 +238,8 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface):
     """Class to read swap_layers filenames from single-swap-layers, create the
     layers and populate the single-layers swap_layers."""
 
+    exclusion_list = ['linux', 'mac']
+
     def __call__(self,
                  context: interfaces.context.ContextInterface,
                  config_path: str,

volatility3/framework/constants/__init__.py

@@ -40,7 +40,7 @@ BANG = "!"
 # We use the SemVer 2.0.0 versioning scheme
 VERSION_MAJOR = 2  # Number of releases of the library with a breaking change
 VERSION_MINOR = 0  # Number of changes that only add to the interface
-VERSION_PATCH = 0  # Number of changes that do not change the interface
+VERSION_PATCH = 2  # Number of changes that do not change the interface
 VERSION_SUFFIX = ""
 
 # TODO: At version 2.0.0, remove the symbol_shift feature
@@ -78,6 +78,7 @@ BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues"
 ProgressCallback = Optional[Callable[[float, str], None]]
 """Type information for ProgressCallback objects"""
 
+OS_CATEGORIES = ['windows', 'mac', 'linux']
 
 class Parallelism(enum.IntEnum):
     """An enumeration listing the different types of parallelism applied to

volatility3/framework/constants/windows/__init__.py

@@ -1,7 +1,7 @@
 # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0
 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
 #
-"""Volatility 3 Linux Constants.
+"""Volatility 3 Windows Constants.
 
 Windows-specific values that aren't found in debug symbols
 """

volatility3/framework/interfaces/automagic.py

@@ -40,6 +40,9 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla
     priority = 10
     """An ordering to indicate how soon this automagic should be run"""
 
+    exclusion_list = []
+    """A list of plugin categories (typically operating systems) which the plugin will not operate on"""
+
     def __init__(self, context: interfaces.context.ContextInterface, config_path: str, *args, **kwargs) -> None:
         super().__init__(context, config_path)
         for requirement in self.get_requirements():

volatility3/framework/layers/qemu.py

@@ -54,7 +54,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
 
     def _read_configuration(self, base_layer: interfaces.layers.DataLayerInterface, name: str) -> Any:
         """Reads the JSON configuration from the end of the file"""
-        chunk_size = 0x4096
+        chunk_size = 4096
         data = b''
         for i in range(base_layer.maximum_address, base_layer.minimum_address, -chunk_size):
             if i != base_layer.maximum_address:
@@ -65,6 +65,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
                     if start_of_json >= 0:
                         data = data[start_of_json:]
                         return json.loads(data)
+                    # No JSON configuration found at the end of the file, return empty dict
                     return dict()
         raise exceptions.LayerException(name, "Invalid JSON configuration at the end of the file")
 
@@ -79,9 +80,11 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
             addr = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long',
                                        offset = index,
                                        layer_name = self._base_layer)
+            # Flags are stored in the n least significant bits, where n equals the bit-length of pagesize
             flags = addr & (page_size - 1)
-            page_size_bits = int(math.log(page_size, 2))
-            addr = (addr >> page_size_bits) << page_size_bits
+            # addr equals the highest multiple of pagesize <= offset
+            # (We assume that page_size is a power of 2)
+            addr = addr ^ (addr & (page_size - 1))
             index += 8
 
             if flags & self.SEGMENT_FLAG_MEM_SIZE:
@@ -126,6 +129,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
             self._configuration = self._read_configuration(base_layer, self.name)
         section_byte = -1
         index = 8
+        section_info = dict()
         current_section_id = -1
         version_id = -1
         name = None
@@ -162,6 +166,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
                                                  offset = index,
                                                  layer_name = self._base_layer)
                 index += 4
+                # Store section info for handling QEVM_SECTION_PARTs later on
+                section_info[current_section_id] = {'name': name, 'version_id': version_id}
                 # Read additional data
                 index = self.extract_data(index, name, version_id)
             elif section_byte == self.QEVM_SECTION_PART or section_byte == self.QEVM_SECTION_END:
@@ -171,7 +177,8 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
                 current_section_id = section_id
                 index += 4
                 # Read additional data
-                index = self.extract_data(index, name, version_id)
+                index = self.extract_data(index, section_info[current_section_id]['name'],
+                                          section_info[current_section_id]['version_id'])
             elif section_byte == self.QEVM_SECTION_FOOTER:
                 section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long',
                                                  offset = index,
@@ -189,7 +196,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
         if name == 'ram':
             if version_id != 4:
                 raise exceptions.LayerException(f"QEMU unknown RAM version_id {version_id}")
-            new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', None) or 4096)
+            new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', 4096))
             self._segments += new_segments
         elif name == 'spapr/htab':
             if version_id != 1:
@@ -208,6 +215,13 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
                                                layer_name = self._base_layer)
                     htab_index, htab_n_valid, htab_n_invalid = htab
                     index += 8 + (htab_n_valid * self.HASH_PTE_SIZE_64)
+        elif name == 'dirty-bitmap':
+            index += 1
+        elif name == 'pbs-state':
+            section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long',
+                                              offset = index,
+                                              layer_name = self._base_layer)
+            index += 8 + section_len
         return index
 
     def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes:
@@ -217,9 +231,12 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
         of the starting data.  It is the responsibility of the layer to turn the provided data chunk into the right
         portion of data necessary.
         """
-        start_offset = offset ^ (offset & 0xfff)
+        page_size = self._configuration.get('page_size', 4096)
+        # start_offset equals the highest multiple of pagesize <= offset
+        # (We assume that page_size is a power of 2)
+        start_offset = offset ^ (offset & (page_size - 1))
         if start_offset in self._compressed:
-            data = (data * 0x1000)
+            data = (data * page_size)
         result = data[offset - start_offset:output_length + offset - start_offset]
         return result
 

volatility3/framework/layers/resources.py

@@ -82,7 +82,7 @@ class ResourceAccessor(object):
         """Determines whether a URLs contents should be cached"""
         parsed_url = urllib.parse.urlparse(url)
 
-        return self._enable_cache and not parsed_url.scheme in self._non_cached_schemes()
+        return self._enable_cache and parsed_url.scheme not in self._non_cached_schemes()
 
     @staticmethod
     def _non_cached_schemes() -> List[str]:

volatility3/framework/objects/__init__.py

@@ -6,9 +6,9 @@ import collections
 import collections.abc
 import logging
 import struct
-from typing import Any, ClassVar, Dict, List, Iterable, Optional, Tuple, Type, Union as TUnion, overload
+from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple, Type, Union as TUnion, overload
 
-from volatility3.framework import interfaces, constants
+from volatility3.framework import constants, interfaces
 from volatility3.framework.objects import templates, utility
 
 vollog = logging.getLogger(__name__)

volatility3/framework/plugins/layerwriter.py

@@ -110,9 +110,9 @@ class LayerWriter(plugins.PluginInterface):
     def _generate_layers(self):
         """List layer names from this run"""
         for name in self.context.layers:
-            yield (0, (name, ))
+            yield (0, (name, self.context.layers[name].__class__.__name__))
 
     def run(self):
         if self.config['list']:
-            return renderers.TreeGrid([("Layer name", str)], self._generate_layers())
+            return renderers.TreeGrid([("Layer name", str), ('Layer type', str)], self._generate_layers())
         return renderers.TreeGrid([("Status", str)], self._generator())

volatility3/framework/plugins/linux/check_creds.py

@@ -44,7 +44,7 @@ class Check_creds(interfaces.plugins.PluginInterface):
 
             cred_addr = task.cred.dereference().vol.offset
 
-            if not cred_addr in creds:
+            if cred_addr not in creds:
                 creds[cred_addr] = []
 
             creds[cred_addr].append(task.pid)

volatility3/framework/plugins/linux/check_syscall.py

@@ -152,7 +152,7 @@ class Check_syscall(plugins.PluginInterface):
         except exceptions.SymbolError:
             ia32_symbol = None
 
-        if ia32_symbol != None:
+        if ia32_symbol is not None:
             ia32_info = self._get_table_info(vmlinux, "ia32_sys_call_table", ptr_sz)
             tables.append(("32bit", ia32_info))
 

volatility3/framework/plugins/linux/kmsg.py

@@ -4,9 +4,9 @@
 import logging
 from abc import ABC, abstractmethod
 from enum import Enum
-from typing import List, Iterator, Tuple, Generator
+from typing import Generator, Iterator, List, Tuple
 
-from volatility3.framework import renderers, interfaces, constants, contexts, class_subclasses
+from volatility3.framework import class_subclasses, constants, contexts, interfaces, renderers
 from volatility3.framework.configuration import requirements
 from volatility3.framework.interfaces import plugins
 from volatility3.framework.objects import utility
@@ -15,39 +15,39 @@ vollog = logging.getLogger(__name__)
 
 
 class DescStateEnum(Enum):
-    desc_miss = -1          # ID mismatch (pseudo state)
-    desc_reserved = 0x0     # reserved, in use by writer
-    desc_committed = 0x1    # committed by writer, could get reopened
-    desc_finalized = 0x2    # committed, no further modification allowed
-    desc_reusable = 0x3     # free, not yet used by any writer
+    desc_miss = -1  # ID mismatch (pseudo state)
+    desc_reserved = 0x0  # reserved, in use by writer
+    desc_committed = 0x1  # committed by writer, could get reopened
+    desc_finalized = 0x2  # committed, no further modification allowed
+    desc_reusable = 0x3  # free, not yet used by any writer
 
 
 class ABCKmsg(ABC):
     """Kernel log buffer reader"""
     LEVELS = (
-        "emerg",     # system is unusable
-        "alert",     # action must be taken immediately
-        "crit",      # critical conditions
-        "err",       # error conditions
-        "warn",      # warning conditions
-        "notice",    # normal but significant condition
-        "info",      # informational
-        "debug",     # debug-level messages
+        "emerg",  # system is unusable
+        "alert",  # action must be taken immediately
+        "crit",  # critical conditions
+        "err",  # error conditions
+        "warn",  # warning conditions
+        "notice",  # normal but significant condition
+        "info",  # informational
+        "debug",  # debug-level messages
     )
 
     FACILITIES = (
-        "kern",      # kernel messages
-        "user",      # random user-level messages
-        "mail",      # mail system
-        "daemon",    # system daemons
-        "auth",      # security/authorization messages
-        "syslog",    # messages generated internally by syslogd
-        "lpr",       # line printer subsystem
-        "news",      # network news subsystem
-        "uucp",      # UUCP subsystem
-        "cron",      # clock daemon
+        "kern",  # kernel messages
+        "user",  # random user-level messages
+        "mail",  # mail system
+        "daemon",  # system daemons
+        "auth",  # security/authorization messages
+        "syslog",  # messages generated internally by syslogd
+        "lpr",  # line printer subsystem
+        "news",  # network news subsystem
+        "uucp",  # UUCP subsystem
+        "cron",  # clock daemon
         "authpriv",  # security/authorization messages (private)
-        "ftp"        # FTP daemon
+        "ftp"  # FTP daemon
     )
 
     def __init__(
@@ -247,12 +247,20 @@ class KmsgFiveTen(ABCKmsg):
     The data block ring 'text_data_ring' contains the records' text strings.
     A pointer to the high level structure is kept in the prb pointer which is
     initialized to a static ringbuffer.
+
+    .. code-block:: c
+
         static struct printk_ringbuffer *prb = &printk_rb_static;
+
     In SMP systems with more than 64 CPUs this ringbuffer size is dynamically
     allocated according the number of CPUs based on the value of
     CONFIG_LOG_CPU_MAX_BUF_SHIFT. The prb pointer is updated consequently to
     this dynamic ringbuffer in setup_log_buf().
+
+    .. code-block:: c
+
         prb = &printk_rb_dynamic;
+
     Behind scenes, log_buf is still used as external buffer.
     When the static printk_ringbuffer struct is initialized, _DEFINE_PRINTKRB
     sets text_data_ring.data pointer to the address in log_buf which points to
@@ -262,12 +270,14 @@ class KmsgFiveTen(ABCKmsg):
     buffer via the prb_init function.
     In that case, the original external static buffer in __log_buf and
     printk_rb_static are unused.
-        ...
+
+    .. code-block:: c
+
         new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN);
         prb_init(&printk_rb_dynamic, new_log_buf, ...);
         log_buf = new_log_buf;
         prb = &printk_rb_dynamic;
-        ...
+
     See printk.c and printk_ringbuffer.c in kernel/printk/ folder for more
     details.
     """

volatility3/framework/plugins/mac/list_files.py

@@ -72,7 +72,7 @@ class List_Files(plugins.PluginInterface):
         key = vnode.vol.offset
         added = False
 
-        if not key in loop_vnodes:
+        if key not in loop_vnodes:
             # We can't do anything with a no-name vnode
             v_name = cls._vnode_name(vnode)
             if v_name is None:
@@ -108,7 +108,7 @@ class List_Files(plugins.PluginInterface):
             added = True
 
             parent = cls._get_parent(context, vnode)
-            while parent and not parent in loop_vnodes:
+            while parent and parent not in loop_vnodes:
                 if not cls._walk_vnode(context, parent, loop_vnodes):
                     break
 

volatility3/framework/plugins/timeliner.py

@@ -12,7 +12,7 @@ import traceback
 from typing import Generator, Iterable, List, Optional, Tuple, Type
 
 from volatility3 import framework
-from volatility3.framework import renderers, automagic, interfaces, plugins, exceptions
+from volatility3.framework import automagic, exceptions, interfaces, plugins, renderers
 from volatility3.framework.configuration import requirements
 
 vollog = logging.getLogger(__name__)
@@ -74,10 +74,6 @@ class Timeliner(interfaces.plugins.PluginInterface):
     @classmethod
     def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
         return [
-            requirements.StringRequirement(name = 'plugins',
-                                           description = "Comma separated list of plugins to run",
-                                           optional = True,
-                                           default = None),
             requirements.BooleanRequirement(
                 name = 'record-config',
                 description = "Whether to record the state of all the plugins once complete",
@@ -110,6 +106,15 @@ class Timeliner(interfaces.plugins.PluginInterface):
         row from each plugin."""
         # Generate the results for each plugin
         data = []
+
+        # Open the bodyfile now, so we can start outputting to it immediately
+        if self.config.get('create-bodyfile', True):
+            file_data = self.open("volatility.body")
+            fp = io.TextIOWrapper(file_data, write_through = True)
+        else:
+            file_data = None
+            fp = None
+
         for plugin in runable_plugins:
             plugin_name = plugin.__class__.__name__
             self._progress_callback((runable_plugins.index(plugin) * 100) // len(runable_plugins),
@@ -130,27 +135,31 @@ class Timeliner(interfaces.plugins.PluginInterface):
                         times.get(TimeLinerType.ACCESSED, renderers.NotApplicableValue()),
                         times.get(TimeLinerType.CHANGED, renderers.NotApplicableValue())
                     ]))
-            except Exception:
-                vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}")
-                vollog.log(logging.DEBUG, traceback.format_exc())
-        for data_item in sorted(data, key = self._sort_function):
-            yield data_item
 
-        # Write out a body file if necessary
-        if self.config.get('create-bodyfile', True):
-            with self.open("volatility.body") as file_data:
-                with io.TextIOWrapper(file_data, write_through = True) as fp:
-                    for (plugin_name, item) in self.timeline:
+                    # Write each entry because the body file doesn't need to be sorted
+                    if fp:
                         times = self.timeline[(plugin_name, item)]
                         # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime
 
                         if self._any_time_present(times):
-                            fp.write("|{} - {}||||||{}|{}|{}|{}\n".format(
+                            fp.write("|{} - {}|0|0|0|0|0|{}|{}|{}|{}\n".format(
                                 plugin_name, self._sanitize_body_format(item),
                                 self._text_format(times.get(TimeLinerType.ACCESSED, "")),
                                 self._text_format(times.get(TimeLinerType.MODIFIED, "")),
                                 self._text_format(times.get(TimeLinerType.CHANGED, "")),
                                 self._text_format(times.get(TimeLinerType.CREATED, ""))))
+            except Exception:
+                vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}")
+                vollog.log(logging.DEBUG, traceback.format_exc())
+
+        for data_item in sorted(data, key = self._sort_function):
+            yield data_item
+
+        # Write out a body file if necessary
+        if self.config.get('create-bodyfile', True):
+            if fp:
+                fp.close()
+                file_data.close()
 
     def _sanitize_body_format(self, value):
         return value.replace("|", "_")
@@ -164,7 +173,7 @@ class Timeliner(interfaces.plugins.PluginInterface):
     def _text_format(self, value):
         """Formats a value as text, in case it is an AbsentValue"""
         if isinstance(value, interfaces.renderers.BaseAbsentValue):
-            return ""
+            return "0"
         if isinstance(value, datetime.datetime):
             return int(value.timestamp())
         return value
@@ -202,7 +211,7 @@ class Timeliner(interfaces.plugins.PluginInterface):
 
                 if isinstance(plugin, TimeLinerInterface):
                     if not len(filter_list) or any(
-                        [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]):
+                            [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]):
                         plugins_to_run.append(plugin)
             except exceptions.UnsatisfiedException as excp:
                 # Remove the failed plugin from the list and continue

volatility3/framework/plugins/windows/ldrmodules.py

@@ -0,0 +1,99 @@
+from volatility3.framework import interfaces, constants
+from volatility3.framework import renderers, interfaces, exceptions
+from volatility3.framework.configuration import requirements
+from volatility3.framework.objects import utility
+from volatility3.framework.renderers import format_hints
+from volatility3.framework.symbols import intermed
+from volatility3.framework.symbols.windows.extensions import pe
+from volatility3.plugins.windows import pslist, vadinfo
+
+class LdrModules(interfaces.plugins.PluginInterface):
+    _required_framework_version = (2, 0, 0)
+    _version = (1, 0, 0)
+ 
+    @classmethod
+    def get_requirements(cls):
+        return [
+            requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]),
+            requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)),
+            requirements.VersionRequirement(name = 'vadinfo', component = vadinfo.VadInfo, version = (2, 0, 0)),
+            requirements.ListRequirement(name = 'pid',
+                                         element_type = int,
+                                         description = "Process IDs to include (all other processes are excluded)",
+                                         optional = True),
+           ]
+
+    def _generator(self, procs):
+
+        pe_table_name = intermed.IntermediateSymbolTable.create(self.context,
+                                                                self.config_path,
+                                                                "windows",
+                                                                "pe",
+                                                                class_types = pe.class_types)
+
+        def filter_function(x: interfaces.objects.ObjectInterface) -> bool:
+            try:
+                return not (x.get_private_memory() == 0 and x.ControlArea)
+            except AttributeError:
+                return False
+
+        filter_func = filter_function   
+
+        for proc in procs:
+            proc_layer_name = proc.add_process_layer()
+
+            # Build dictionaries from different module lists, where the DllBase address is the key and value is the module object
+            load_order_mod = dict((mod.DllBase, mod)
+                                for mod in proc.load_order_modules())
+            init_order_mod = dict((mod.DllBase, mod)
+                                for mod in proc.init_order_modules())
+            mem_order_mod = dict((mod.DllBase, mod)
+                                for mod in proc.mem_order_modules())
+
+            # Build dictionary of mapped files, where the VAD start address is the key and value is the file name of the mapped file
+            mapped_files = {}
+            for vad in vadinfo.VadInfo.list_vads(proc, filter_func = filter_func):
+                dos_header = self.context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER",
+                                   offset = vad.get_start(),
+                                   layer_name = proc_layer_name)
+                try:
+                    # Filter out VADs that do not start with a MZ header
+                    if dos_header.e_magic != 0x5A4D:
+                        continue
+                except exceptions.PagedInvalidAddressException:
+                    continue
+
+                mapped_files[vad.get_start()] = vad.get_file_name()
+
+            for base in mapped_files.keys():
+                # Does the base address exist in the PEB DLL lists?
+                load_mod = load_order_mod.get(base, None)
+                init_mod = init_order_mod.get(base, None)
+                mem_mod = mem_order_mod.get(base, None)
+
+                yield (0, [int(proc.UniqueProcessId),
+                            str(proc.ImageFileName.cast("string",
+                                                max_length = proc.ImageFileName.vol.count,
+                                                errors = 'replace')),
+                            format_hints.Hex(base),
+                            load_mod != None,
+                            init_mod != None,
+                            mem_mod != None,
+                            mapped_files[base]])
+
+    def run(self):
+        filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None))
+        kernel = self.context.modules[self.config['kernel']]
+
+        return renderers.TreeGrid([("Pid", int),
+                        ("Process", str),
+                        ("Base", format_hints.Hex),
+                        ("InLoad", bool),
+                        ("InInit", bool),
+                        ("InMem", bool),
+                        ("MappedPath", str)],
+                        self._generator(
+                           pslist.PsList.list_processes(context = self.context,
+                                                        layer_name = kernel.layer_name,
+                                                        symbol_table = kernel.symbol_table_name,
+                                                        filter_func = filter_func)))

volatility3/framework/plugins/windows/mftscan.py

@@ -0,0 +1,164 @@
+# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0
+# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
+#
+
+import datetime
+import logging
+
+from volatility3.framework import constants, exceptions, interfaces, renderers
+from volatility3.framework.configuration import requirements
+from volatility3.framework.renderers import conversion, format_hints
+from volatility3.framework.symbols import intermed
+from volatility3.framework.symbols.windows.extensions import mft
+from volatility3.plugins import timeliner, yarascan
+
+vollog = logging.getLogger(__name__)
+
+
+class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface):
+    """Scans for MFT FILE objects present in a particular windows memory image."""
+
+    _required_framework_version = (2, 0, 0)
+
+    @classmethod
+    def get_requirements(cls):
+        return [
+            requirements.TranslationLayerRequirement(name = 'primary',
+                                                     description = 'Memory layer for the kernel',
+                                                     architectures = ["Intel32", "Intel64"]),
+            requirements.VersionRequirement(name = 'yarascanner', component = yarascan.YaraScanner,
+                                            version = (2, 0, 0)),
+        ]
+
+    def _generator(self):
+        layer = self.context.layers[self.config['primary']]
+
+        # Yara Rule to scan for MFT Header Signatures
+        rules = yarascan.YaraScan.process_yara_options({'yara_rules': '/FILE0|FILE\*|BAAD/'})
+
+        # Read in the Symbol File
+        symbol_table = intermed.IntermediateSymbolTable.create(context = self.context,
+                                                               config_path = self.config_path,
+                                                               sub_path = "windows",
+                                                               filename = "mft",
+                                                               class_types = {
+                                                                   'FILE_NAME_ENTRY': mft.MFTFileName,
+                                                                   'MFT_ENTRY': mft.MFTEntry
+                                                               })
+
+        # get each of the individual Field Sets
+        mft_object = symbol_table + constants.BANG + "MFT_ENTRY"
+        attribute_object = symbol_table + constants.BANG + "ATTRIBUTE"
+        header_object = symbol_table + constants.BANG + "ATTR_HEADER"
+        si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY"
+        fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY"
+
+        # Scan the layer for Raw MFT records and parse the fields
+        for offset, _rule_name, _name, _value in layer.scan(context = self.context,
+                                                            scanner = yarascan.YaraScanner(rules = rules)):
+            try:
+                mft_record = self.context.object(mft_object, offset = offset, layer_name = layer.name)
+                # We will update this on each pass in the next loop and use it as the new offset.
+                attr_base_offset = mft_record.FirstAttrOffset
+
+                attr_header = self.context.object(header_object,
+                                                  offset = offset + attr_base_offset,
+                                                  layer_name = layer.name)
+
+                # There is no field that has a count of Attributes
+                # Keep Attempting to read attributes until we get an invalid attr_header.AttrType
+
+                while attr_header.AttrType.is_valid_choice:
+                    vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}")
+
+                    # Offset past the headers to the attribute data
+                    attr_data_offset = offset + attr_base_offset + self.context.symbol_space.get_type(
+                        attribute_object).relative_child_offset("Attr_Data")
+
+                    # MFT Flags determine the file type or dir
+                    # If we don't have a valid enum, coerce to hex so we can keep the record
+                    try:
+                        mft_flag = mft_record.Flags.lookup()
+                    except ValueError:
+                        mft_flag = hex(mft_record.Flags)
+
+                    # Standard Information Attribute
+                    if attr_header.AttrType.lookup() == 'STANDARD_INFORMATION':
+                        attr_data = self.context.object(si_object, offset = attr_data_offset, layer_name = layer.name)
+
+                        yield 0, (
+                            format_hints.Hex(attr_data_offset),
+                            mft_record.get_signature(),
+                            mft_record.RecordNumber,
+                            mft_record.LinkCount,
+                            mft_flag,
+                            renderers.NotApplicableValue(),
+                            attr_header.AttrType.lookup(),
+                            conversion.wintime_to_datetime(attr_data.CreationTime),
+                            conversion.wintime_to_datetime(attr_data.ModifiedTime),
+                            conversion.wintime_to_datetime(attr_data.UpdatedTime),
+                            conversion.wintime_to_datetime(attr_data.AccessedTime),
+                            renderers.NotApplicableValue(),
+                        )
+
+                    # File Name Attribute
+                    if attr_header.AttrType.lookup() == 'FILE_NAME':
+                        attr_data = self.context.object(fn_object, offset = attr_data_offset, layer_name = layer.name)
+                        file_name = attr_data.get_full_name()
+
+                        # If we don't have a valid enum, coerce to hex so we can keep the record
+                        try:
+                            permissions = attr_data.Flags.lookup()
+                        except ValueError:
+                            permissions = hex(attr_data.Flags)
+
+                        yield 1, (format_hints.Hex(attr_data_offset), mft_record.get_signature(),
+                                  mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions,
+                                  attr_header.AttrType.lookup(),
+                                  conversion.wintime_to_datetime(attr_data.CreationTime),
+                                  conversion.wintime_to_datetime(attr_data.ModifiedTime),
+                                  conversion.wintime_to_datetime(attr_data.UpdatedTime),
+                                  conversion.wintime_to_datetime(attr_data.AccessedTime), file_name)
+
+                    # If there's no advancement the loop will never end, so break it now
+                    if attr_header.Length == 0:
+                        break
+
+                    # Update the base offset to point to the next attribute
+                    attr_base_offset += attr_header.Length
+                    # Get the next attribute
+                    attr_header = self.context.object(header_object,
+                                                      offset = offset + attr_base_offset,
+                                                      layer_name = layer.name)
+
+            except exceptions.PagedInvalidAddressException:
+                pass
+
+    def generate_timeline(self):
+        for row in self._generator():
+            _depth, row_data = row
+
+            # Only Output FN Records
+            if row_data[6] == 'FILE_NAME':
+                filename = row_data[-1]
+                description = f"MFT FILE_NAME entry for {filename}"
+                yield (description, timeliner.TimeLinerType.CREATED, row_data[7])
+                yield (description, timeliner.TimeLinerType.MODIFIED, row_data[8])
+                yield (description, timeliner.TimeLinerType.CHANGED, row_data[9])
+                yield (description, timeliner.TimeLinerType.ACCESSED, row_data[10])
+
+    def run(self):
+        return renderers.TreeGrid([
+            ('Offset', format_hints.Hex),
+            ('Record Type', str),
+            ('Record Number', int),
+            ('Link Count', int),
+            ('MFT Type', str),
+            ('Permissions', str),
+            ('Attribute Type', str),
+            ('Created', datetime.datetime),
+            ('Modified', datetime.datetime),
+            ('Updated', datetime.datetime),
+            ('Accessed', datetime.datetime),
+            ('Filename', str),
+        ], self._generator())

volatility3/framework/plugins/windows/sessions.py

@@ -0,0 +1,103 @@
+# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0
+# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
+#
+
+import datetime
+import logging
+
+from volatility3.framework import renderers, interfaces
+from volatility3.framework.configuration import requirements
+from volatility3.framework.objects import utility
+from volatility3.plugins.windows import pslist
+from volatility3.plugins import timeliner
+
+vollog = logging.getLogger(__name__)
+
+
+class Sessions(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface):
+    """lists Processes with Session information extracted from Environmental Variables"""
+
+    _required_framework_version = (2, 0, 0)
+
+    @classmethod
+    def get_requirements(cls):
+        return [
+            requirements.ModuleRequirement(name = 'kernel',
+                                           description = 'Windows kernel',
+                                           architectures = ["Intel32", "Intel64"]),
+            requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)),
+            requirements.ListRequirement(name = 'pid',
+                                         element_type = int,
+                                         description = "Process IDs to include (all other processes are excluded)",
+                                         optional = True)
+        ]
+
+    def _generator(self):
+        kernel = self.context.modules[self.config['kernel']]
+        filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None))
+
+        # Collect all the values as we will want to group them later
+        sessions = {}
+
+        for proc in pslist.PsList.list_processes(self.context,
+                                                 kernel.layer_name,
+                                                 kernel.symbol_table_name,
+                                                 filter_func = filter_func):
+
+            session_id = proc.get_session_id()
+
+            # Detect RDP, Console or set default value
+            session_type = renderers.NotAvailableValue()
+
+            # Construct Username from Process Env
+            user_domain = ''
+            user_name = ''
+
+            for var, val in proc.environment_variables():
+                if var.lower() == 'username':
+                    user_name = val
+                elif var.lower() == 'userdomain':
+                    user_domain = val
+                if var.lower() == 'sessionname':
+                    session_type = val
+
+            # Concat Domain and User
+            full_user = f'{user_domain}/{user_name}'
+            if full_user == '/':
+                full_user = renderers.NotAvailableValue()
+
+            # Collect all the values in to a row we can yield after sorting.
+            row = {
+                "session_id": session_id,
+                "process_id": proc.UniqueProcessId,
+                "process_name": utility.array_to_string(proc.ImageFileName),
+                "user_name": full_user,
+                "process_start": proc.get_create_time(),
+                "session_type": session_type
+            }
+
+            # Add row to correct session so we can sort it later
+            if session_id in sessions:
+                sessions[session_id].append(row)
+            else:
+                sessions[session_id] = [row]
+
+        # Group and yield each row
+        for rows in sessions.values():
+            for row in rows:
+                yield 0, (row.get('session_id'), row.get('session_type'), row.get('process_id'),
+                          row.get('process_name'), row.get('user_name'), row.get('process_start'))
+
+    def generate_timeline(self):
+        for row in self._generator():
+            _depth, row_data = row
+            # Only add to timeline if we have the username
+            # Without the user context PSList output is identical
+            if isinstance(row_data[4], str):
+                description = f"Process: {row_data[2]} {row_data[3]} started by user {row_data[4]}"
+                yield (description, timeliner.TimeLinerType.CREATED, row_data[5])
+
+    def run(self):
+
+        return renderers.TreeGrid([("Session ID", int), ('Session Type', str), ("Process ID", int), ("Process", str),
+                                   ("User Name", str), ("Create Time", datetime.datetime)], self._generator())

volatility3/framework/plugins/yarascan.py

@@ -3,7 +3,7 @@
 #
 
 import logging
-from typing import Iterable, Tuple, List, Dict, Any
+from typing import Any, Dict, Iterable, List, Tuple
 
 from volatility3.framework import interfaces, renderers
 from volatility3.framework.configuration import requirements
@@ -15,8 +15,11 @@ vollog = logging.getLogger(__name__)
 
 try:
     import yara
+
+    if tuple([int(x) for x in yara.__version__.split('.')]) < (3, 8):
+        raise ImportError
 except ImportError:
-    vollog.info("Python Yara module not found, plugin (and dependent plugins) not available")
+    vollog.info("Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available")
     raise
 
 
@@ -40,7 +43,10 @@ class YaraScan(plugins.PluginInterface):
     """Scans kernel memory using yara rules (string or file)."""
 
     _required_framework_version = (2, 0, 0)
-    _version = (1, 0, 0)
+    _version = (1, 1, 0)
+
+    # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string
+    # or something that makes more sense
 
     @classmethod
     def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
@@ -83,6 +89,8 @@ class YaraScan(plugins.PluginInterface):
             if config.get('wide', False):
                 rule += " wide ascii"
             rules = yara.compile(sources = {'n': f'rule r1 {{strings: $a = {rule} condition: $a}}'})
+        elif config.get('yara_source', None) is not None:
+            rules = yara.compile(source = config['yara_source'])
         elif config.get('yara_file', None) is not None:
             rules = yara.compile(file = resources.ResourceAccessor().open(config['yara_file'], "rb"))
         elif config.get('yara_compiled_file', None) is not None:

volatility3/framework/renderers/__init__.py

@@ -272,20 +272,26 @@ class TreeGrid(interfaces.renderers.TreeGrid):
     def _append(self, parent: Optional[interfaces.renderers.TreeNode], values: Any) -> TreeNode:
         """Adds a new node at the top level if parent is None, or under the
         parent node otherwise, after all other children."""
-        children = self.children(parent)
-        return self._insert(parent, len(children), values)
+        return self._insert(parent, None, values)
 
-    def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: int, values: Any) -> TreeNode:
+    def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: Optional[int], values: Any) -> TreeNode:
         """Inserts an element into the tree at a specific position."""
         parent_path = ""
         children = self._find_children(parent)
         if parent is not None:
             parent_path = parent.path + self.path_sep
-        newpath = parent_path + str(position)
+        if position is None:
+            newpath = parent_path + str(len(children))
+        else:
+            newpath = parent_path + str(position)
+            for node, _ in children[position:]:
+                self.visit(node, lambda child, _: child.path_changed(newpath, True), None)
+
         tree_item = TreeNode(newpath, self, parent, values)
-        for node, _ in children[position:]:
-            self.visit(node, lambda child, _: child.path_changed(newpath, True), None)
-        children.insert(position, (tree_item, []))
+        if position is None:
+            children.append((tree_item, []))
+        else:
+            children.insert(position, (tree_item, []))
         return tree_item
 
     def is_ancestor(self, node, descendant):

volatility3/framework/symbols/windows/extensions/__init__.py

@@ -84,7 +84,7 @@ class MMVAD_SHORT(objects.StructType):
 
         if tag in ["VadS", "VadF"]:
             target = "_MMVAD_SHORT"
-        elif tag != None and tag.startswith("Vad"):
+        elif tag is not None and tag.startswith("Vad"):
             target = "_MMVAD"
         elif depth == 0:
             # the root node at depth 0 is allowed to not have a tag
@@ -651,7 +651,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject):
         except AttributeError:
             return False
 
-        return value != 0 and value != None
+        if value:
+            return True
+
+        return False
 
     def get_vad_root(self):
 

volatility3/framework/symbols/windows/extensions/mft.py

@@ -0,0 +1,21 @@
+# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0
+# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
+#
+
+from volatility3.framework import objects
+
+
+class MFTEntry(objects.StructType):
+    """This represents the base MFT Record"""
+
+    def get_signature(self) -> str:
+        signature = self.Signature.cast('string', max_length = 4, encoding = 'latin-1')
+        return signature
+
+
+class MFTFileName(objects.StructType):
+    """This represents an MFT $FILE_NAME Attribute"""
+
+    def get_full_name(self) -> str:
+        output = self.Name.cast("string", encoding = "utf16", max_length = self.NameLength * 2, errors = "replace")
+        return output

volatility3/framework/symbols/windows/extensions/pool.py

@@ -128,8 +128,8 @@ class POOL_HEADER(objects.StructType):
                         #  ---------------
                         if addr - optional_headers_length < 0:
                             continue
-                        padding_length = struct.unpack(
-                            "<I", infomask_data[addr - optional_headers_length:addr - optional_headers_length + 4])[0]
+                        padding_length, = struct.unpack(
+                            "<I", infomask_data[addr - optional_headers_length:addr - optional_headers_length + 4])
                         padding_length -= lengths_of_optional_headers[padding_available or 0]
 
                     # Certain versions of windows have PADDING_INFO lengths that are too long

volatility3/framework/symbols/windows/extensions/registry.py

@@ -264,15 +264,18 @@ class CM_KEY_VALUE(objects.StructType):
         if self_type == RegValueTypes.REG_DWORD:
             if len(data) != struct.calcsize("<L"):
                 raise ValueError(f"Size of data does not match the type of registry value {self.get_name()}")
-            return struct.unpack("<L", data)[0]
+            res, = struct.unpack("<L", data)
+            return res
         if self_type == RegValueTypes.REG_DWORD_BIG_ENDIAN:
             if len(data) != struct.calcsize(">L"):
                 raise ValueError(f"Size of data does not match the type of registry value {self.get_name()}")
-            return struct.unpack(">L", data)[0]
+            res, =  struct.unpack(">L", data)
+            return res
         if self_type == RegValueTypes.REG_QWORD:
             if len(data) != struct.calcsize("<Q"):
                 raise ValueError(f"Size of data does not match the type of registry value {self.get_name()}")
-            return struct.unpack("<Q", data)[0]
+            res, = struct.unpack("<Q", data)
+            return res
         if self_type in [
                 RegValueTypes.REG_SZ, RegValueTypes.REG_EXPAND_SZ, RegValueTypes.REG_LINK, RegValueTypes.REG_MULTI_SZ,
                 RegValueTypes.REG_BINARY, RegValueTypes.REG_FULL_RESOURCE_DESCRIPTOR, RegValueTypes.REG_RESOURCE_LIST,

volatility3/framework/symbols/windows/mft.json

@@ -0,0 +1,467 @@
+{
+    "metadata": {
+        "producer": {
+          "version": "0.0.1",
+          "name": "kevthehermit-by-hand",
+          "comment": "Using structures defined in File System Forensic Analysis pg 353+",
+          "datetime": "2022-01-03T13:37:00"
+        },
+        "format": "6.1.0"
+      },
+    "base_types": {
+      "unsigned long": {
+          "kind": "int", 
+          "size": 4, 
+          "signed": false, 
+          "endian": "little"
+      }, 
+      "unsigned long long": {
+          "kind": "int", 
+          "size": 8, 
+          "signed": false, 
+          "endian": "little"
+      }, 
+      "long": {
+          "kind": "int", 
+          "size": 4, 
+          "signed": true, 
+          "endian": "little"
+      },
+      "unsigned int": {
+          "kind": "int",
+          "size": 4,
+          "signed": false,
+          "endian": "little"
+      }, 
+      "unsigned short": {
+          "kind": "int", 
+          "size": 2, 
+          "signed": false, 
+          "endian": "little"
+      },
+      "unsigned char": {
+          "kind": "int", 
+          "size": 1, 
+          "signed": false, 
+          "endian": "little"
+      },
+      "wchar": {
+          "kind": "int", 
+          "size": 2, 
+          "signed": true, 
+          "endian": "little"
+      }
+    },
+    "symbols": {},
+    "enums": {
+        "AttrTypeEnum": {
+            "base": "unsigned char",
+            "constants": {
+              "STANDARD_INFORMATION": 16,
+              "ATTRIBUTE_LIST": 32,
+              "FILE_NAME": 48,
+              "OBJECT_ID": 64,
+              "SECURITY_DESCRIPTOR": 80,
+              "VOLUME_NAME": 96,
+              "VOLUME_INFORMATION": 112,
+              "DATA": 128,
+              "INDEX_ROOT": 114,
+              "INDEX_ALLOCATION": 160,
+              "BITMAP": 176,
+              "REPARSE_POINT": 192,
+              "EA_INFORMATION": 208,
+              "EA": 224,
+              "PROPERTY_SET": 240,
+              "LOGGED_UTILITY_STREAM": 256
+            },
+            "size": 1
+          },
+        "NameSpaceEnum": {
+            "base":"unsigned char",
+            "constants": {
+                "POSIX": 0,
+                "Win32": 1,
+                "DOS": 2,
+                "Win32 DOS": 3
+            },
+            "size": 1
+        },
+        "MFTFlagsEnum": {
+            "base":"unsigned char",
+            "constants": {
+                "Removed": 0,
+                "File": 1,
+                "Directory": 2,
+                "DirInUse": 3
+            },
+            "size": 1
+        },
+        "PermissionFlagEnum": {
+            "base":"unsigned char",
+            "constants": {
+                "ReadOnly": 1,
+                "Hidden": 2,
+                "System": 4,
+                "Archive": 32,
+                "ArchiveHidden": 34,
+                "ArchiveSystem": 36,
+                "ArchiveHiddenSystem": 38,
+                "Device": 60,
+                "Normal": 128,
+                "Temporary": 256,
+                "TempArchive": 288,
+                "SparseFile": 512,
+                "ReparsePoint": 1024,
+                "Compressed": 2048,
+                "Offline": 4096,
+                "NotIndexed": 8192,
+                "Encrypted": 16384,
+                "Directory": 268435456,
+                "IndexView": 536870912
+            },
+            "size": 1
+        }
+    },
+    "user_types": {
+        "MFT_ENTRY": {
+            "fields": {
+                "Signature": {
+                    "offset": 0,
+                    "type": {
+                        "count": 1,
+                        "kind": "array",
+                        "subtype": {
+                          "kind": "base",
+                          "name": "unsigned char"
+                        }
+                      }
+                },
+                "UpdateSequenceOffset": {
+                    "offset": 4,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned short"
+                    }
+                },
+                "NumFixupEntries": {
+                    "offset": 6,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned short"
+                    }
+                },
+                "LSN": {
+                    "offset": 8,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                    }
+                },
+                "SequenceValue": {
+                    "offset": 16,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned short"
+                    }
+                },
+                "LinkCount": {
+                    "offset": 18,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned short"
+                    }
+                },
+                "FirstAttrOffset": {
+                    "offset": 20,
+                    "type":{
+                        "kind": "base",
+                        "name": "unsigned short"
+                    }
+                },
+                "Flags": {
+                    "offset": 22,
+                    "type":{
+                        "kind": "enum",
+                        "name": "MFTFlagsEnum"
+                    }
+                },
+                "RealSize": {
+                    "offset": 24,
+                    "type":{
+                        "kind": "base",
+                        "name": "unsigned int"
+                    }
+                },
+                "AlocatedSize": {
+                    "offset": 28,
+                    "type":{
+                        "kind": "base",
+                        "name": "unsigned int"
+                    }
+                },
+                "BaseReference": {
+                    "offset": 32,
+                    "type":{
+                        "kind": "base",
+                        "name": "unsigned long long"
+                    }
+                },
+                "NextAttrID": {
+                    "offset": 40,
+                    "type":{
+                        "kind": "base",
+                        "name": "unsigned short"
+                    }
+                },
+                "RecordNumber": {
+                    "offset": 44,
+                    "type":{
+                        "kind": "base",
+                        "name": "unsigned long"
+                    }
+                }
+            },
+            "kind": "struct",
+            "size": 1024
+        },
+        "ATTRIBUTE": {
+            "fields":{
+                "Attr_Header": {
+                    "offset": 0,
+                    "type": {
+                        "kind": "struct",
+                        "name": "mft!ATTR_HEADER"
+                    }
+                },
+                "Resident_Header": {
+                    "offset": 16,
+                    "type": {
+                        "kind": "struct",
+                        "name": "mft!RESIDENT_HEADER"
+                    }
+                },
+                "Attr_Data": {
+                    "offset": 24,
+                    "type": {
+                        "kind": "struct",
+                        "name": "mft!ATTR_HEADER"
+                    }
+                }
+            },
+            "kind": "struct",
+            "size": 96
+        },
+        "ATTR_HEADER": {
+            "fields": {
+                "AttrType": {
+                    "offset": 0,
+                    "type": {
+                        "kind": "enum",
+                        "name": "AttrTypeEnum"
+                      }
+                },"Length": {
+                    "offset": 4,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned int"
+                      }
+                },
+                "NonResidentFlag": {
+                    "offset": 8,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned char"                      }
+                },
+                "NameLength": {
+                    "offset": 9,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned char"  
+                      }
+                },
+                "NameOffset": {
+                    "offset": 10,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned short"
+                      }
+                },
+                "Flags": {
+                    "offset": 12,
+                    "type": {
+                        "kind": "enum",
+                        "name": "MFTFlagsEnum"
+                    }
+                },
+                "AttributeID": {
+                    "offset": 14,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned short"
+                      }
+                }
+            },
+            "kind": "struct",
+            "size": 16
+        },"RESIDENT_HEADER": {
+            "fields": {
+                "AttrSize": {
+                    "offset": 0,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned int"
+                      }
+                },"AttrOffset": {
+                    "offset": 4,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned int"
+                      }
+                },
+                "IndexFlag": {
+                    "offset": 8,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned short"                      }
+                }
+            },
+            "kind": "struct",
+            "size": 8
+        },
+        "STANDARD_INFORMATION_ENTRY": {
+            "fields": {
+                "CreationTime": {
+                    "offset": 0,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                      }
+                },
+                "ModifiedTime": {
+                    "offset": 8,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                      }
+                },
+                "UpdatedTime": {
+                    "offset": 16,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                      }
+                },
+                "AccessedTime": {
+                    "offset": 24,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                      }
+                },
+                "flags": {
+                    "offset": 32,
+                    "type": {
+                        "kind": "enum",
+                        "name": "PermissionFlagEnum"
+                      }
+                }
+            },
+            "kind": "struct",
+            "size": 1024
+        },
+        "FILE_NAME_ENTRY": {
+            "fields": {
+                "ParentDirectory": {
+                    "offset": 0,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                      }
+                },
+                "CreationTime": {
+                    "offset": 8,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                      }
+                },
+                "ModifiedTime": {
+                    "offset": 16,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                      }
+                },
+                "UpdatedTime": {
+                    "offset": 24,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                      }
+                },
+                "AccessedTime": {
+                    "offset": 32,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                      }
+                },
+                "AllocatedFileSize": {
+                    "offset": 40,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                      }
+                },
+                "RealFileSize": {
+                    "offset": 48,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned long long"
+                      }
+                },
+                "Flags": {
+                    "offset": 56,
+                    "type": {
+                        "kind": "enum",
+                        "name": "PermissionFlagEnum"
+                      }
+                },
+                "ReparseValue": {
+                    "offset": 60,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned int"
+                      }
+                },
+                "NameLength": {
+                    "offset": 64,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned char"
+                      }
+                },
+                "NameSpace": {
+                    "offset": 65,
+                    "type": {
+                        "kind": "base",
+                        "name": "unsigned char"
+                      }
+                },
+                "Name": {
+                    "offset": 66,
+                    "type": {
+                      "count": 10,
+                      "kind": "array",
+                      "subtype": {
+                        "kind": "base",
+                        "name": "wchar"
+                      }
+                    }
+                  }
+            },
+            "kind": "struct",
+            "size": 1024
+        }
+    }
+}

volatility3/framework/symbols/windows/pdbutil.py

@@ -131,9 +131,15 @@ class PDBUtility(interfaces.configuration.VersionableInterface):
         # Check it is actually the MZ header
         if mz_sig != b"MZ":
             return None
-
-        nt_header_start = ord(layer.read(offset + 0x3C, 1))
-        optional_header_size = struct.unpack('<H', layer.read(offset + nt_header_start + 0x14, 2))[0]
+        
+        nt_header_start, = struct.unpack("<I", layer.read(offset + 0x3C, 4))
+        pe_sig = layer.read(offset + nt_header_start, 2)
+        
+        # Check it is actually the Nt Headers
+        if pe_sig != b"PE":
+            return None
+        
+        optional_header_size, = struct.unpack('<H', layer.read(offset + nt_header_start + 0x14, 2))
         # Just enough to tell us the max size
         pe_header = layer.read(offset, nt_header_start + 0x16 + optional_header_size)
         pe_data = pefile.PE(data = pe_header)
@@ -357,4 +363,4 @@ class PdbSignatureScanner(interfaces.layers.ScannerInterface):
 
                 guid = (16 * '{:02X}').format(g0, g1, g2, g3, g4, g5, g6, g7, g8, g9, ga, gb, gc, gd, ge, gf)
                 if match.start(0) < self.chunk_size:
-                    yield (guid, a, pdb_name, match.start(0))
+                    yield (guid, a, pdb_name, data_offset + match.start(0))