From 48e29ae530ecfb65fa3311a7547c16ea77f7f5cd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 May 2022 23:44:29 +0100 Subject: [PATCH 001/979] CLI: Add support for a configuration options file --- volatility3/cli/__init__.py | 58 ++++++++++++++++++++++------ volatility3/cli/volshell/__init__.py | 51 +++++++++++++++++------- 2 files changed, 83 insertions(+), 26 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 8851e2b18..62be115d3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,7 +19,7 @@ import os import sys import tempfile import traceback -from typing import Any, Dict, Type, Union +from typing import Any, Dict, List, Tuple, Type, Union from urllib import parse, request import volatility3.plugins @@ -92,6 +92,9 @@ class CommandLine: renderers = dict([(x.name.lower(), x) for x in framework.class_subclasses(text_renderer.CLIRenderer)]) + # Load up system defaults + delayed_logs, default_config = self.load_system_defaults('vol.json') + parser = volargparse.HelpfulArgParser(add_help = False, prog = self.CLI_NAME, description = "An open-source memory forensics framework") @@ -174,6 +177,8 @@ class CommandLine: default = False, action = 'store_true') + parser.set_defaults(**default_config) + # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h'] @@ -184,17 +189,7 @@ class CommandLine: banner_output = sys.stderr banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") - if partial_args.plugin_dirs: - volatility3.plugins.__path__ = [os.path.abspath(p) - for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH - - if partial_args.symbol_dirs: - volatility3.symbols.__path__ = [os.path.abspath(p) - for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS - - if partial_args.cache_path: - constants.CACHE_PATH = partial_args.cache_path - + ### Start up logging if partial_args.log: file_logger = logging.FileHandler(partial_args.log) file_logger.setLevel(1) @@ -210,6 +205,21 @@ class CommandLine: else: console.setLevel(10 - (partial_args.verbosity - 2)) + for level, msg in delayed_logs: + vollog.log(level, msg) + + ### Alter constants if necessary + if partial_args.plugin_dirs: + volatility3.plugins.__path__ = [os.path.abspath(p) + for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH + + if partial_args.symbol_dirs: + volatility3.symbols.__path__ = [os.path.abspath(p) + for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS + + if partial_args.cache_path: + constants.CACHE_PATH = partial_args.cache_path + vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") @@ -366,6 +376,30 @@ class CommandLine: raise ValueError(f"File does not exist: {filename}") return parse.urlunparse(single_location) + def load_system_defaults(self, filename: str) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: + """Modify the main configuration based on the default configuration override""" + # Build the config path + default_config_path = os.path.join(os.path.expanduser("~"), ".config", "volatility3", filename) + if sys.platform == 'win32': + default_config_path = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3", + filename) + + delayed_logs = [] + + # Process it if the files exist + if os.path.exists(default_config_path): + result = json.load(open(default_config_path, 'rb')) + if not isinstance(result, dict): + delayed_logs.append((logging.INFO, + f'Default configuration file {default_config_path} does not contain a dictionary')) + else: + delayed_logs.append( + (logging.INFO, f"Loading default configuration options from {default_config_path}")) + delayed_logs.append((logging.DEBUG, + f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}")) + return delayed_logs, result + return delayed_logs, {} + def process_exceptions(self, excp): """Provide useful feedback if an exception occurs during a run of a plugin.""" # Ensure there's nothing in the cache diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 769e958fd..647be8a1a 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -45,6 +45,9 @@ class VolShell(cli.CommandLine): framework.require_interface_version(2, 0, 0) + # Load up system defaults + delayed_logs, default_config = self.load_system_defaults('volshell.json') + parser = argparse.ArgumentParser(prog = self.CLI_NAME, description = "A tool for interactivate forensic analysis of memory images") parser.add_argument("-c", @@ -68,13 +71,16 @@ class VolShell(cli.CommandLine): default = "", type = str) parser.add_argument("-v", "--verbosity", help = "Increase output verbosity", default = 0, action = "count") + parser.add_argument("--log", + help = "Log output to a file as well as the console", + default = None, + type = str) parser.add_argument("-o", "--output-dir", help = "Directory in which to output any generated files", default = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')), type = str) parser.add_argument("-q", "--quiet", help = "Remove progress feedback", default = False, action = 'store_true') - parser.add_argument("--log", help = "Log output to a file as well as the console", default = None, type = str) parser.add_argument("-f", "--file", metavar = 'FILE', @@ -97,6 +103,10 @@ class VolShell(cli.CommandLine): help = f"Change the default path ({constants.CACHE_PATH}) used to store the cache", default = constants.CACHE_PATH, type = str) + parser.add_argument("--offline", + help = "Do not search online for additional JSON files", + default = False, + action = 'store_true') # Volshell specific flags os_specific = parser.add_mutually_exclusive_group(required = False) @@ -108,10 +118,33 @@ class VolShell(cli.CommandLine): os_specific.add_argument("-l", "--linux", default = False, action = "store_true", help = "Run a Linux volshell") os_specific.add_argument("-m", "--mac", default = False, action = "store_true", help = "Run a Mac volshell") + parser.set_defaults(**default_config) + # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h'] partial_args, _ = parser.parse_known_args(known_args) + + ### Start up logging + if partial_args.log: + file_logger = logging.FileHandler(partial_args.log) + file_logger.setLevel(1) + file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S', + fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + file_logger.setFormatter(file_formatter) + rootlog.addHandler(file_logger) + vollog.info("Logging started") + if partial_args.verbosity < 3: + if partial_args.verbosity < 1: + sys.tracebacklimit = None + console.setLevel(30 - (partial_args.verbosity * 10)) + else: + console.setLevel(10 - (partial_args.verbosity - 2)) + + for level, msg in delayed_logs: + vollog.log(level, msg) + + ### Alter constants if necessary if partial_args.plugin_dirs: volatility3.plugins.__path__ = [os.path.abspath(p) for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH @@ -126,23 +159,13 @@ class VolShell(cli.CommandLine): vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") - if partial_args.log: - file_logger = logging.FileHandler(partial_args.log) - file_logger.setLevel(0) - file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S', - fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s') - file_logger.setFormatter(file_formatter) - vollog.addHandler(file_logger) - vollog.info("Logging started") - - if partial_args.verbosity < 3: - console.setLevel(30 - (partial_args.verbosity * 10)) - else: - console.setLevel(10 - (partial_args.verbosity - 2)) if partial_args.clear_cache: framework.clear_cache() + if partial_args.offline: + constants.OFFLINE = partial_args.offline + # Do the initialization ctx = contexts.Context() # Construct a blank context failures = framework.import_files(volatility3.plugins, From 9ea1a960cdece1c6897735adc58f592f73c44e86 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 30 May 2022 01:34:19 +0100 Subject: [PATCH 002/979] CLI: Synchonize volshell code a little better courtesy of @digitalisx --- volatility3/cli/volshell/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 647be8a1a..e2e7bc394 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -15,12 +15,14 @@ from volatility3.cli.volshell import generic, linux, mac, windows from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins # Make sure we log everything + +rootlog = logging.getLogger() vollog = logging.getLogger() vollog.setLevel(0) -# Trim the console down by default console = logging.StreamHandler() console.setLevel(logging.WARNING) formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') +# Trim the console down by default console.setFormatter(formatter) vollog.addHandler(console) From 2e4c257e15e83b33fb40cd9369c90dac2c1786e9 Mon Sep 17 00:00:00 2001 From: ikelos Date: Mon, 30 May 2022 01:53:07 +0100 Subject: [PATCH 003/979] Update volatility3/cli/__init__.py Co-authored-by: Donghyun Kim --- volatility3/cli/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 62be115d3..6881311bf 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -388,7 +388,8 @@ class CommandLine: # Process it if the files exist if os.path.exists(default_config_path): - result = json.load(open(default_config_path, 'rb')) + with open(default_config_path, 'rb') as config_json: + result = json.load(config_json) if not isinstance(result, dict): delayed_logs.append((logging.INFO, f'Default configuration file {default_config_path} does not contain a dictionary')) From 0b33a79dc6465884105a60cb3d2054c61885a2eb Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Wed, 2 Nov 2022 21:10:51 +0000 Subject: [PATCH 004/979] Report IRP entries that point inside a hidden module. This is a common rootkit technique. --- volatility3/framework/plugins/windows/driverirp.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index 7f9bc6b08..4013bdb8f 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -48,7 +48,10 @@ class DriverIrp(interfaces.plugins.PluginInterface): for i, address in enumerate(driver.MajorFunction): module_symbols = collection.get_module_symbols_by_absolute_location(address) + module_found = False + for module_name, symbol_generator in module_symbols: + module_found = True symbols_found = False for symbol in symbol_generator: @@ -60,6 +63,11 @@ class DriverIrp(interfaces.plugins.PluginInterface): yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], format_hints.Hex(address), module_name, renderers.NotAvailableValue())) + if not module_found: + yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], + format_hints.Hex(address), renderers.NotAvailableValue(), renderers.NotAvailableValue())) + + def run(self): return renderers.TreeGrid([ From 85e86d45c547654afe7c2dca86f6c6d200cb05df Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 8 Mar 2023 20:42:24 +0000 Subject: [PATCH 005/979] Linux: fix black lint issues --- volatility3/framework/plugins/linux/envars.py | 1 - volatility3/framework/plugins/linux/iomem.py | 1 - 2 files changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 028eb2a57..5cbf0f502 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -70,7 +70,6 @@ class Envars(plugins.PluginInterface): # if mm exists attempt to get envars if mm: - # get process layer to read envars from proc_layer_name = task.add_process_layer() if proc_layer_name is None: diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index fddea4668..8efbf3b57 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -128,7 +128,6 @@ class IOMem(interfaces.plugins.PluginInterface): # only continue if iomem_root address was located if iomem_root_offset is not None: - # recursively parse the resources starting from the root resource at 'iomem_resource' for depth, (name, start, end) in self.parse_resource( self.context, vmlinux_module_name, iomem_root_offset From cfec8e4b1214329726855ffc89541f9eec29f3ec Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 15:18:58 +0000 Subject: [PATCH 006/979] Core: Pointer is_readable should check the native layer --- volatility3/framework/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index a04eedd87..3b1745718 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -442,7 +442,7 @@ class Pointer(Integer): def is_readable(self, layer_name: Optional[str] = None) -> bool: """Determines whether the address of this pointer can be read from memory.""" - layer_name = layer_name or self.vol.layer_name + layer_name = layer_name or self.vol.native_layer_name return self._context.layers[layer_name].is_valid(self, self.vol.subtype.size) def __getattr__(self, attr: str) -> Any: From b5ef7248ab1ef5ab5cab9fc5b6d05a88ef26c7bf Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 11 Mar 2023 15:17:24 +0000 Subject: [PATCH 007/979] Linux: Fix psscan task native_layer --- volatility3/framework/plugins/linux/psscan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index f4bfd347e..7b03d8c87 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -135,6 +135,7 @@ class PsScan(interfaces.plugins.PluginInterface): raise exceptions.LayerException( kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" ) + memory_layer_name = kernel_layer.dependencies[0] memory_layer = context.layers[kernel_layer.dependencies[0]] # scan the memory_layer for these needles @@ -145,7 +146,8 @@ class PsScan(interfaces.plugins.PluginInterface): ptask = context.object( vmlinux.symbol_table_name + constants.BANG + "task_struct", offset=address - sched_class_offset, - layer_name="memory_layer", + layer_name=memory_layer_name, + native_layer_name=kernel_layer_name, ) # sanity check exit_state From 1b5a6b626733a289ae088e92a292d259c3e1931f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Apr 2023 19:57:15 +0100 Subject: [PATCH 008/979] Core: Bump the copyright year of the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 502e26f10..471735af8 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 17 May 2023 21:41:37 +0300 Subject: [PATCH 009/979] added 'PoolConstraint' of Thread objects to 'PoolScanner.default_constraints' as part of adding support for thread pool tag scanning --- .../framework/plugins/windows/poolscanner.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index e131c5f78..028241bb8 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -224,6 +224,20 @@ class PoolScanner(plugins.PluginInterface): size=(600, None), page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), + # threads on windows before windows8 + PoolConstraint(b'Thr\xe5', # -> “protected” allocation, MSB is set. + type_name = symbol_table + constants.BANG + "_ETHREAD", + object_type="Thread", + size = (600, None), # -> 0x0258 - size of strcut in win5.1 + page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + ), + # threads on windows starting with windows8 + PoolConstraint(b'Thre', + type_name = symbol_table + constants.BANG + "_ETHREAD", + object_type="Thread", + size = (600, None), # -> 0x0258 - size of strcut in win5.1 + page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + ), # files on windows before windows 8 PoolConstraint( b"Fil\xe5", From 550d1096ebe2c2d1f22a6c6446495051d54e84ec Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sat, 20 May 2023 14:20:15 +0300 Subject: [PATCH 010/979] temporary fix to ETHREAD class, add 'is_valid' method --- .../framework/symbols/windows/extensions/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ba00a4053..cfaa3aced 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -492,9 +492,13 @@ class KMUTANT(objects.StructType, pool.ExecutiveObject): return header.NameInfo.Name.String # type: ignore -class ETHREAD(objects.StructType): +class ETHREAD(objects.StructType, pool.ExecutiveObject): """A class for executive thread objects.""" + def is_valid(self) -> bool: + """Determine if the object is valid.""" + return True # temporary, need to implement validation later. + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 0fd475e10bb9644c28abef36c8f4c7dabec318d6 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sat, 20 May 2023 15:49:09 +0300 Subject: [PATCH 011/979] added permanent implementation for 'ETHREAD.is_valid' --- .../symbols/windows/extensions/__init__.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index cfaa3aced..e17128897 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -497,7 +497,30 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): def is_valid(self) -> bool: """Determine if the object is valid.""" - return True # temporary, need to implement validation later. + + try: + + # validation by thread creation time: + ctime = self.get_create_time() + if not isinstance(ctime, datetime.datetime): + return False + + # validation by parent process: + own_proc = self.owning_process() + # return own_proc.is_valid() + if own_proc.UniqueProcessId % 4 != 0: # NT pids are divisible by 4 + return False + + # passed all valitations + return True + except: + return False + + def get_create_time(self): + return conversion.wintime_to_datetime(self.CreateTime.QuadPart) + + def get_exit_time(self): + return conversion.wintime_to_datetime(self.ExitTime.QuadPart) def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 601870829e33c32688fbecccede7423f1e3b7839 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:12:46 +0300 Subject: [PATCH 012/979] twicked thread constraint for edge cases --- volatility3/framework/plugins/windows/poolscanner.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 028241bb8..ce1015789 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -222,17 +222,21 @@ class PoolScanner(plugins.PluginInterface): type_name=symbol_table + constants.BANG + "_EPROCESS", object_type="Process", size=(600, None), + skip_type_test = True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows before windows8 - PoolConstraint(b'Thr\xe5', # -> “protected” allocation, MSB is set. + PoolConstraint( + b'Thr\xe5', # -> “protected” allocation, MSB is set. type_name = symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", size = (600, None), # -> 0x0258 - size of strcut in win5.1 + skip_type_test = True, page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE ), # threads on windows starting with windows8 - PoolConstraint(b'Thre', + PoolConstraint( + b'Thre', type_name = symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", size = (600, None), # -> 0x0258 - size of strcut in win5.1 From 046c8e4d1e7f3bf9d6857de5601ebd4ab089af6d Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:18:02 +0300 Subject: [PATCH 013/979] fix of is_valid - removed reliace on owning _eprocess and added exclusion for system process (does not have creation time) --- .../symbols/windows/extensions/__init__.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index e17128897..fe4884dbc 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -499,18 +499,21 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): """Determine if the object is valid.""" try: - - # validation by thread creation time: - ctime = self.get_create_time() - if not isinstance(ctime, datetime.datetime): - return False - - # validation by parent process: - own_proc = self.owning_process() - # return own_proc.is_valid() - if own_proc.UniqueProcessId % 4 != 0: # NT pids are divisible by 4 + + # validation by TID: + if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 return False + # validation by PID of parent process: + if self.Cid.UniqueProcess % 4 != 0: + return False + + # validation by thread creation time: + if self.Cid.UniqueProcess != 4: # The System process (PID 4) has no create time + ctime = self.get_create_time() + if not isinstance(ctime, datetime.datetime): + return False + # passed all valitations return True except: From fab818b48d950a1210c77e97a4da09cf24f2bd03 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:24:01 +0300 Subject: [PATCH 014/979] added thrdscan plugin to utilize added support for ethread pool tag scanning --- .../framework/plugins/windows/thrdscan.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 volatility3/framework/plugins/windows/thrdscan.py diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py new file mode 100644 index 000000000..b0b80fe46 --- /dev/null +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -0,0 +1,107 @@ +## +## plugin for testing addition of threads scan support to poolscanner.py +## +import logging +import datetime +from typing import Iterable + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import poolscanner + +vollog = logging.getLogger(__name__) + + +class ThrdScan(interfaces.plugins.PluginInterface): + """Scans for windows threads.""" + + # cuz installed Framework interface version 2 + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) + ), + ] + + @classmethod + def scan_threads( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Scans for threads using the poolscanner module and constraints. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + + Returns: + A list of _ETHREAD objects found by scanning memory for the "Thre" / "Thr\\xE5" pool signatures + """ + + constraints = poolscanner.PoolScanner.builtin_constraints( + symbol_table, [b"Thr\xe5", b"Thre"] + ) + + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, symbol_table, constraints + ): + _constraint, mem_object, _header = result + yield mem_object + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + for ethread in self.scan_threads( + self.context, kernel.layer_name, kernel.symbol_table_name + ): + try: + thread_offset = ethread.vol.offset + owner_proc_pid = ethread.Cid.UniqueProcess + thread_tid = ethread.Cid.UniqueThread + thread_start_addr = ethread.StartAddress + thread_create_time = ethread.get_create_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_exit_time = ethread.get_exit_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + except (ValueError, exceptions.InvalidAddressException): + vollog.debug( + "Thread :{}, invalid address {} in layer {}".format( + thread_tid, thread_start_addr, kernel.layer_name + ) + ) + continue + + yield ( + 0, + ( + hex(format_hints.Hex(thread_offset)), + owner_proc_pid, + thread_tid, + hex(thread_start_addr), + str(thread_create_time) if isinstance(thread_create_time, datetime.datetime) else "", + str(thread_exit_time) if isinstance(thread_exit_time, datetime.datetime) else "" + ) + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", str), + ("PID", int), + ("TID", int), + ("Start Address", str), + ("Create Time", str), + ("Exit Time", str), + ], + self._generator(), + ) \ No newline at end of file From 5749b3ef5283fc36d064a766e5228dc2eef5b402 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 19:22:22 +0300 Subject: [PATCH 015/979] added test for thrdscan plugin --- test/test_volatility.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index aaad615bc..5e41899da 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -189,6 +189,16 @@ def test_windows_svcscan(image, volatility, python): assert rc == 0 +def test_windows_thrdscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.thrdscan.ThrdScan", image, volatility, python) + # find pid 4 (of system process) which starts with lowest tids + assert out.find(b"\t4\t8") != -1 + assert out.find(b"\t4\t12") != -1 + assert out.find(b"\t4\t16") != -1 + #assert out.find(b"this raieses AssertionError") != -1 + assert rc == 0 + + def test_windows_privileges(image, volatility, python): rc, out, err = runvol_plugin( "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"] From 950a76d1e0dfb460215c3b6e5dc1108a0421863f Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 11:17:23 +0300 Subject: [PATCH 016/979] fixed typo --- .../framework/symbols/windows/extensions/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index fe4884dbc..4ad74f61a 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -514,10 +514,11 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False - # passed all valitations - return True - except: + except exceptions.InvalidAddressException: return False + + # passed all validations + return True def get_create_time(self): return conversion.wintime_to_datetime(self.CreateTime.QuadPart) From bef5149ba64e640c39e83774e0fa969893d5ee18 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 11:39:38 +0300 Subject: [PATCH 017/979] changed TreeGrid yielded types to specific simpletypes instead of str --- .../framework/plugins/windows/thrdscan.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index b0b80fe46..8b445991f 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -84,24 +84,24 @@ class ThrdScan(interfaces.plugins.PluginInterface): yield ( 0, ( - hex(format_hints.Hex(thread_offset)), + format_hints.Hex(thread_offset), owner_proc_pid, thread_tid, - hex(thread_start_addr), - str(thread_create_time) if isinstance(thread_create_time, datetime.datetime) else "", - str(thread_exit_time) if isinstance(thread_exit_time, datetime.datetime) else "" + format_hints.Hex(thread_start_addr), + thread_create_time, + thread_exit_time, ) ) def run(self): return renderers.TreeGrid( [ - ("Offset", str), + ("Offset", format_hints.Hex), ("PID", int), ("TID", int), - ("Start Address", str), - ("Create Time", str), - ("Exit Time", str), + ("Start Address", format_hints.Hex), + ("Create Time", datetime.datetime), + ("Exit Time", datetime.datetime), ], self._generator(), ) \ No newline at end of file From c6c501ecb57e1757a510d019a3752af78dda96cc Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:31:25 +0300 Subject: [PATCH 018/979] implemented generate_timeline method in ThrdScan --- .../framework/plugins/windows/thrdscan.py | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 8b445991f..ca3f4fc69 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -9,15 +9,16 @@ from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import poolscanner +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class ThrdScan(interfaces.plugins.PluginInterface): +class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -93,15 +94,40 @@ class ThrdScan(interfaces.plugins.PluginInterface): ) ) + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + row_dict = {} + ( + row_dict["Offset"], + row_dict["PID"], + row_dict["TID"], + row_dict["StartAddress"], + row_dict["CreateTime"], + row_dict["ExitTime"], + ) = row_data + + # Skip threads with no creation time + # - mainly system process threads + if not isinstance(row_dict["CreateTime"], datetime.datetime): + continue + description = (f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})") + + # yield created time, and if there is exit time, yield it too. + yield (description, timeliner.TimeLinerType.CREATED, row_dict["CreateTime"]) + if isinstance(row_dict["ExitTime"], datetime.datetime): + yield (description, timeliner.TimeLinerType.MODIFIED, row_dict["ExitTime"]) + + def run(self): return renderers.TreeGrid( [ ("Offset", format_hints.Hex), ("PID", int), ("TID", int), - ("Start Address", format_hints.Hex), - ("Create Time", datetime.datetime), - ("Exit Time", datetime.datetime), + ("StartAddress", format_hints.Hex), + ("CreateTime", datetime.datetime), + ("ExitTime", datetime.datetime), ], self._generator(), ) \ No newline at end of file From 9bdd249aadb5b60ca5e45c91c396f1c591f9b45e Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:39:12 +0300 Subject: [PATCH 019/979] added _version to ThrdScan --- volatility3/framework/plugins/windows/thrdscan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index ca3f4fc69..cbbe64988 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -19,6 +19,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) # cuz installed Framework interface version 2 _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls): From 5072728e897c1489e11f3422ea6974cf542d7e64 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:50:22 +0300 Subject: [PATCH 020/979] formated with black --- .../framework/plugins/windows/thrdscan.py | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index cbbe64988..813883c2d 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -18,7 +18,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """Scans for windows threads.""" # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod @@ -33,7 +33,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) ), ] - + @classmethod def scan_threads( cls, @@ -53,7 +53,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """ constraints = poolscanner.PoolScanner.builtin_constraints( - symbol_table, [b"Thr\xe5", b"Thre"] + symbol_table, [b"Thr\xe5", b"Thre"] ) for result in poolscanner.PoolScanner.generate_pool_scan( @@ -69,12 +69,16 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) self.context, kernel.layer_name, kernel.symbol_table_name ): try: - thread_offset = ethread.vol.offset - owner_proc_pid = ethread.Cid.UniqueProcess - thread_tid = ethread.Cid.UniqueThread - thread_start_addr = ethread.StartAddress - thread_create_time = ethread.get_create_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object - thread_exit_time = ethread.get_exit_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_offset = ethread.vol.offset + owner_proc_pid = ethread.Cid.UniqueProcess + thread_tid = ethread.Cid.UniqueThread + thread_start_addr = ethread.StartAddress + thread_create_time = ( + ethread.get_create_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_exit_time = ( + ethread.get_exit_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object except (ValueError, exceptions.InvalidAddressException): vollog.debug( "Thread :{}, invalid address {} in layer {}".format( @@ -86,13 +90,13 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) yield ( 0, ( - format_hints.Hex(thread_offset), - owner_proc_pid, - thread_tid, - format_hints.Hex(thread_start_addr), - thread_create_time, - thread_exit_time, - ) + format_hints.Hex(thread_offset), + owner_proc_pid, + thread_tid, + format_hints.Hex(thread_start_addr), + thread_create_time, + thread_exit_time, + ), ) def generate_timeline(self): @@ -112,13 +116,16 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) # - mainly system process threads if not isinstance(row_dict["CreateTime"], datetime.datetime): continue - description = (f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})") - + description = f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})" + # yield created time, and if there is exit time, yield it too. yield (description, timeliner.TimeLinerType.CREATED, row_dict["CreateTime"]) if isinstance(row_dict["ExitTime"], datetime.datetime): - yield (description, timeliner.TimeLinerType.MODIFIED, row_dict["ExitTime"]) - + yield ( + description, + timeliner.TimeLinerType.MODIFIED, + row_dict["ExitTime"], + ) def run(self): return renderers.TreeGrid( @@ -131,4 +138,4 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ("ExitTime", datetime.datetime), ], self._generator(), - ) \ No newline at end of file + ) From 5d32ca542c8918224f070163f02ff454f0bffffa Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 28 May 2023 20:52:26 +0300 Subject: [PATCH 021/979] fix black formatting --- .../framework/plugins/windows/poolscanner.py | 20 +++++++++---------- .../symbols/windows/extensions/__init__.py | 17 ++++++++-------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index ce1015789..5539e1e84 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -222,25 +222,25 @@ class PoolScanner(plugins.PluginInterface): type_name=symbol_table + constants.BANG + "_EPROCESS", object_type="Process", size=(600, None), - skip_type_test = True, + skip_type_test=True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows before windows8 PoolConstraint( - b'Thr\xe5', # -> “protected” allocation, MSB is set. - type_name = symbol_table + constants.BANG + "_ETHREAD", + b"Thr\xe5", # -> “protected” allocation, MSB is set. + type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size = (600, None), # -> 0x0258 - size of strcut in win5.1 - skip_type_test = True, - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + size=(600, None), # -> 0x0258 - size of strcut in win5.1 + skip_type_test=True, + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows starting with windows8 PoolConstraint( - b'Thre', - type_name = symbol_table + constants.BANG + "_ETHREAD", + b"Thre", + type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size = (600, None), # -> 0x0258 - size of strcut in win5.1 - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + size=(600, None), # -> 0x0258 - size of strcut in win5.1 + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # files on windows before windows 8 PoolConstraint( diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 4ad74f61a..8790f41a7 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -499,17 +499,18 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): """Determine if the object is valid.""" try: - # validation by TID: - if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 + if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 return False - + # validation by PID of parent process: if self.Cid.UniqueProcess % 4 != 0: return False - + # validation by thread creation time: - if self.Cid.UniqueProcess != 4: # The System process (PID 4) has no create time + if ( + self.Cid.UniqueProcess != 4 + ): # The System process (PID 4) has no create time ctime = self.get_create_time() if not isinstance(ctime, datetime.datetime): return False @@ -518,14 +519,14 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return False # passed all validations - return True - + return True + def get_create_time(self): return conversion.wintime_to_datetime(self.CreateTime.QuadPart) def get_exit_time(self): return conversion.wintime_to_datetime(self.ExitTime.QuadPart) - + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 524ff59107ee857c6d4e86697fd6db7f92c05156 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Fri, 7 Jul 2023 20:22:58 +0300 Subject: [PATCH 022/979] bumped MINOR_VERSION to 2.5.2 after changes, and updated dependent thrdscan plugin's required version to this --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3a6b24ea8..09dded076 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,7 +44,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 4 # Number of changes that only add to the interface +VERSION_MINOR = 5 # Number of changes that only add to the interface VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 813883c2d..4afc29cdb 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + # version 2.5.2 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 5, 2) _version = (1, 0, 0) @classmethod From abf1c2e03d67f02ea5c31cc9f4b16029cbbe946d Mon Sep 17 00:00:00 2001 From: RuBublik Date: Fri, 7 Jul 2023 20:25:02 +0300 Subject: [PATCH 023/979] fixed typos --- volatility3/framework/plugins/windows/poolscanner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 5539e1e84..1f70cfb8c 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -230,7 +230,7 @@ class PoolScanner(plugins.PluginInterface): b"Thr\xe5", # -> “protected” allocation, MSB is set. type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size=(600, None), # -> 0x0258 - size of strcut in win5.1 + size=(600, None), # -> 0x0258 - size of struct in win5.1 skip_type_test=True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), @@ -239,7 +239,7 @@ class PoolScanner(plugins.PluginInterface): b"Thre", type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size=(600, None), # -> 0x0258 - size of strcut in win5.1 + size=(600, None), # -> 0x0258 - size of struct in win5.1 page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # files on windows before windows 8 From 338238c6396aa4c53a579a61a64eb4139de6cc76 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 16 Jul 2023 00:10:31 +0300 Subject: [PATCH 024/979] fixed build number - resets when MINOR version goes up --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09dded076..de1674885 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 4afc29cdb..6e19f9bd5 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # version 2.5.2 adds support for scanning for 'Ethread' structures by pool tags - _required_framework_version = (2, 5, 2) + # version 2.5.0 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 5, 0) _version = (1, 0, 0) @classmethod From 43bef641dc9b79f6ba3f5c0e345e9d76ffcd1318 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Sep 2023 20:37:38 +0100 Subject: [PATCH 025/979] Revert "Linux: Fix slight issue in envvars renaming" This reverts commit 60292c2da1efa886d0f46bd720af537a6069a623. --- volatility3/framework/plugins/linux/envars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 758943312..c4e3ed3c9 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -1,4 +1,4 @@ -from volatility3.plugins.linux import envvars +from volatility3.plugins import envvars import logging vollog = logging.getLogger(__name__) From 56cb5ea93ba74819ba4fe3a9bec39220d6d60d43 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Sep 2023 20:37:51 +0100 Subject: [PATCH 026/979] Revert "Linux: Rename linux.envars to linux.envvars" This reverts commit d9a365d96fcd990c7faba32ab7aa63523203e9f8. --- volatility3/framework/plugins/linux/envars.py | 121 +++++++++++++++++- .../framework/plugins/linux/envvars.py | 121 ------------------ 2 files changed, 115 insertions(+), 127 deletions(-) delete mode 100644 volatility3/framework/plugins/linux/envvars.py diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index c4e3ed3c9..5cbf0f502 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -1,12 +1,121 @@ -from volatility3.plugins import envvars +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import logging +from volatility3.framework import exceptions, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist + vollog = logging.getLogger(__name__) -class Envars(envvars.Envvars): - def run(self, *args, **kwargs): - vollog.warning( - "The linux.envars plugin has been renamed to linux.envvars and will only be accessible through the new name in a future release" +class Envars(plugins.PluginInterface): + """Lists processes with their environment variables""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + ] + + def _generator(self, tasks): + """Generates a listing of processes along with environment variables""" + + # walk the process list and return the envars + for task in tasks: + pid = task.pid + + # get process name as string + name = utility.array_to_string(task.comm) + + # try and get task parent + try: + ppid = task.parent.pid + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." + ) + ppid = 0 + + # kernel threads never have an mm as they do not have userland mappings + try: + mm = task.mm + except exceptions.InvalidAddressException: + # no mm so cannot get envars + vollog.debug( + f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." + ) + mm = None + continue + + # if mm exists attempt to get envars + if mm: + # get process layer to read envars from + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + vollog.debug( + f"Unable to construct process layer for task {pid} {name}, will not extract any envars." + ) + continue + proc_layer = self.context.layers[proc_layer_name] + + # get the size of the envars with sanity checking + envars_size = task.mm.env_end - task.mm.env_start + if not (0 < envars_size <= 8192): + vollog.debug( + f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." + ) + continue + + # attempt to read all envars data + try: + envar_data = proc_layer.read(task.mm.env_start, envars_size) + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." + ) + continue + + # parse envar data, envars are null terminated, keys and values are separated by '=' + envar_data = envar_data.rstrip(b"\x00") + for envar_pair in envar_data.split(b"\x00"): + try: + key, value = envar_pair.decode().split("=", 1) + except ValueError: + vollog.debug( + f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" + ) + continue + yield (0, (pid, ppid, name, key, value)) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + return renderers.TreeGrid( + [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], + self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ), ) - return super().run(*args, **kwargs) diff --git a/volatility3/framework/plugins/linux/envvars.py b/volatility3/framework/plugins/linux/envvars.py deleted file mode 100644 index 1d6c8b784..000000000 --- a/volatility3/framework/plugins/linux/envvars.py +++ /dev/null @@ -1,121 +0,0 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -import logging - -from volatility3.framework import exceptions, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins -from volatility3.framework.objects import utility -from volatility3.plugins.linux import pslist - -vollog = logging.getLogger(__name__) - - -class Envvars(plugins.PluginInterface): - """Lists processes with their environment variables""" - - _required_framework_version = (2, 0, 0) - - @classmethod - def get_requirements(cls): - # Since we're calling the plugin, make sure we have the plugin's requirements - return [ - requirements.ModuleRequirement( - name="kernel", - description="Linux kernel", - architectures=["Intel32", "Intel64"], - ), - requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) - ), - requirements.ListRequirement( - name="pid", - description="Filter on specific process IDs", - element_type=int, - optional=True, - ), - ] - - def _generator(self, tasks): - """Generates a listing of processes along with environment variables""" - - # walk the process list and return the envars - for task in tasks: - pid = task.pid - - # get process name as string - name = utility.array_to_string(task.comm) - - # try and get task parent - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." - ) - ppid = 0 - - # kernel threads never have an mm as they do not have userland mappings - try: - mm = task.mm - except exceptions.InvalidAddressException: - # no mm so cannot get envars - vollog.debug( - f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." - ) - mm = None - continue - - # if mm exists attempt to get envars - if mm: - # get process layer to read envars from - proc_layer_name = task.add_process_layer() - if proc_layer_name is None: - vollog.debug( - f"Unable to construct process layer for task {pid} {name}, will not extract any envars." - ) - continue - proc_layer = self.context.layers[proc_layer_name] - - # get the size of the envars with sanity checking - envars_size = task.mm.env_end - task.mm.env_start - if not (0 < envars_size <= 8192): - vollog.debug( - f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." - ) - continue - - # attempt to read all envars data - try: - envar_data = proc_layer.read(task.mm.env_start, envars_size) - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." - ) - continue - - # parse envar data, envars are null terminated, keys and values are separated by '=' - envar_data = envar_data.rstrip(b"\x00") - for envar_pair in envar_data.split(b"\x00"): - try: - key, value = envar_pair.decode().split("=", 1) - except ValueError: - vollog.debug( - f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" - ) - continue - yield (0, (pid, ppid, name, key, value)) - - def run(self): - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - - return renderers.TreeGrid( - [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], - self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ), - ) From 62466c7953cb0264bc3bb491a694a55732115d7a Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 3 Oct 2023 21:35:37 +0300 Subject: [PATCH 027/979] fixed merge conflicts with 'volatilityfoundation:develop' branch - bumped VERSION_PATCH --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index de1674885..c3ebaca27 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 57d995a81d3b6a9d1843439477c0be6933215df0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 8 Oct 2023 03:15:23 +0200 Subject: [PATCH 028/979] manually instantiate queue_entry for tasks symbol --- volatility3/framework/plugins/mac/pslist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 88045a277..c0e149fc2 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object_from_symbol(symbol_name="tasks") + queue_entry = kernel.object("queue_entry", kernel.get_symbol("tasks").address) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 6e5d41c38b3c494b9a43d5a7fca515aa84e1b6d4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 8 Oct 2023 03:37:17 +0200 Subject: [PATCH 029/979] manually instantiate queue_entry for tasks symbol --- volatility3/framework/plugins/mac/pslist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index c0e149fc2..e8c490dff 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object("queue_entry", kernel.get_symbol("tasks").address) + queue_entry = kernel.object(object_type="queue_entry", offset=kernel.get_symbol("tasks").address) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 3a656266711b21951ea8c48fe9d8ac4a4cf64775 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 23 Oct 2023 17:47:57 +0200 Subject: [PATCH 030/979] black formatting --- volatility3/framework/plugins/mac/pslist.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index e8c490dff..9835644b8 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,9 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object(object_type="queue_entry", offset=kernel.get_symbol("tasks").address) + queue_entry = kernel.object( + object_type="queue_entry", offset=kernel.get_symbol("tasks").address + ) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 604c23bcbc88679a48d1348ef3c8d2163726ab44 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Thu, 14 Dec 2023 14:50:17 +0100 Subject: [PATCH 031/979] Import Address Table Plugin --- volatility3/framework/plugins/windows/iat.py | 132 +++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 volatility3/framework/plugins/windows/iat.py diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py new file mode 100644 index 000000000..1cf51cfa2 --- /dev/null +++ b/volatility3/framework/plugins/windows/iat.py @@ -0,0 +1,132 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + +import logging +import io +from typing import Callable, List +from volatility3.framework.symbols import intermed +from volatility3.framework import renderers, interfaces, exceptions, constants +from volatility3.framework.configuration import requirements +from volatility3.plugins.windows import pslist +from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.symbols.windows.extensions import pe +import pefile + +vollog = logging.getLogger(__name__) + + +class IAT(interfaces.plugins.PluginInterface): + """Extract Import Address Table to list API (functions) used by a program contained in external libraries""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process ID to include (all other processes are excluded)", + optional=True, + ), + ] + + def _generator(self, procs): + kernel = self.context.modules[self.config["kernel"]] + + for proc in procs: + try: + proc_id = proc.UniqueProcessId + proc_layer_name = proc.add_process_layer() + peb = self.context.object( + kernel.symbol_table_name + constants.BANG + "_PEB", + layer_name=proc_layer_name, + offset=proc.Peb, + ) + + if proc_layer_name is None: + raise TypeError("Layer must be a string not None") + + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "windows", + "pe", + class_types=pe.class_types, + ) + pe_data = io.BytesIO() + + dos_header = self.context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=peb.ImageBaseAddress, + layer_name=proc_layer_name, + ) + + for offset, data in dos_header.reconstruct(): + pe_data.seek(offset) + pe_data.write(data) + + pe_obj = pefile.PE(data=pe_data.getvalue(), fast_load=True) + pe_obj.parse_data_directories( + [pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_IMPORT"]] + ) + if hasattr(pe_obj, "DIRECTORY_ENTRY_IMPORT"): + for entry in pe_obj.DIRECTORY_ENTRY_IMPORT: + dll_entry = entry.dll + if dll_entry: + dll_entry = dll_entry.decode() + else: + dll_entry = renderers.NotAvailableValue + + # Iterate over imported functions + for imp in entry.imports: + import_name = imp.name + if import_name: + import_name = imp.name.decode() + else: + import_name = renderers.NotAvailableValue() + yield ( + 0, + ( + proc_id, + proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ), + dll_entry, + import_name, + ), + ) + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) + ) + continue + + def run(self): + kernel = self.context.modules[self.config["kernel"]] + + return renderers.TreeGrid( + [("PID", int), ("Process", str), ("Library", str), ("Function", str)], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=pslist.PsList.create_pid_filter( + self.config.get("pid", None) + ), + ) + ), + ) From 059f2d012896d96fa0e2d20469d7771ec27b0b6d Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 16 Dec 2023 21:19:35 +0100 Subject: [PATCH 032/979] Adding function addr + bound info. Formatting code. --- volatility3/framework/plugins/windows/iat.py | 32 +++++++++++++++----- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 1cf51cfa2..11c273859 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,16 +1,13 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -import logging -import io -from typing import Callable, List +import logging, io, pefile from volatility3.framework.symbols import intermed from volatility3.framework import renderers, interfaces, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.plugins.windows import pslist -from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.renderers import format_hints from volatility3.framework.symbols.windows.extensions import pe -import pefile vollog = logging.getLogger(__name__) @@ -86,6 +83,12 @@ class IAT(interfaces.plugins.PluginInterface): else: dll_entry = renderers.NotAvailableValue + bound = True + # Initially set to 0 if not bound + time_date_stamp = entry.struct.TimeDateStamp + if not time_date_stamp: + bound = False + # Iterate over imported functions for imp in entry.imports: import_name = imp.name @@ -93,6 +96,12 @@ class IAT(interfaces.plugins.PluginInterface): import_name = imp.name.decode() else: import_name = renderers.NotAvailableValue() + function_address = ( + pe_obj.OPTIONAL_HEADER.ImageBase + imp.address + ) + if not function_address: + function_address = renderers.NotAvailableValue + yield ( 0, ( @@ -103,7 +112,9 @@ class IAT(interfaces.plugins.PluginInterface): errors="replace", ), dll_entry, + bound, import_name, + format_hints.Hex(function_address), ), ) except exceptions.InvalidAddressException as excp: @@ -118,7 +129,14 @@ class IAT(interfaces.plugins.PluginInterface): kernel = self.context.modules[self.config["kernel"]] return renderers.TreeGrid( - [("PID", int), ("Process", str), ("Library", str), ("Function", str)], + [ + ("PID", int), + ("Name", str), + ("Library", str), + ("Bound", bool), + ("Function", str), + ("Address", format_hints.Hex), + ], self._generator( pslist.PsList.list_processes( context=self.context, From 84401847ff50009c120dcbbfdf17868b86c2a01b Mon Sep 17 00:00:00 2001 From: Valentin Obst Date: Wed, 20 Dec 2023 18:38:07 +0100 Subject: [PATCH 033/979] add sanity check in Linux find_aslr to skip unrelocated init_task --- volatility3/framework/automagic/linux.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2eebcc2dc..fda71b766 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -156,6 +156,18 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): and init_task.state.cast("unsigned int") != 0 ): continue + elif init_task.active_mm.cast("long unsigned int") == module.get_symbol( + "init_mm" + ).address and init_task.tasks.next.cast( + "long unsigned int" + ) == init_task.tasks.prev.cast( + "long unsigned int" + ): + # The idle task steals `mm` from previously running task, i.e., + # `init_mm` is only used as long as no CPU has ever been idle. + # This catches cases where we found a fragment of the + # unrelocated ELF file instead of the running kernel. + continue # This we get for free aslr_shift = ( From 409ce95980b0e4c5a29222f4acb90b681886d16f Mon Sep 17 00:00:00 2001 From: hsarkey Date: Thu, 7 Dec 2023 14:04:44 -0500 Subject: [PATCH 034/979] Windows: Added '--refined' option to windows malfind plugin Also updated malfind to include "\x55\x48" and "\x55\x89" as part of the refined_criteria list. --- volatility3/framework/plugins/windows/malfind.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 6ed078996..1c73fdf1c 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -46,6 +46,12 @@ class Malfind(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) ), + requirements.BooleanRequirement( + name="refined", + description="Refine the output. Only show regions with an MZ header or that start with well known opcode combinations (i.e. PUSH EBP). WARNING: This can cause you to overlook regions with wiped headers or shell code blocks starting with NOP sleds, etc.. However, it will in general result in less noisy output.", + default=False, + optional=True, + ), ] @classmethod @@ -138,6 +144,9 @@ class Malfind(interfaces.plugins.PluginInterface): yield vad, data def _generator(self, procs): + # set refined criteria + refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] + # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] @@ -151,6 +160,10 @@ class Malfind(interfaces.plugins.PluginInterface): for vad, data in self.list_injections( self.context, kernel.layer_name, kernel.symbol_table_name, proc ): + # check if refined option was passed + if self.config["refined"] and data[0:2] not in refined_criteria: + continue + # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 if is_32bit_arch or proc.get_is_wow64(): architecture = "intel" From 25637a41e05e0bc5fccded01cf1913d45668ac25 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 8 Jan 2024 23:35:21 +0200 Subject: [PATCH 035/979] added account for XP timestamps - bit shifted --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 8790f41a7..9a286bc26 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -522,6 +522,9 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return True def get_create_time(self): + # For Windows XPs + if self.has_member("ThreadsProcess"): + return conversion.wintime_to_datetime(self.CreateTime.QuadPart >> 3) return conversion.wintime_to_datetime(self.CreateTime.QuadPart) def get_exit_time(self): From 550112b848913954e8b03d8c40bff9e2fd7902d7 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 8 Jan 2024 23:37:38 +0200 Subject: [PATCH 036/979] added another sanity check to ETHREAD.is_valid --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 9a286bc26..f4e2cc485 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -515,6 +515,9 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False + if not (1998 < ctime.year < 2030): + return False + except exceptions.InvalidAddressException: return False From 7c370121181af5da71b75d2b844def89341ddecd Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 9 Jan 2024 00:16:18 +0200 Subject: [PATCH 037/979] bumped version constants to mark change of interface (add of support for ETHREAD) --- volatility3/framework/constants/__init__.py | 4 ++-- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index c3ebaca27..9aaafb933 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 6 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 6e19f9bd5..80906b3b9 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # version 2.5.0 adds support for scanning for 'Ethread' structures by pool tags - _required_framework_version = (2, 5, 0) + # version 2.6.0 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 6, 0) _version = (1, 0, 0) @classmethod From b0d84e55ab1846004318e87bd7b5f427e7d70551 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 9 Jan 2024 21:32:29 +0200 Subject: [PATCH 038/979] fix indentation (typo) --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index f4e2cc485..a0af29d18 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -516,7 +516,7 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return False if not (1998 < ctime.year < 2030): - return False + return False except exceptions.InvalidAddressException: return False From abc448009eddd8563835f233e8e95095a7b32a15 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 10 Jan 2024 15:56:00 -0300 Subject: [PATCH 039/979] Linux: Add netfilter hooks enumeration plugin. --- .../framework/plugins/linux/netfilter.py | 715 ++++++++++++++++++ 1 file changed, 715 insertions(+) create mode 100644 volatility3/framework/plugins/linux/netfilter.py diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py new file mode 100644 index 000000000..cc4786b4e --- /dev/null +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -0,0 +1,715 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +from dataclasses import dataclass, field +from abc import ABC, abstractmethod +import logging + +from typing import Iterator, List, Tuple +from volatility3.framework import ( + class_subclasses, + constants, + interfaces, + renderers, +) +from volatility3.framework.renderers import format_hints +from volatility3.framework.configuration import requirements +from volatility3.framework.symbols import linux +from volatility3.plugins.linux import lsmod + +vollog = logging.getLogger(__name__) + + +@dataclass +class Proto: + name: str + hooks: Tuple[str] = field(default_factory=tuple) + + +PROTO_NOT_IMPLEMENTED = Proto(name="UNSPEC") + +NF_INET_HOOKS = ("PRE_ROUTING", "LOCAL_IN", "FORWARD", "LOCAL_OUT", "POST_ROUTING") +NF_DEC_HOOKS = ( + "PRE_ROUTING", + "LOCAL_IN", + "FORWARD", + "LOCAL_OUT", + "POST_ROUTING", + "HELLO", + "ROUTE", +) +NF_ARP_HOOKS = ("IN", "OUT", "FORWARD") +NF_NETDEV_HOOKS = ("INGRESS", "EGRESS") +LARGEST_HOOK_NUMBER = max( + len(NF_INET_HOOKS), len(NF_DEC_HOOKS), len(NF_ARP_HOOKS), len(NF_NETDEV_HOOKS) +) + + +class AbstractNetfilter(ABC): + """Netfilter Abstract Base Classes handling details across various + Netfilter implementations, including constants, helpers, and common + routines. + """ + + PROTO_HOOKS = ( + PROTO_NOT_IMPLEMENTED, # NFPROTO_UNSPEC + Proto(name="INET", hooks=NF_INET_HOOKS), # From kernels 3.14 + Proto(name="IPV4", hooks=NF_INET_HOOKS), + Proto(name="ARP", hooks=("IN", "OUT", "FORWARD")), + PROTO_NOT_IMPLEMENTED, + Proto(name="NETDEV", hooks=("INGRESS", "EGRESS")), + PROTO_NOT_IMPLEMENTED, + Proto(name="BRIDGE", hooks=NF_INET_HOOKS), + PROTO_NOT_IMPLEMENTED, + PROTO_NOT_IMPLEMENTED, + Proto(name="IPV6", hooks=NF_INET_HOOKS), + PROTO_NOT_IMPLEMENTED, + Proto(name="DECNET", hooks=NF_INET_HOOKS), # Removed in kernel 6.1 + ) + NF_MAX_HOOKS = LARGEST_HOOK_NUMBER + 1 + + def __init__( + self, + context: interfaces.context.ContextInterface, + config: interfaces.configuration.HierarchicalDict, + ): + self._context = context + self._config = config + symbol_table = self._config["kernel"] + self.vmlinux = context.modules[symbol_table] + self.layer_name = self.vmlinux.layer_name + + modules = lsmod.Lsmod.list_modules(context, symbol_table) + self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( + context, symbol_table, modules + ) + + self._set_data_sizes() + + def _set_data_sizes(self): + self.ptr_size = self.vmlinux.get_type("pointer").size + self.list_head_size = self.vmlinux.get_type("list_head").size + + @classmethod + def run_all( + cls, + context: interfaces.context.ContextInterface, + config: interfaces.configuration.HierarchicalDict, + ) -> Iterator[Tuple[int, str, str, int, int, str, bool]]: + """It calls each subclass symtab_checks() to test the required + conditions to that specific kernel implementation. + + Args: + context: The volatility3 context on which to operate + config: Core configuration + + Yields: + The kmsg records. Same as _run() + """ + vmlinux = context.modules[config["kernel"]] + + implementation_inst = None # type: ignore + for subclass in class_subclasses(cls): + if not subclass.symtab_checks(vmlinux=vmlinux): + vollog.log( + constants.LOGLEVEL_VVVV, + "Netfilter implementation '%s' doesn't match this memory dump", + subclass.__name__, + ) + continue + + vollog.log( + constants.LOGLEVEL_VVVV, + "Netfilter implementation '%s' matches!", + subclass.__name__, + ) + implementation_inst = subclass(context=context, config=config) + # More than one class could be executed for an specific kernel version + # For instance: Netfilter Ingress hooks + yield from implementation_inst._run() + + if implementation_inst is None: + vollog.error("Unsupported Netfilter kernel implementation") + + def _run(self) -> Iterator[Tuple[int, str, str, int, int, str, bool]]: + """Iterates over namespaces and protocols, executing various callbacks that + allow customization of the code to the specific data structure used in a + particular kernel implementation + + get_hooks_container_by_protocol(net, proto_name) + It returns the data structure used in a specific kernel implementation + to store the hooks for a respective namespace and protocol, basically: + For Ingress hooks: + network_namespace[] -> net_device[] -> nf_hooks_ingress[] + For all the other Netfilter hooks: + <= 4.2.8 + nf_hooks[] + >= 4.3 + network_namespace[] -> nf.hooks[] + + get_hook_ops(hook_container, proto_idx, hook_idx) + Give the 'hook_container' got in get_hooks_container_by_protocol(), it + returns an iterable of 'nf_hook_ops' elements for a respective protocol + and hook type. + + Returns: + netns [int]: Network namespace id + proto_name [str]: Protocol name + hook_name [str]: Hook name + priority [int]: Priority + hook_ops_hook [int]: Hook address + module_name [str]: Linux kernel module name + hooked [bool]: hooked? + """ + for netns, net in self.get_net_namespaces(): + for proto_idx, proto_name, hook_idx, hook_name in self._proto_hook_loop(): + hooks_container = self.get_hooks_container_by_protocol(net, proto_name) + + for hook_container in hooks_container: + for hook_ops in self.get_hook_ops( + hook_container, proto_idx, hook_idx + ): + if not hook_ops: + continue + + priority = int(hook_ops.priority) + hook_ops_hook = hook_ops.hook + module_name = self.get_module_name_for_address(hook_ops_hook) + hooked = module_name is not None + + yield netns, proto_name, hook_name, priority, hook_ops_hook, module_name, hooked + + @classmethod + @abstractmethod + def symtab_checks(cls, vmlinux: interfaces.context.ModuleInterface) -> bool: + """This method on each sublasss will be called to evaluate if the kernel + being analyzed fulfill the type & symbols requirements for the implementation. + The first class returning True will be instantiated and called via the + run() method. + + Returns: + bool: True if the kernel being analyzed fulfill the class requirements. + """ + + def _proto_hook_loop(self) -> Iterator[Tuple[int, str, int, str]]: + """Flattens the protocol families and hooks""" + for proto_idx, proto in enumerate(AbstractNetfilter.PROTO_HOOKS): + if proto == PROTO_NOT_IMPLEMENTED: + continue + if proto.name not in self.subscribed_protocols(): + # This protocol is not managed in this object + continue + for hook_idx, hook_name in enumerate(proto.hooks): + yield proto_idx, proto.name, hook_idx, hook_name + + def build_nf_hook_ops_array(self, nf_hook_entries): + """Function helper to build the nf_hook_ops array when it is not part of the + struct 'nf_hook_entries' definition. + + nf_hook_ops was stored adjacent in memory to the nf_hook_entry array, in the + new struct 'nf_hook_entries'. However, this 'nf_hooks_ops' array 'orig_ops' is + not part of the 'nf_hook_entries' struct. So, we need to calculate the offset. + + struct nf_hook_entries { + u16 num_hook_entries; /* plus padding */ + struct nf_hook_entry hooks[]; + //const struct nf_hook_ops *orig_ops[]; + } + """ + nf_hook_entry_size = self.vmlinux.get_type("nf_hook_entry").size + orig_ops_addr = ( + nf_hook_entries.hooks.vol.offset + + nf_hook_entry_size * nf_hook_entries.num_hook_entries + ) + orig_ops = self._context.object( + object_type=self.get_symbol_fullname("array"), + offset=orig_ops_addr, + subtype=self.vmlinux.get_type("pointer"), + layer_name=self.layer_name, + count=nf_hook_entries.num_hook_entries, + ) + + return orig_ops + + def subscribed_protocols(self) -> Tuple[str]: + """Allows to select which PROTO_HOOKS protocols will be processed by the + Netfiler subclass. + """ + + # Most implementation handlers respond to these protocols, except for + # the ingress hook, which specifically handles the 'NETDEV' protocol. + # However, there is no corresponding Netfilter hook implementation for + # the INET protocol in the kernel. AFAIU, this is used as + # 'NFPROTO_INET = NFPROTO_IPV4 || NFPROTO_IPV6' + # in other parts of the kernel source code. + return ("IPV4", "ARP", "BRIDGE", "IPV6", "DECNET") + + def get_module_name_for_address(self, addr) -> str: + """Helper to obtain the module and symbol name in the format needed for the + output of this plugin. + """ + module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( + self.vmlinux, self.handlers, addr + ) + + if module_name == "UNKNOWN": + module_name = None + + if symbol_name != "N/A": + module_name = f"[{symbol_name}]" + + return module_name + + def get_net_namespaces(self): + """Common function to retrieve the different namespaces. + From 4.3 on, all the implementations use network namespaces. + """ + nethead = self.vmlinux.object_from_symbol("net_namespace_list") + symbol_net_name = self.get_symbol_fullname("net") + for net in nethead.to_list(symbol_net_name, "list"): + net_ns_id = net.ns.inum + yield net_ns_id, net + + def get_hooks_container_by_protocol(self, net, proto_name): + """Returns the data structure used in a specific kernel implementation to store + the hooks for a respective namespace and protocol. + + Except for kernels < 4.3, all the implementations use network namespaces. + Also the data structure which contains the hooks, even though it changes its + implementation and/or data type, it is always in this location. + """ + yield net.nf.hooks + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + """Given the hook_container obtained from get_hooks_container_by_protocol(), it + returns an iterable of 'nf_hook_ops' elements for a corresponding protocol + and hook type. + + This is the most variable/unstable part of all Netfilter hook designs, it + changes almost in every single implementation. + """ + raise NotImplementedError("You must implement this method") + + def get_symbol_fullname(self, symbol_basename: str) -> str: + """Given a short symbol or type name, it returns its full name""" + return self.vmlinux.symbol_table_name + constants.BANG + symbol_basename + + @staticmethod + def get_member_type( + vol_type: interfaces.objects.Template, member_name: str + ) -> List[str]: + """Returns a list of types/subtypes belonging to the given type member. + + Args: + vol_type (interfaces.objects.Template): A vol3 type object + member_name (str): The member name + + Returns: + list: A list of types/subtypes + """ + _size, vol_obj = vol_type.vol.members[member_name] + type_name = vol_obj.type_name + type_basename = type_name.split(constants.BANG)[1] + member_type = [type_basename] + cur_type = vol_obj + while hasattr(cur_type, "subtype"): + subtype_name = cur_type.subtype.type_name + subtype_basename = subtype_name.split(constants.BANG)[1] + member_type.append(subtype_basename) + cur_type = cur_type.subtype + + return member_type + + +class NetfilterImp_to_4_3(AbstractNetfilter): + """At this point, Netfilter hooks were implemented as a linked list of struct + 'nf_hook_ops' type. One linked list per protocol per hook type. + It was like that until 4.2.8. + + struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return vmlinux.has_symbol("nf_hooks") + + def get_net_namespaces(self): + # In kernels <= 4.2.8 netfilter hooks are not implemented per namespaces + netns, net = renderers.NotAvailableValue(), renderers.NotAvailableValue() + yield netns, net + + def get_hooks_container_by_protocol(self, net, proto_name): + nf_hooks = self.vmlinux.object_from_symbol("nf_hooks") + if not nf_hooks: + return + + yield nf_hooks + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + list_head = hook_container[proto_idx][hook_idx] + nf_hooks_ops_name = self.get_symbol_fullname("nf_hook_ops") + return list_head.to_list(nf_hooks_ops_name, "list") + + +class NetfilterImp_4_3_to_4_9(AbstractNetfilter): + """Netfilter hooks were added to network namepaces in 4.3. + It is still implemented as a linked list of 'struct nf_hook_ops' type but inside a + network namespace. One linked list per protocol per hook type. + + struct net { ... struct netns_nf nf; ... } + struct netns_nf { ... + struct list_head hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; ... } + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("netns_nf") + and vmlinux.get_type("netns_nf").has_member("hooks") + and cls.get_member_type(vmlinux.get_type("netns_nf"), "hooks") + == ["array", "array", "list_head"] + ) + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + list_head = hook_container[proto_idx][hook_idx] + nf_hooks_ops_name = self.get_symbol_fullname("nf_hook_ops") + return list_head.to_list(nf_hooks_ops_name, "list") + + +class NetfilterImp_4_9_to_4_14(AbstractNetfilter): + """In this range of kernel versions, the doubly-linked lists of netfilter hooks were + replaced by an array of arrays of 'nf_hook_entry' pointers in a singly-linked lists. + struct net { ... struct netns_nf nf; ... } + struct netns_nf { .. + struct nf_hook_entry __rcu *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; ... } + + Also in v4.10 the struct nf_hook_entry changed, a hook function pointer was added to + it. However, for simplicity of this design, we will still take the hook address from + the 'nf_hook_ops'. As per v5.0-rc2, the hook address is duplicated in both sides. + - v4.9: + struct nf_hook_entry { + struct nf_hook_entry *next; + struct nf_hook_ops ops; + const struct nf_hook_ops *orig_ops; }; + - v4.10: + struct nf_hook_entry { + struct nf_hook_entry *next; + nf_hookfn *hook; + void *priv; + const struct nf_hook_ops *orig_ops; }; + (*) Even though the hook address is in the struct 'nf_hook_entry', we use the + original 'nf_hook_ops' hook address value, the one which was filled by the user, to + make it uniform to all the implementations. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + hooks_type = ["array", "array", "pointer", "nf_hook_entry"] + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("netns_nf") + and vmlinux.get_type("netns_nf").has_member("hooks") + and cls.get_member_type(vmlinux.get_type("netns_nf"), "hooks") == hooks_type + ) + + def _get_hook_ops(self, hook_container, proto_idx, hook_idx): + list_head = hook_container[proto_idx][hook_idx] + nf_hooks_ops_name = self.get_symbol_fullname("nf_hook_ops") + return list_head.to_list(nf_hooks_ops_name, "list") + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + nf_hook_entry_list = hook_container[proto_idx][hook_idx] + while nf_hook_entry_list: + yield nf_hook_entry_list.orig_ops + nf_hook_entry_list = nf_hook_entry_list.next + + +class NetfilterImp_4_14_to_4_16(AbstractNetfilter): + """'nf_hook_ops' was removed from struct 'nf_hook_entry'. Instead, it was stored + adjacent in memory to the 'nf_hook_entry' array, in the new struct 'nf_hook_entries' + However, 'orig_ops' is not part of the 'nf_hook_entries' struct definition. So, we + have to craft it by hand. + + struct net { ... struct netns_nf nf; ... } + struct netns_nf { + struct nf_hook_entries *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; ... } + struct nf_hook_entries { + u16 num_hook_entries; /* plus padding */ + struct nf_hook_entry hooks[]; + //const struct nf_hook_ops *orig_ops[]; } + struct nf_hook_entry { + nf_hookfn *hook; + void *priv; } + + (*) Even though the hook address is in the struct 'nf_hook_entry', we use the + original 'nf_hook_ops' hook address value, the one which was filled by the user, to + make it uniform to all the implementations. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + hooks_type = ["array", "array", "pointer", "nf_hook_entries"] + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("netns_nf") + and vmlinux.get_type("netns_nf").has_member("hooks") + and cls.get_member_type(vmlinux.get_type("netns_nf"), "hooks") == hooks_type + ) + + def get_nf_hook_entries(self, nf_hooks_addr, proto_idx, hook_idx): + """This allows to support different hook array implementations from this version + on. For instance, in kernels >= 4.16 this multi-dimensional array is split in + one-dimensional array of pointers to 'nf_hooks_entries' per each protocol.""" + return nf_hooks_addr[proto_idx][hook_idx] + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + nf_hook_entries = self.get_nf_hook_entries(hook_container, proto_idx, hook_idx) + if not nf_hook_entries: + return + + nf_hook_ops_name = self.get_symbol_fullname("nf_hook_ops") + nf_hook_ops_ptr_arr = self.build_nf_hook_ops_array(nf_hook_entries) + for nf_hook_ops_ptr in nf_hook_ops_ptr_arr: + nf_hook_ops = nf_hook_ops_ptr.dereference().cast(nf_hook_ops_name) + yield nf_hook_ops + + +class NetfilterImp_4_16_to_latest(NetfilterImp_4_14_to_4_16): + """The multidimensional array of nf_hook_entries was split in a one-dimensional + array per each protocol. + + struct net { + struct netns_nf nf; ... } + struct netns_nf { + struct nf_hook_entries * hooks_ipv4[NF_INET_NUMHOOKS]; + struct nf_hook_entries * hooks_ipv6[NF_INET_NUMHOOKS]; + struct nf_hook_entries * hooks_arp[NF_ARP_NUMHOOKS]; + struct nf_hook_entries * hooks_bridge[NF_INET_NUMHOOKS]; + struct nf_hook_entries * hooks_decnet[NF_DN_NUMHOOKS]; ... } + struct nf_hook_entries { + u16 num_hook_entries; /* plus padding */ + struct nf_hook_entry hooks[]; + //const struct nf_hook_ops *orig_ops[]; } + struct nf_hook_entry { + nf_hookfn *hook; + void *priv; } + + (*) Even though the hook address is in the struct nf_hook_entry, we use the original + nf_hook_ops hook address value, the one which was filled by the user, to make it + uniform to all the implementations. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("netns_nf") + and vmlinux.get_type("netns_nf").has_member("hooks_ipv4") + ) + + def get_hooks_container_by_protocol(self, net, proto_name): + try: + if proto_name == "IPV4": + net_nf_hooks = net.nf.hooks_ipv4 + elif proto_name == "ARP": + net_nf_hooks = net.nf.hooks_arp + elif proto_name == "BRIDGE": + net_nf_hooks = net.nf.hooks_bridge + elif proto_name == "IPV6": + net_nf_hooks = net.nf.hooks_ipv6 + elif proto_name == "DECNET": + net_nf_hooks = net.nf.hooks_decnet + else: + return + + yield net_nf_hooks + + except AttributeError: + # Protocol family disabled at kernel compilation + # CONFIG_NETFILTER_FAMILY_ARP=n || + # CONFIG_NETFILTER_FAMILY_BRIDGE=n || + # CONFIG_DECNET=n + pass + + def _get_nf_hook_entries_ptr(self, nf_hooks_addr, proto_idx, hook_idx): + nf_hook_entries_ptr = nf_hooks_addr[hook_idx] + return nf_hook_entries_ptr + + def get_nf_hook_entries(self, nf_hooks_addr, proto_idx, hook_idx): + return nf_hooks_addr[hook_idx] + + +class AbstractNetfilterNetDev(AbstractNetfilter): + """Base class to handle the Netfilter NetDev hooks. + It won't be executed. It has some common functions to all Netfilter NetDev hook + implementions. + + Netfilter NetDev hooks are set per network device which belongs to a network + namespace. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return False + + def subscribed_protocols(self): + return ("NETDEV",) + + def get_hooks_container_by_protocol(self, net, proto_name): + if proto_name != "NETDEV": + return + + net_device_type = self.vmlinux.get_type("net_device") + net_device_name = self.get_symbol_fullname("net_device") + for net_device in net.dev_base_head.to_list(net_device_name, "dev_list"): + if net_device_type.has_member("nf_hooks_ingress"): + # CONFIG_NETFILTER_INGRESS=y + yield net_device.nf_hooks_ingress + + if net_device_type.has_member("nf_hooks_egress"): + # CONFIG_NETFILTER_EGRESS=y + yield net_device.nf_hooks_egress + + +class NetfilterIngressImp_4_2_to_4_9(AbstractNetfilterNetDev): + """This is the first version of Netfilter Ingress hooks which was implemented using + a doubly-linked list of 'nf_hook_ops'. + struct list_head nf_hooks_ingress; + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + hooks_type = ["list_head"] + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("net_device") + and vmlinux.get_type("net_device").has_member("nf_hooks_ingress") + and cls.get_member_type(vmlinux.get_type("net_device"), "nf_hooks_ingress") + == hooks_type + ) + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + nf_hooks_ingress = hook_container + nf_hook_ops_name = self.get_symbol_fullname("nf_hook_ops") + return nf_hooks_ingress.to_list(nf_hook_ops_name, "list") + + +class NetfilterIngressImp_4_9_to_4_14(AbstractNetfilterNetDev): + """In 4.9 it was changed to a simple singly-linked list. + struct nf_hook_entry * nf_hooks_ingress; + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + hooks_type = ["pointer", "nf_hook_entry"] + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("net_device") + and vmlinux.get_type("net_device").has_member("nf_hooks_ingress") + and cls.get_member_type(vmlinux.get_type("net_device"), "nf_hooks_ingress") + == hooks_type + ) + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + nf_hooks_ingress_ptr = hook_container + if not nf_hooks_ingress_ptr: + return + + while nf_hooks_ingress_ptr: + nf_hook_entry = nf_hooks_ingress_ptr.dereference() + orig_ops = nf_hook_entry.orig_ops.dereference() + yield orig_ops + nf_hooks_ingress_ptr = nf_hooks_ingress_ptr.next + + +class NetfilterIngressImp_4_14_to_latest(AbstractNetfilterNetDev): + """In 4.14 the hook list was converted to an array of pointers inside the struct + 'nf_hook_entries': + struct nf_hook_entries * nf_hooks_ingress; + struct nf_hook_entries { + u16 num_hook_entries; + struct nf_hook_entry hooks[]; + //const struct nf_hook_ops *orig_ops[]; } + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + hooks_type = ["pointer", "nf_hook_entries"] + return ( + vmlinux.has_symbol("net_namespace_list") + and vmlinux.has_type("net_device") + and vmlinux.get_type("net_device").has_member("nf_hooks_ingress") + and cls.get_member_type(vmlinux.get_type("net_device"), "nf_hooks_ingress") + == hooks_type + ) + + def get_hook_ops(self, hook_container, proto_idx, hook_idx): + nf_hook_entries = hook_container + if not nf_hook_entries: + return + + nf_hook_ops_name = self.get_symbol_fullname("nf_hook_ops") + nf_hook_ops_ptr_arr = self.build_nf_hook_ops_array(nf_hook_entries) + for nf_hook_ops_ptr in nf_hook_ops_ptr_arr: + nf_hook_ops = nf_hook_ops_ptr.dereference().cast(nf_hook_ops_name) + yield nf_hook_ops + + +class Netfilter(interfaces.plugins.PluginInterface): + """Lists Netfilter hooks.""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + ] + + def _format_fields(self, fields): + ( + netns, + proto_name, + hook_name, + priority, + hook_func, + module_name, + hooked, + ) = fields + return ( + netns, + proto_name, + hook_name, + priority, + format_hints.Hex(hook_func), + module_name, + str(hooked), + ) + + def _generator(self): + for fields in AbstractNetfilter.run_all( + context=self.context, config=self.config + ): + yield (0, self._format_fields(fields)) + + def run(self): + headers = [ + ("Net NS", int), + ("Proto", str), + ("Hook", str), + ("Priority", int), + ("Handler", format_hints.Hex), + ("Module", str), + ("Is Hooked", str), + ] + return renderers.TreeGrid(headers, self._generator()) From 5ad7dd3bbcbad273e21610462e7e2517e87653ed Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 10 Jan 2024 16:06:55 -0300 Subject: [PATCH 040/979] Update netdev class names as it now supports both netdev hooks; ingress and egress --- volatility3/framework/plugins/linux/netfilter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index cc4786b4e..a9dbc742a 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -572,7 +572,7 @@ class AbstractNetfilterNetDev(AbstractNetfilter): yield net_device.nf_hooks_egress -class NetfilterIngressImp_4_2_to_4_9(AbstractNetfilterNetDev): +class NetfilterNetDevImp_4_2_to_4_9(AbstractNetfilterNetDev): """This is the first version of Netfilter Ingress hooks which was implemented using a doubly-linked list of 'nf_hook_ops'. struct list_head nf_hooks_ingress; @@ -595,7 +595,7 @@ class NetfilterIngressImp_4_2_to_4_9(AbstractNetfilterNetDev): return nf_hooks_ingress.to_list(nf_hook_ops_name, "list") -class NetfilterIngressImp_4_9_to_4_14(AbstractNetfilterNetDev): +class NetfilterNetDevImp_4_9_to_4_14(AbstractNetfilterNetDev): """In 4.9 it was changed to a simple singly-linked list. struct nf_hook_entry * nf_hooks_ingress; """ @@ -623,7 +623,7 @@ class NetfilterIngressImp_4_9_to_4_14(AbstractNetfilterNetDev): nf_hooks_ingress_ptr = nf_hooks_ingress_ptr.next -class NetfilterIngressImp_4_14_to_latest(AbstractNetfilterNetDev): +class NetfilterNetDevImp_4_14_to_latest(AbstractNetfilterNetDev): """In 4.14 the hook list was converted to an array of pointers inside the struct 'nf_hook_entries': struct nf_hook_entries * nf_hooks_ingress; From 5b5012d08ad7aff2dd48c5786e1bad2c62274989 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 10 Jan 2024 16:42:34 -0300 Subject: [PATCH 041/979] Fix. Use the global lists. DEC hooks were wrong --- volatility3/framework/plugins/linux/netfilter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index a9dbc742a..2b0a16a96 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -55,16 +55,16 @@ class AbstractNetfilter(ABC): PROTO_NOT_IMPLEMENTED, # NFPROTO_UNSPEC Proto(name="INET", hooks=NF_INET_HOOKS), # From kernels 3.14 Proto(name="IPV4", hooks=NF_INET_HOOKS), - Proto(name="ARP", hooks=("IN", "OUT", "FORWARD")), + Proto(name="ARP", hooks=NF_ARP_HOOKS), PROTO_NOT_IMPLEMENTED, - Proto(name="NETDEV", hooks=("INGRESS", "EGRESS")), + Proto(name="NETDEV", hooks=NF_NETDEV_HOOKS), PROTO_NOT_IMPLEMENTED, Proto(name="BRIDGE", hooks=NF_INET_HOOKS), PROTO_NOT_IMPLEMENTED, PROTO_NOT_IMPLEMENTED, Proto(name="IPV6", hooks=NF_INET_HOOKS), PROTO_NOT_IMPLEMENTED, - Proto(name="DECNET", hooks=NF_INET_HOOKS), # Removed in kernel 6.1 + Proto(name="DECNET", hooks=NF_DEC_HOOKS), # Removed in kernel 6.1 ) NF_MAX_HOOKS = LARGEST_HOOK_NUMBER + 1 From da6c2d863d17bdcef950cc8a8f9b7aa7c9e11b2a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 10 Jan 2024 16:43:36 -0300 Subject: [PATCH 042/979] Fix netdev egress hooks --- .../framework/plugins/linux/netfilter.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 2b0a16a96..33bb5c456 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -136,11 +136,13 @@ class AbstractNetfilter(ABC): allow customization of the code to the specific data structure used in a particular kernel implementation - get_hooks_container_by_protocol(net, proto_name) + get_hooks_container(net, proto_name, hook_name) It returns the data structure used in a specific kernel implementation to store the hooks for a respective namespace and protocol, basically: For Ingress hooks: network_namespace[] -> net_device[] -> nf_hooks_ingress[] + For egress hooks: + network_namespace[] -> net_device[] -> nf_hooks_egress[] For all the other Netfilter hooks: <= 4.2.8 nf_hooks[] @@ -148,7 +150,7 @@ class AbstractNetfilter(ABC): network_namespace[] -> nf.hooks[] get_hook_ops(hook_container, proto_idx, hook_idx) - Give the 'hook_container' got in get_hooks_container_by_protocol(), it + Give the 'hook_container' got in get_hooks_container(), it returns an iterable of 'nf_hook_ops' elements for a respective protocol and hook type. @@ -163,7 +165,7 @@ class AbstractNetfilter(ABC): """ for netns, net in self.get_net_namespaces(): for proto_idx, proto_name, hook_idx, hook_name in self._proto_hook_loop(): - hooks_container = self.get_hooks_container_by_protocol(net, proto_name) + hooks_container = self.get_hooks_container(net, proto_name, hook_name) for hook_container in hooks_container: for hook_ops in self.get_hook_ops( @@ -270,7 +272,7 @@ class AbstractNetfilter(ABC): net_ns_id = net.ns.inum yield net_ns_id, net - def get_hooks_container_by_protocol(self, net, proto_name): + def get_hooks_container(self, net, proto_name, hook_name): """Returns the data structure used in a specific kernel implementation to store the hooks for a respective namespace and protocol. @@ -281,7 +283,7 @@ class AbstractNetfilter(ABC): yield net.nf.hooks def get_hook_ops(self, hook_container, proto_idx, hook_idx): - """Given the hook_container obtained from get_hooks_container_by_protocol(), it + """Given the hook_container obtained from get_hooks_container(), it returns an iterable of 'nf_hook_ops' elements for a corresponding protocol and hook type. @@ -338,7 +340,7 @@ class NetfilterImp_to_4_3(AbstractNetfilter): netns, net = renderers.NotAvailableValue(), renderers.NotAvailableValue() yield netns, net - def get_hooks_container_by_protocol(self, net, proto_name): + def get_hooks_container(self, net, proto_name, hook_name): nf_hooks = self.vmlinux.object_from_symbol("nf_hooks") if not nf_hooks: return @@ -508,7 +510,7 @@ class NetfilterImp_4_16_to_latest(NetfilterImp_4_14_to_4_16): and vmlinux.get_type("netns_nf").has_member("hooks_ipv4") ) - def get_hooks_container_by_protocol(self, net, proto_name): + def get_hooks_container(self, net, proto_name, hook_name): try: if proto_name == "IPV4": net_nf_hooks = net.nf.hooks_ipv4 @@ -556,20 +558,19 @@ class AbstractNetfilterNetDev(AbstractNetfilter): def subscribed_protocols(self): return ("NETDEV",) - def get_hooks_container_by_protocol(self, net, proto_name): - if proto_name != "NETDEV": - return - + def get_hooks_container(self, net, proto_name, hook_name): net_device_type = self.vmlinux.get_type("net_device") net_device_name = self.get_symbol_fullname("net_device") for net_device in net.dev_base_head.to_list(net_device_name, "dev_list"): - if net_device_type.has_member("nf_hooks_ingress"): - # CONFIG_NETFILTER_INGRESS=y - yield net_device.nf_hooks_ingress + if hook_name == "INGRESS": + if net_device_type.has_member("nf_hooks_ingress"): + # CONFIG_NETFILTER_INGRESS=y + yield net_device.nf_hooks_ingress - if net_device_type.has_member("nf_hooks_egress"): - # CONFIG_NETFILTER_EGRESS=y - yield net_device.nf_hooks_egress + elif hook_name == "EGRESS": + if net_device_type.has_member("nf_hooks_egress"): + # CONFIG_NETFILTER_EGRESS=y + yield net_device.nf_hooks_egress class NetfilterNetDevImp_4_2_to_4_9(AbstractNetfilterNetDev): From 52dcfb45f45b9442b944f124f66cb0c363b584af Mon Sep 17 00:00:00 2001 From: bbarnacle Date: Tue, 28 Nov 2023 11:20:51 -0500 Subject: [PATCH 043/979] Windows: Add regex filtering to dumpfiles In volatility2, the windows.dumpfiles plugin allows you to filter the dumped files using a regular expression. This PR adds the same functionality to volatility3. The regular expression is passed in using --regex=REGEX and all files matching REGEX will be dumped. The --ignore-case flag can be passed to make the search case-insensitive. The search is case-sensitive by default. The matching volatility2 functionality can be found here: https://github.com/volatilityfoundation/volatility/blob/a438e768194a9e05eb4d9ee 9338b881c0fa25937/volatility/plugins/dumpfiles.py#L844 Manual testing was performed on windows memory images across different windows versions to verify the expected output. --- .../framework/plugins/windows/dumpfiles.py | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index dd82d897e..4aef660da 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -4,11 +4,12 @@ import logging import ntpath +import re from typing import List, Tuple, Type, Optional, Generator from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints +from volatility3.framework.renderers import format_hints, UnreadableValue from volatility3.plugins.windows import handles from volatility3.plugins.windows import pslist @@ -53,6 +54,15 @@ class DumpFiles(interfaces.plugins.PluginInterface): description="Dump a single _FILE_OBJECT at this physical address", optional=True, ), + requirements.StringRequirement( + name="regex", description="Dump files matching REGEX", optional=True + ), + requirements.BooleanRequirement( + name="ignore-case", + description="Ignore case in pattern match", + default=False, + optional=True, + ), requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), @@ -208,6 +218,11 @@ class DumpFiles(interfaces.plugins.PluginInterface): def _generator(self, procs: List, offsets: List): kernel = self.context.modules[self.config["kernel"]] + if self.config["regex"]: + if self.config["ignore-case"]: + file_re = re.compile(self.config["regex"], re.I) + else: + file_re = re.compile(self.config["regex"]) if procs: # The handles plugin doesn't expose any staticmethod/classmethod, and it also requires stashing @@ -243,6 +258,14 @@ class DumpFiles(interfaces.plugins.PluginInterface): obj_type = entry.get_object_type(type_map, cookie) if obj_type == "File": file_obj = entry.Body.cast("_FILE_OBJECT") + + if self.config["regex"]: + name = file_obj.file_name_with_device() + if isinstance(name, UnreadableValue): + continue + if not file_re.search(name): + continue + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): @@ -272,6 +295,13 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_obj.is_valid(): continue + if self.config["regex"]: + name = file_obj.file_name_with_device() + if isinstance(name, UnreadableValue): + continue + if not file_re.search(name): + continue + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): From 62366d4f6f6b57e6e2f6920df67a88125dac9484 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 14 Jan 2024 22:37:56 +0000 Subject: [PATCH 044/979] Symbols: Add support for metadata verification of symbol tables --- volatility3/framework/constants/__init__.py | 4 +- volatility3/framework/plugins/linux/kmsg.py | 11 +- volatility3/framework/symbols/__init__.py | 49 +- volatility3/framework/symbols/intermed.py | 9 + volatility3/framework/symbols/metadata.py | 42 +- volatility3/schemas/schema-6.3.0.json | 503 ++++++++++++++++++++ 6 files changed, 613 insertions(+), 5 deletions(-) create mode 100644 volatility3/schemas/schema-6.3.0.json diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09dded076..9aaafb933 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_MINOR = 6 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d1f17bf94..d79248256 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -10,10 +10,12 @@ from typing import Generator, Iterator, List, Tuple from volatility3.framework import ( class_subclasses, constants, + exceptions, interfaces, renderers, ) from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility vollog = logging.getLogger(__name__) @@ -495,7 +497,7 @@ class Kmsg_5_10_to_(ABCKmsg): class Kmsg(interfaces.plugins.PluginInterface): """Kernel log buffer reader""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 6, 0) _version = (1, 0, 2) @@ -514,6 +516,13 @@ class Kmsg(interfaces.plugins.PluginInterface): yield (0, values) def run(self): + if not self.context.symbol_space.verify_table_versions( + "dwarf2json", lambda version, _: (not version) or version > (0, 4, 1) + ): + raise exceptions.SymbolSpaceError( + "Invalid symbol table, please ensure the ISF table produced by dwarf2json was produced using a version > 0.4.1" + ) + return renderers.TreeGrid( [ ("facility", str), diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index 10cf39cf1..49ff94f4b 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -4,9 +4,20 @@ import collections import collections.abc +import datetime import enum import logging -from typing import Any, Dict, Iterable, Iterator, TypeVar, List +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + Optional, + Tuple, + TypeVar, + List, +) from volatility3.framework import constants, exceptions, interfaces, objects @@ -113,6 +124,42 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self._resolved = {} del self._dict[key] + def verify_table_versions( + self, + producer: str, + validator: Callable[[Optional[Tuple], Optional[datetime.datetime]], bool], + tables: List[str] = None, + ) -> bool: + """Verifies the producer metadata and version of tables + + Args: + producer: String name of a table producer to have validation performed + validator: callable that takes an optional version and an optional datetime that returns False if table is invalid + + Returns: + False if an invalid table was found or True if no invalid table was found + """ + if tables is None: + tables = self._dict.keys() + for table_name in tables: + table = self._dict[table_name] + if not table.producer: + vollog.debug( + f"Symbol table {table_name} could not be validated because no producer metadata was found" + ) + continue + if table.producer.name == producer: + # Run the verification + if not validator( + table.producer.version, + table.producer.datetime, + ): + vollog.debug(f"Symbol table {table_name} does not pass validator") + return False + else: + continue + return True + ### Resolution functions class UnresolvedTemplate(objects.templates.ReferenceTemplate): diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 24e00bbd8..8c40e466a 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -183,6 +183,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): types = _construct_delegate_function("types", True) enumerations = _construct_delegate_function("enumerations", True) metadata = _construct_delegate_function("metadata", True) + producer = _construct_delegate_function("producer", True) clear_symbol_cache = _construct_delegate_function("clear_symbol_cache") get_type = _construct_delegate_function("get_type") get_symbol = _construct_delegate_function("get_symbol") @@ -372,6 +373,14 @@ class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass=ABCMeta): table.""" return None + @property + def producer(self) -> Optional["metadata.ProducerMetadata"]: + """Returns a metadata object containing information about the symbol + table.""" + return metadata.ProducerMetadata( + self._json_object.get("metadata", {}).get("producer", {}) + ) + def clear_symbol_cache(self) -> None: """Clears the symbol cache of the symbol table.""" self._symbol_cache.clear() diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 61947be69..f765e43c9 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -2,9 +2,49 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import datetime +import logging from typing import Optional, Tuple, Union -from volatility3.framework import interfaces +from volatility3.framework import constants, interfaces + +vollog = logging.getLogger(__name__) + + +class ProducerMetadata(interfaces.symbols.MetadataInterface): + """Class to handle the Producer metadata from an ISF""" + + @property + def name(self) -> Optional[str]: + return self._json_data.get("name", None) + + @property + def version(self) -> Optional[Tuple[int]]: + """Returns the version of the ISF file producer""" + version = self._json_data.get("version", None) + if not version: + return None + if all([x in "0123456789." for x in version]): + return tuple([int(x) for x in version.split(".")]) + else: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Metadata version contains unexpected characters: '{version}'", + ) + + @property + def datetime(self) -> Optional[datetime.datetime]: + """Returns a timestamp for when the file was produced""" + if "datetime" not in self._json_data: + return None + try: + timestamp = datetime.datetime.strptime( + self._json_data["datetime"], "YYYY-MM-DD" + ) + except (TypeError, ValueError): + vollog.debug("Invalid timestamp in producer information of symbol table") + return None + return timestamp class WindowsMetadata(interfaces.symbols.MetadataInterface): diff --git a/volatility3/schemas/schema-6.3.0.json b/volatility3/schemas/schema-6.3.0.json new file mode 100644 index 000000000..2cd8ebbda --- /dev/null +++ b/volatility3/schemas/schema-6.3.0.json @@ -0,0 +1,503 @@ +{ + "$schema": "http://json-schema.org/schema#", + "id": "http://volatilityfoundation.org/intermediate-format/schema", + "title": "Symbol Container", + "type": "object", + "definitions": { + "metadata_producer": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+.[0-9]+.[0-9]+$" + }, + "datetime": { + "type": "string", + "format": "date-time" + } + }, + "required":[ + "name", + "version" + ] + }, + "metadata_windows_pe": { + "type": "object", + "properties": { + "major": { + "type": "integer" + }, + "minor": { + "type": "integer" + }, + "revision": { + "type": "integer" + }, + "build": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "major", + "minor", + "revision" + ] + }, + "metadata_windows_pdb": { + "type": "object", + "properties": { + "GUID": { + "type": "string" + }, + "age": { + "type": "integer" + }, + "database": { + "type": "string" + }, + "machine_type": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "GUID", + "age", + "database", + "machine_type" + ] + }, + "metadata_windows": { + "type": "object", + "properties": { + "pe": { + "$ref": "#/definitions/metadata_windows_pe" + }, + "pdb": { + "$ref": "#/definitions/metadata_windows_pdb" + } + }, + "additionalProperties": false + }, + "metadata_nix": { + "type": "object", + "properties": { + "symbols": { + "type": "array", + "items": { + "$ref": "#/definitions/metadata_nix_item" + } + }, + "types": { + "type": "array", + "items": { + "$ref": "#/definitions/metadata_nix_item" + } + } + }, + "additionalProperties": false + }, + "metadata_format": { + "type": "string", + "pattern": "^6.[0-9]+.[0-9]+$" + }, + "metadata_nix_item": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "pattern": "^(dwarf|symtab|system-map)$" + }, + "name": { + "type": "string" + }, + "hash_type": { + "type": "string", + "pattern": "^(sha256)$" + }, + "hash_value": { + "type": "string", + "pattern": "^[a-fA-F0-9]+$" + } + }, + "additionalProperties": false + }, + "element_metadata": { + "type": "object", + "oneOf": [ + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + } + }, + "required": [ + "format" + ], + "additionalProperties": false + }, + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + }, + "windows": { + "$ref": "#/definitions/metadata_windows" + } + }, + "required": [ + "format", + "windows" + ], + "additionalProperties": false + }, + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + }, + "linux": { + "$ref": "#/definitions/metadata_nix" + } + }, + "required": [ + "format", + "linux" + ], + "additionalProperties": false + }, + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + }, + "mac": { + "$ref": "#/definitions/metadata_nix" + } + }, + "required": [ + "format", + "mac" + ], + "additionalProperties": false + } + ] + }, + "element_enum": { + "properties": { + "size": { + "type": "integer" + }, + "base": { + "type": "string" + }, + "constants": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + } + }, + "required": [ + "size", + "base", + "constants" + ], + "additionalProperties": false + }, + "element_symbol": { + "properties": { + "address": { + "type": "number" + }, + "linkage_name": { + "type": "string" + }, + "type": { + "$ref": "#/definitions/type_descriptor" + }, + "constant_data": { + "type": "string", + "media": { + "binaryEncoding": "base64", + "readOnly": true + } + } + }, + "required": [ + "address" + ], + "additionalProperties": false + }, + "element_base_type": { + "properties": { + "size": { + "type": "integer" + }, + "signed": { + "type": "boolean" + }, + "kind": { + "type": "string", + "pattern": "^(void|int|float|char|bool)$" + }, + "endian": { + "type": "string", + "pattern": "^(little|big)$" + } + }, + "required": [ + "size", + "kind", + "signed", + "endian" + ], + "additionalProperties": false + }, + "element_user_type": { + "properties": { + "kind": { + "type": "string", + "pattern": "^(struct|union|class)$" + }, + "size": { + "type": "integer" + }, + "fields": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/field" + } + } + }, + "required": [ + "kind", + "size", + "fields" + ], + "additionalProperties": false + }, + "field": { + "properties": { + "type": { + "$ref": "#/definitions/type_descriptor" + }, + "offset": { + "type": "integer" + }, + "anonymous": { + "type": "boolean" + } + }, + "required": [ + "type", + "offset" + ], + "additionalProperties": false + }, + "type_descriptor": { + "oneOf": [ + { + "$ref": "#/definitions/type_pointer" + }, + { + "$ref": "#/definitions/type_base" + }, + { + "$ref": "#/definitions/type_array" + }, + { + "$ref": "#/definitions/type_struct" + }, + { + "$ref": "#/definitions/type_enum" + }, + { + "$ref": "#/definitions/type_function" + }, + { + "$ref": "#/definitions/type_bitfield" + } + ] + }, + "type_pointer": { + "properties": { + "kind": { + "type": "string", + "pattern": "^pointer$" + }, + "base": { + "type": "string" + }, + "subtype": { + "$ref": "#/definitions/type_descriptor" + } + }, + "required": [ + "kind", + "subtype" + ], + "additionalProperties": false + }, + "type_base": { + "properties": { + "kind": { + "type": "string", + "pattern": "^base$" + }, + "name": { + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "additionalProperties": false + }, + "type_array": { + "properties": { + "kind": { + "type": "string", + "pattern": "^array$" + }, + "subtype": { + "$ref": "#/definitions/type_descriptor" + }, + "count": { + "type": "integer" + } + }, + "required": [ + "kind", + "subtype", + "count" + ], + "additionalProperties": false + }, + "type_struct": { + "properties": { + "kind": { + "type": "string", + "pattern": "^(struct|class|union)$" + }, + "name": { + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "additionalProperties": false + }, + "type_enum": { + "properties": { + "kind": { + "type": "string", + "pattern": "^enum$" + }, + "name": { + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "additionalProperties": false + }, + "type_function": { + "properties": { + "kind": { + "type": "string", + "pattern": "^function$" + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + "type_bitfield": { + "properties": { + "kind": { + "type": "string", + "pattern": "^bitfield$" + }, + "bit_position": { + "type": "integer" + }, + "bit_length": { + "type": "integer" + }, + "type": { + "oneOf": [ + { + "$ref": "#/definitions/type_base" + }, + { + "$ref": "#/definitions/type_enum" + } + ] + } + }, + "required": [ + "kind", + "bit_position", + "bit_length", + "type" + ], + "additionalProperties": false + } + }, + "properties": { + "metadata": { + "$ref": "#/definitions/element_metadata" + }, + "base_types": { + "additionalProperties": { + "$ref": "#/definitions/element_base_type" + } + }, + "user_types": { + "additionalProperties": { + "$ref": "#/definitions/element_user_type" + } + }, + "enums": { + "additionalProperties": { + "$ref": "#/definitions/element_enum" + } + }, + "symbols": { + "additionalProperties": { + "$ref": "#/definitions/element_symbol" + } + } + }, + "required": [ + "metadata", + "base_types", + "user_types", + "enums", + "symbols" + ], + "additionalProperties": false +} From 7c8f4ca4ac5c23da3afa1d0ec6404c3fada69a27 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 14 Jan 2024 22:44:29 +0000 Subject: [PATCH 045/979] Schemas: As of 6.3.0 we require a producer metadata, not just format --- volatility3/schemas/schema-6.3.0.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/schemas/schema-6.3.0.json b/volatility3/schemas/schema-6.3.0.json index 2cd8ebbda..ba6e970bb 100644 --- a/volatility3/schemas/schema-6.3.0.json +++ b/volatility3/schemas/schema-6.3.0.json @@ -139,7 +139,8 @@ } }, "required": [ - "format" + "format", + "producer" ], "additionalProperties": false }, @@ -157,6 +158,7 @@ }, "required": [ "format", + "producer", "windows" ], "additionalProperties": false @@ -175,6 +177,7 @@ }, "required": [ "format", + "producer", "linux" ], "additionalProperties": false @@ -193,6 +196,7 @@ }, "required": [ "format", + "producer", "mac" ], "additionalProperties": false From 269058021dc171c7b46a6a2cb20d78d646b9f8fd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jan 2024 20:34:39 +0000 Subject: [PATCH 046/979] Plugins: Fix up kmsg code issues --- volatility3/framework/plugins/linux/kmsg.py | 1 - volatility3/framework/symbols/metadata.py | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d79248256..dd707a7ff 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -15,7 +15,6 @@ from volatility3.framework import ( renderers, ) from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility vollog = logging.getLogger(__name__) diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index f765e43c9..149829371 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -26,11 +26,11 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): return None if all([x in "0123456789." for x in version]): return tuple([int(x) for x in version.split(".")]) - else: - vollog.log( - constants.LOGLEVEL_VVVV, - f"Metadata version contains unexpected characters: '{version}'", - ) + vollog.log( + constants.LOGLEVEL_VVVV, + f"Metadata version contains unexpected characters: '{version}'", + ) + return None @property def datetime(self) -> Optional[datetime.datetime]: From 4e89040dadfacac01ba0c0a3727969a7b0fb0047 Mon Sep 17 00:00:00 2001 From: hsarkey Date: Sun, 14 Jan 2024 13:20:56 -0500 Subject: [PATCH 047/979] Updated changes to the windows.malfind plugin. Eliminated --refined as a command line option and instead added an additional column called "Notes" to provide info on common headers. --- .../framework/plugins/windows/malfind.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 1c73fdf1c..a14f8889d 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -46,12 +46,6 @@ class Malfind(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) ), - requirements.BooleanRequirement( - name="refined", - description="Refine the output. Only show regions with an MZ header or that start with well known opcode combinations (i.e. PUSH EBP). WARNING: This can cause you to overlook regions with wiped headers or shell code blocks starting with NOP sleds, etc.. However, it will in general result in less noisy output.", - default=False, - optional=True, - ), ] @classmethod @@ -144,9 +138,6 @@ class Malfind(interfaces.plugins.PluginInterface): yield vad, data def _generator(self, procs): - # set refined criteria - refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] - # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] @@ -155,14 +146,21 @@ class Malfind(interfaces.plugins.PluginInterface): ) for proc in procs: + # by default, "Notes" column will be set to none + notes = "None" process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( self.context, kernel.layer_name, kernel.symbol_table_name, proc ): - # check if refined option was passed - if self.config["refined"] and data[0:2] not in refined_criteria: - continue + # Check for unique headers and update "Notes" column if criteria is met + match data[0:2]: + case b"MZ" | b"\x55\x8B": + notes = "MZ header" + case b"\x55\x8B": + notes = "PE header" + case b"\x55\x48" | b"\x55\x89": + notes = "Function prologue" # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 if is_32bit_arch or proc.get_is_wow64(): @@ -209,6 +207,7 @@ class Malfind(interfaces.plugins.PluginInterface): vad.get_commit_charge(), vad.get_private_memory(), file_output, + notes, format_hints.HexBytes(data), disasm, ), @@ -229,6 +228,7 @@ class Malfind(interfaces.plugins.PluginInterface): ("CommitCharge", int), ("PrivateMemory", int), ("File output", str), + ("Notes", str), ("Hexdump", format_hints.HexBytes), ("Disasm", interfaces.renderers.Disassembly), ], From e6e138f12d06509da829008867392bc2346f18f9 Mon Sep 17 00:00:00 2001 From: hsarkey Date: Wed, 17 Jan 2024 17:24:26 -0500 Subject: [PATCH 048/979] Updated windows.malfind to have a "Notes" column which will indicate if a process meets a "refined criteria", meaning it has a common header type (MZ, PE, or a function prologue). --- volatility3/framework/plugins/windows/malfind.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index a14f8889d..e5a842611 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -140,6 +140,9 @@ class Malfind(interfaces.plugins.PluginInterface): def _generator(self, procs): # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] + + # set refined criteria to know when to add to "Notes" column + refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] is_32bit_arch = not symbols.symbol_table_is_64bit( self.context, kernel.symbol_table_name @@ -154,12 +157,12 @@ class Malfind(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, proc ): # Check for unique headers and update "Notes" column if criteria is met - match data[0:2]: - case b"MZ" | b"\x55\x8B": + if data[0:2] in refined_criteria: + if data[0:2] == b"MZ": notes = "MZ header" - case b"\x55\x8B": + elif data[0:2] == b"\x55\x8B": notes = "PE header" - case b"\x55\x48" | b"\x55\x89": + else: notes = "Function prologue" # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 From ef36cb5a6c6a30f0bc986cc275c83a734b9d535b Mon Sep 17 00:00:00 2001 From: hsarkey Date: Wed, 17 Jan 2024 17:51:27 -0500 Subject: [PATCH 049/979] Updated windows.malfind to have a "Notes" column to indicate if a process meets "refined" criteria, which includes common headers like MZ,PE or function prologues. Fixed black formatting issue. --- volatility3/framework/plugins/windows/malfind.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index e5a842611..284144077 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -140,7 +140,7 @@ class Malfind(interfaces.plugins.PluginInterface): def _generator(self, procs): # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] - + # set refined criteria to know when to add to "Notes" column refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] @@ -150,7 +150,7 @@ class Malfind(interfaces.plugins.PluginInterface): for proc in procs: # by default, "Notes" column will be set to none - notes = "None" + notes = "None" process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( From a50ebb6014ff11ea165454e0a08b967d01cce9e8 Mon Sep 17 00:00:00 2001 From: Calvin Kusek Date: Tue, 2 Jan 2024 12:00:41 -0500 Subject: [PATCH 050/979] Windows: Display additional process info for windows.pstree --- .../framework/plugins/windows/pstree.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/pstree.py b/volatility3/framework/plugins/windows/pstree.py index a39fe7485..2be96277c 100644 --- a/volatility3/framework/plugins/windows/pstree.py +++ b/volatility3/framework/plugins/windows/pstree.py @@ -5,7 +5,7 @@ import datetime import logging from typing import Callable, Dict, Set, Tuple -from volatility3.framework import objects, interfaces, renderers +from volatility3.framework import objects, interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import pslist @@ -132,6 +132,25 @@ class PsTree(interfaces.plugins.PluginInterface): proc.get_exit_time(), ) + try: + audit = proc.SeAuditProcessCreationInfo.ImageFileName.Name + # If 'audit' is set to the empty string, display NotAvailableValue + row += (audit.get_string() or renderers.NotAvailableValue(),) + except exceptions.InvalidAddressException: + row += (renderers.NotAvailableValue(),) + + try: + process_params = proc.get_peb().ProcessParameters + row += ( + process_params.CommandLine.get_string(), + process_params.ImagePathName.get_string(), + ) + except exceptions.InvalidAddressException: + row += ( + renderers.NotAvailableValue(), + renderers.NotAvailableValue(), + ) + yield (self._levels[pid] - 1, row) for child_pid in self._children.get(pid, []): yield from yield_processes( @@ -161,6 +180,9 @@ class PsTree(interfaces.plugins.PluginInterface): ("Wow64", bool), ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime), + ("Audit", str), + ("Cmd", str), + ("Path", str), ], self._generator( filter_func=pslist.PsList.create_pid_filter( From ae5375a622c82177f6b29adfb8b0f410ecb03059 Mon Sep 17 00:00:00 2001 From: Brandon Barnacle Date: Wed, 17 Jan 2024 09:49:39 -0500 Subject: [PATCH 051/979] PR comment changes Change the --regex flag to --filter. Add a check so that --filter cannot be used with --physaddr or --virtaddr. Change self.config["filter"] check to check if file_re has been set. --- .../framework/plugins/windows/dumpfiles.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 4aef660da..8865007d8 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -55,11 +55,11 @@ class DumpFiles(interfaces.plugins.PluginInterface): optional=True, ), requirements.StringRequirement( - name="regex", description="Dump files matching REGEX", optional=True + name="filter", description="Dump files matching regular expression FILTER", optional=True ), requirements.BooleanRequirement( name="ignore-case", - description="Ignore case in pattern match", + description="Ignore case in filter match", default=False, optional=True, ), @@ -218,11 +218,11 @@ class DumpFiles(interfaces.plugins.PluginInterface): def _generator(self, procs: List, offsets: List): kernel = self.context.modules[self.config["kernel"]] - if self.config["regex"]: - if self.config["ignore-case"]: - file_re = re.compile(self.config["regex"], re.I) - else: - file_re = re.compile(self.config["regex"]) + file_re = None + if self.config["filter"]: + flags = re.I if self.config["ignore-case"] else 0 + file_re = re.compile(self.config["filter"], flags) + if procs: # The handles plugin doesn't expose any staticmethod/classmethod, and it also requires stashing @@ -259,7 +259,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if obj_type == "File": file_obj = entry.Body.cast("_FILE_OBJECT") - if self.config["regex"]: + if file_re: name = file_obj.file_name_with_device() if isinstance(name, UnreadableValue): continue @@ -295,7 +295,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_obj.is_valid(): continue - if self.config["regex"]: + if file_re: name = file_obj.file_name_with_device() if isinstance(name, UnreadableValue): continue @@ -345,6 +345,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): procs = list() kernel = self.context.modules[self.config["kernel"]] + if self.config["filter"] and (self.config["virtaddr"] or self.config["physaddr"]): + raise ValueError("Cannot use filter flag with an address flag") + if self.config.get("virtaddr", None) is not None: offsets.append((self.config["virtaddr"], True)) elif self.config.get("physaddr", None) is not None: From 4dc6f637b055f912bafaa9f6079759539277d60f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Jan 2024 23:50:45 +0000 Subject: [PATCH 052/979] Core: Apply black 24.1.0 to the whole codebase --- volatility3/cli/text_renderer.py | 8 +++--- volatility3/framework/__init__.py | 6 ++--- volatility3/framework/automagic/mac.py | 6 ++--- volatility3/framework/automagic/module.py | 6 ++--- .../framework/automagic/symbol_finder.py | 12 ++++----- volatility3/framework/automagic/windows.py | 20 +++++++------- .../framework/configuration/requirements.py | 6 ++--- volatility3/framework/layers/avml.py | 6 ++--- volatility3/framework/layers/elf.py | 6 ++--- volatility3/framework/layers/intel.py | 6 ++--- volatility3/framework/layers/lime.py | 6 ++--- volatility3/framework/layers/qemu.py | 6 ++--- volatility3/framework/layers/xen.py | 6 ++--- volatility3/framework/objects/__init__.py | 6 ++--- volatility3/framework/plugins/mac/pslist.py | 4 +-- .../framework/plugins/windows/crashinfo.py | 6 ++--- .../framework/plugins/windows/mftscan.py | 1 - .../framework/plugins/windows/netscan.py | 10 ++++--- .../plugins/windows/registry/printkey.py | 6 ++--- volatility3/framework/renderers/conversion.py | 6 ++--- volatility3/framework/symbols/__init__.py | 26 +++++++++---------- .../symbols/windows/extensions/__init__.py | 6 ++--- volatility3/plugins/windows/statistics.py | 10 +++---- 23 files changed, 88 insertions(+), 93 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ffb8d516b..6e58ee68d 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -389,9 +389,11 @@ class JsonRenderer(CLIRenderer): interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), - datetime.datetime: lambda x: x.isoformat() - if not isinstance(x, interfaces.renderers.BaseAbsentValue) - else None, + datetime.datetime: lambda x: ( + x.isoformat() + if not isinstance(x, interfaces.renderers.BaseAbsentValue) + else None + ), "default": lambda x: x, } diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 9c17846a8..1565b2267 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -206,9 +206,9 @@ def _zipwalk(path: str): if not file.is_dir(): dirlist = zip_results.get(os.path.dirname(file.filename), []) dirlist.append(os.path.basename(file.filename)) - zip_results[ - os.path.join(path, os.path.dirname(file.filename)) - ] = dirlist + zip_results[os.path.join(path, os.path.dirname(file.filename))] = ( + dirlist + ) for value in zip_results: yield value, zip_results[value] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index aa75fbc3d..e51753139 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -138,9 +138,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): config_path = join("automagic", "MacIntelHelper", new_layer_name) context.config[join(config_path, "memory_layer")] = layer_name context.config[join(config_path, "page_map_offset")] = dtb - context.config[ - join(config_path, MacSymbolFinder.banner_config_key) - ] = str(banner, "latin-1") + context.config[join(config_path, MacSymbolFinder.banner_config_key)] = ( + str(banner, "latin-1") + ) new_layer = intel.Intel32e( context, diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index ee56a040c..ff13db905 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -34,9 +34,9 @@ class KernelModule(interfaces.automagic.AutomagicInterface): return None # The requirement is unfulfilled and is a ModuleRequirement - context.config[ - interfaces.configuration.path_join(new_config_path, "class") - ] = "volatility3.framework.contexts.Module" + context.config[interfaces.configuration.path_join(new_config_path, "class")] = ( + "volatility3.framework.contexts.Module" + ) for req in requirement.requirements: if ( diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index bf1c8ff16..21e594549 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -150,12 +150,12 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join - context.config[ - path_join(config_path, requirement.name, "class") - ] = clazz - context.config[ - path_join(config_path, requirement.name, "isf_url") - ] = isf_path + context.config[path_join(config_path, requirement.name, "class")] = ( + clazz + ) + context.config[path_join(config_path, requirement.name, "isf_url")] = ( + isf_path + ) context.config[ path_join(config_path, requirement.name, "symbol_mask") ] = layer.address_mask diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index a8530829b..52296f5ad 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -402,19 +402,19 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): if swap_location: context.config[current_layer_path] = current_layer_name try: - context.config[ - layer_loc_path - ] = requirements.URIRequirement.location_from_file( - swap_location + context.config[layer_loc_path] = ( + requirements.URIRequirement.location_from_file( + swap_location + ) ) except ValueError: vollog.warning( f"Volatility swap_location {swap_location} could not be validated - swap layer disabled" ) continue - context.config[ - layer_class_path - ] = "volatility3.framework.layers.physical.FileLayer" + context.config[layer_class_path] = ( + "volatility3.framework.layers.physical.FileLayer" + ) # Add the requirement new_req = requirements.TranslationLayerRequirement( @@ -424,9 +424,9 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): ) swap_req.add_requirement(new_req) - context.config[ - path_join(swap_sub_config, "number_of_elements") - ] = counter + context.config[path_join(swap_sub_config, "number_of_elements")] = ( + counter + ) context.config[swap_sub_config] = True swap_req.construct(context, swap_config) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index abdffdbe4..1c0622574 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -550,9 +550,9 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} - context.config[ - interfaces.configuration.path_join(config_path, self.name) - ] = True + context.config[interfaces.configuration.path_join(config_path, self.name)] = ( + True + ) return {} @classmethod diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index c825464cc..2e5572192 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -224,7 +224,7 @@ class AVMLStacker(interfaces.automagic.StackerLayerInterface): except exceptions.LayerException: return None new_name = context.layers.free_layer_name("AVMLLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return AVMLLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index a10d36592..b2fd6d4d1 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -115,9 +115,9 @@ class Elf64Stacker(interfaces.automagic.StackerLayerInterface): vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("Elf64Layer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) try: return Elf64Layer(context, new_name, new_name) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 7d3b86a12..ae477854d 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -277,9 +277,9 @@ class Intel(linear.LinearlyMappedLayer): This allows translation layers to provide maps of contiguous regions in one layer """ - stashed_offset = ( - stashed_mapped_offset - ) = stashed_size = stashed_mapped_size = stashed_map_layer = None + stashed_offset = stashed_mapped_offset = stashed_size = stashed_mapped_size = ( + stashed_map_layer + ) = None for offset, size, mapped_offset, mapped_size, map_layer in self._mapping( offset, length, ignore_errors ): diff --git a/volatility3/framework/layers/lime.py b/volatility3/framework/layers/lime.py index 28d646640..8b93932ab 100644 --- a/volatility3/framework/layers/lime.py +++ b/volatility3/framework/layers/lime.py @@ -104,7 +104,7 @@ class LimeStacker(interfaces.automagic.StackerLayerInterface): except LimeFormatException: return None new_name = context.layers.free_layer_name("LimeLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return LimeLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 501b8655e..ff483291c 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -486,9 +486,9 @@ class QemuStacker(interfaces.automagic.StackerLayerInterface): except exceptions.LayerException: return None new_name = context.layers.free_layer_name("QemuSuspendLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) layer = QemuSuspendLayer(context, new_name, new_name) cls.stacker_slow_warning() return layer diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index f7881a091..927b30430 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -173,8 +173,8 @@ class XenCoreDumpStacker(elf.Elf64Stacker): vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("XenCoreDumpLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return XenCoreDumpLayer(context, new_name, new_name) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 831856e3d..316a30bec 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -768,12 +768,10 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): raise IndexError(f"Member not present in array template: {child}") @overload - def __getitem__(self, i: int) -> interfaces.objects.Template: - ... + def __getitem__(self, i: int) -> interfaces.objects.Template: ... @overload - def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: - ... + def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: ... def __getitem__(self, i): """Returns the i-th item from the array.""" diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 88045a277..9b570f3f9 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -49,9 +49,7 @@ class PsList(interfaces.plugins.PluginInterface): ] @classmethod - def get_list_tasks( - cls, method: str - ) -> Callable[ + def get_list_tasks(cls, method: str) -> Callable[ [interfaces.context.ContextInterface, str, Callable[[int], bool]], Iterable[interfaces.objects.ObjectInterface], ]: diff --git a/volatility3/framework/plugins/windows/crashinfo.py b/volatility3/framework/plugins/windows/crashinfo.py index 4ecd85087..862eb6080 100644 --- a/volatility3/framework/plugins/windows/crashinfo.py +++ b/volatility3/framework/plugins/windows/crashinfo.py @@ -46,9 +46,9 @@ class Crashinfo(interfaces.plugins.PluginInterface): bitmap_size = format_hints.Hex(summary_header.BitmapSize) bitmap_pages = format_hints.Hex(summary_header.Pages) else: - bitmap_header_size = ( - bitmap_size - ) = bitmap_pages = renderers.NotApplicableValue() + bitmap_header_size = bitmap_size = bitmap_pages = ( + renderers.NotApplicableValue() + ) yield ( 0, diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 4298b4e4e..91a2e9152 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -175,7 +175,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): class ADS(interfaces.plugins.PluginInterface): - """Scans for Alternate Data Stream""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index d0bbd5cbd..62ead3ab7 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -487,10 +487,12 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if not isinstance(row_data[9], datetime.datetime): continue row_data = [ - "N/A" - if isinstance(i, renderers.UnreadableValue) - or isinstance(i, renderers.UnparsableValue) - else i + ( + "N/A" + if isinstance(i, renderers.UnreadableValue) + or isinstance(i, renderers.UnparsableValue) + else i + ) for i in row_data ] description = ( diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index e248c19bc..180f8f9d9 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -193,9 +193,9 @@ class PrintKey(interfaces.plugins.PluginInterface): vollog.debug( "Couldn't read registry value type, so data is unreadable" ) - value_data: Union[ - interfaces.renderers.BaseAbsentValue, bytes - ] = renderers.UnreadableValue() + value_data: Union[interfaces.renderers.BaseAbsentValue, bytes] = ( + renderers.UnreadableValue() + ) else: try: value_data = node.decode_data() diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index bf7da9ecb..bb18fcc8a 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -28,9 +28,9 @@ def wintime_to_datetime( def unixtime_to_datetime( unixtime: int, ) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: - ret: Union[ - interfaces.renderers.BaseAbsentValue, datetime.datetime - ] = renderers.UnparsableValue() + ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = ( + renderers.UnparsableValue() + ) if unixtime > 0: with contextlib.suppress(ValueError): diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index 10cf39cf1..d1e7a104d 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -35,9 +35,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): def __init__(self) -> None: super().__init__() - self._dict: Dict[ - str, interfaces.symbols.BaseSymbolTableInterface - ] = collections.OrderedDict() + self._dict: Dict[str, interfaces.symbols.BaseSymbolTableInterface] = ( + collections.OrderedDict() + ) # Permanently cache all resolved symbols self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} @@ -73,9 +73,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self, offset: int, size: int = 0, table_name: str = None ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" - table_list: Iterable[ - interfaces.symbols.BaseSymbolTableInterface - ] = self._dict.values() + table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = ( + self._dict.values() + ) if table_name is not None: if table_name in self._dict: table_list = [self._dict[table_name]] @@ -179,15 +179,15 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): if child.vol.type_name not in self._resolved: traverse_list.append(child.vol.type_name) try: - self._resolved[ - child.vol.type_name - ] = self._weak_resolve( - SymbolType.TYPE, child.vol.type_name + self._resolved[child.vol.type_name] = ( + self._weak_resolve( + SymbolType.TYPE, child.vol.type_name + ) ) except exceptions.SymbolError: - self._resolved[ - child.vol.type_name - ] = self.UnresolvedTemplate(child.vol.type_name) + self._resolved[child.vol.type_name] = ( + self.UnresolvedTemplate(child.vol.type_name) + ) # Stash the replacement replacements.add((traverser, child)) elif child.children: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d435851d7..846e5bd90 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -452,9 +452,9 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): ].is_valid(self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name: Union[ - str, interfaces.renderers.BaseAbsentValue - ] = renderers.UnreadableValue() + name: Union[str, interfaces.renderers.BaseAbsentValue] = ( + renderers.UnreadableValue() + ) # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 9915312e3..7f56b75f8 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -31,13 +31,9 @@ class Statistics(plugins.PluginInterface): # Do mass mapping and determine the number of different layers and how many pages go to each one layer = self.context.layers[self.config["primary"]] - page_count = ( - swap_count - ) = ( - invalid_page_count - ) = ( - large_page_count - ) = large_swap_count = large_invalid_count = other_invalid = 0 + page_count = swap_count = invalid_page_count = large_page_count = ( + large_swap_count + ) = large_invalid_count = other_invalid = 0 if isinstance(layer, intel.Intel): page_addr = 0 From 497d291ef4393e2580052a3dfddbef10e4dc2338 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 29 Jan 2024 00:07:40 +0000 Subject: [PATCH 053/979] Windows: Fix black on dumpfiles --- volatility3/framework/plugins/windows/dumpfiles.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 8865007d8..48539c752 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -55,7 +55,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): optional=True, ), requirements.StringRequirement( - name="filter", description="Dump files matching regular expression FILTER", optional=True + name="filter", + description="Dump files matching regular expression FILTER", + optional=True, ), requirements.BooleanRequirement( name="ignore-case", @@ -223,7 +225,6 @@ class DumpFiles(interfaces.plugins.PluginInterface): flags = re.I if self.config["ignore-case"] else 0 file_re = re.compile(self.config["filter"], flags) - if procs: # The handles plugin doesn't expose any staticmethod/classmethod, and it also requires stashing # private variables, so we need an instance (for now, anyway). We _could_ call Handles._generator() @@ -345,7 +346,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): procs = list() kernel = self.context.modules[self.config["kernel"]] - if self.config["filter"] and (self.config["virtaddr"] or self.config["physaddr"]): + if self.config["filter"] and ( + self.config["virtaddr"] or self.config["physaddr"] + ): raise ValueError("Cannot use filter flag with an address flag") if self.config.get("virtaddr", None) is not None: From 837d3b7b9fce65d5ef6543b6db9ecbad5092d4b0 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 14 Dec 2023 13:47:00 -0600 Subject: [PATCH 054/979] Windows: Add parsing of service binary/dll Parses the binary or dll associated with each service from the Windows SYSTEM hive and includes it in the output for each service. --- .../framework/plugins/windows/svcscan.py | 139 +++++++++++++++++- 1 file changed, 133 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 60562915e..720f745d8 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -4,20 +4,30 @@ import logging import os -from typing import List +from typing import Dict, List, NamedTuple, Union -from volatility3.framework import interfaces, renderers, constants, symbols, exceptions +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions from volatility3.framework.symbols.windows.extensions import services -from volatility3.plugins.windows import poolscanner, vadyarascan, pslist +from volatility3.plugins.windows import poolscanner, pslist, vadyarascan +from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) +ServiceBinaryInfo = NamedTuple( + "ServiceBinaryInfo", + [ + ("dll", Union[str, interfaces.renderers.BaseAbsentValue]), + ("binary", Union[str, interfaces.renderers.BaseAbsentValue]), + ], +) + + class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" @@ -42,10 +52,16 @@ class SvcScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="vadyarascan", plugin=vadyarascan.VadYaraScan, version=(1, 0, 0) ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), ] @staticmethod - def get_record_tuple(service_record: interfaces.objects.ObjectInterface): + def get_record_tuple( + service_record: interfaces.objects.ObjectInterface, + binary_info: ServiceBinaryInfo, + ): return ( format_hints.Hex(service_record.vol.offset), service_record.Order, @@ -56,6 +72,8 @@ class SvcScan(interfaces.plugins.PluginInterface): service_record.get_name(), service_record.get_display(), service_record.get_binary(), + binary_info.binary, + binary_info.dll, ) @staticmethod @@ -150,6 +168,86 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types=native_types, ) + def _get_service_key(self, kernel): + for hive in hivelist.HiveList.list_hives( + context=self.context, + base_config_path=self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_string="machine\\system", + hive_offsets=None, + ): + # Get ControlSet\Services. + try: + return hive.get_key(r"CurrentControlSet\Services") + except (KeyError, exceptions.InvalidAddressException): + try: + return hive.get_key(r"ControlSet001\Services") + except (KeyError, exceptions.InvalidAddressException): + pass + + return None + + @staticmethod + def _get_service_dll( + service_key, + ) -> Union[str, interfaces.renderers.BaseAbsentValue]: + try: + param_key = next( + key + for key in service_key.get_subkeys() + if key.get_name() == "Parameters" + ) + return ( + next( + val + for val in param_key.get_values() + if val.get_name() == "ServiceDll" + ) + .decode_data() + .decode("utf-16") + .rstrip("\x00") + ) + + except UnicodeDecodeError: + return renderers.UnparsableValue() + except StopIteration: + return renderers.UnreadableValue() + + @staticmethod + def _get_service_binary( + service_key, + ) -> Union[str, interfaces.renderers.BaseAbsentValue]: + try: + return ( + next( + val + for val in service_key.get_values() + if val.get_name() == "ImagePath" + ) + .decode_data() + .decode("utf-16") + .rstrip("\x00") + ) + + except UnicodeDecodeError: + return renderers.UnparsableValue() + except StopIteration: + return renderers.UnreadableValue() + + @staticmethod + def _get_service_binary_map( + services_key: interfaces.objects.ObjectInterface, + ) -> Dict[str, ServiceBinaryInfo]: + services = services_key.get_subkeys() + return { + service_key.get_name(): ServiceBinaryInfo( + SvcScan._get_service_dll(service_key), + SvcScan._get_service_binary(service_key), + ) + for service_key in services + } + def _generator(self): kernel = self.context.modules[self.config["kernel"]] @@ -157,6 +255,15 @@ class SvcScan(interfaces.plugins.PluginInterface): self.context, kernel.symbol_table_name, self.config_path ) + # Building the dictionary ahead of time is much better for performance + # vs looking up each service's DLL individually. + services_key = self._get_service_key(kernel) + service_binary_dll_map = ( + self._get_service_binary_map(services_key) + if services_key is not None + else {} + ) + relative_tag_offset = self.context.symbol_space.get_type( service_table_name + constants.BANG + "_SERVICE_RECORD" ).relative_child_offset("Tag") @@ -209,7 +316,16 @@ class SvcScan(interfaces.plugins.PluginInterface): if not service_record.is_valid(): continue - yield (0, self.get_record_tuple(service_record)) + service_info = service_binary_dll_map.get( + service_record.get_name(), + ServiceBinaryInfo( + renderers.UnreadableValue(), renderers.UnreadableValue() + ), + ) + yield ( + 0, + self.get_record_tuple(service_record, service_info), + ) else: service_header = self.context.object( service_table_name + constants.BANG + "_SERVICE_HEADER", @@ -227,7 +343,16 @@ class SvcScan(interfaces.plugins.PluginInterface): if service_record in seen: break seen.append(service_record) - yield (0, self.get_record_tuple(service_record)) + service_info = service_binary_dll_map.get( + service_record.get_name(), + ServiceBinaryInfo( + renderers.UnreadableValue(), renderers.UnreadableValue() + ), + ) + yield ( + 0, + self.get_record_tuple(service_record, service_info), + ) def run(self): return renderers.TreeGrid( @@ -241,6 +366,8 @@ class SvcScan(interfaces.plugins.PluginInterface): ("Name", str), ("Display", str), ("Binary", str), + ("Binary (Registry)", str), + ("Dll", str), ], self._generator(), ) From 77b01106356ab30ec531799a66d38db3951033c4 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 14 Dec 2023 14:16:40 -0600 Subject: [PATCH 055/979] Windows: Fixes svc symbols and OS version detection There were some changes made to the service structures between certain versions of Windows that caused the PID/binary path to fail to parse across quite a few samples. This adds the appropriate version detection logic and updated symbol files to ensure that services are parsed correctly across Windows versions. --- volatility3/framework/interfaces/context.py | 2 +- .../framework/plugins/windows/svcscan.py | 50 +++- .../services/services-win10-17763-x86.json | 248 +++++++++++++++++ .../services/services-win10-18362-x64.json | 255 ++++++++++++++++++ .../services/services-win10-18362-x86.json | 248 +++++++++++++++++ .../services/services-win10-19041-x64.json | 255 ++++++++++++++++++ .../services/services-win10-19041-x86.json | 248 +++++++++++++++++ .../services/services-win10-25398-x64.json | 255 ++++++++++++++++++ .../framework/symbols/windows/versions.py | 34 +++ 9 files changed, 1584 insertions(+), 11 deletions(-) create mode 100644 volatility3/framework/symbols/windows/services/services-win10-17763-x86.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-18362-x64.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-18362-x86.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-19041-x64.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-19041-x86.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-25398-x64.json diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 29cb41379..a95f2b464 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -87,7 +87,7 @@ class ContextInterface(metaclass=ABCMeta): offset: int, native_layer_name: str = None, **arguments, - ): + ) -> "interfaces.objects.ObjectInterface": """Object factory, takes a context, symbol, offset and optional layer_name. diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 720f745d8..74fe2d802 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -106,26 +106,46 @@ class SvcScan(interfaces.plugins.PluginInterface): and is_64bit ): symbol_filename = "services-xp-2003-x64" + elif ( + versions.is_win10_25398_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win10-25398-x64" + elif ( + versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win10-19041-x64" + elif ( + versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win10-19041-x86" + elif ( + versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win10-18362-x64" + elif ( + versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win10-18362-x86" elif ( versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) and is_64bit ): symbol_filename = "services-win10-16299-x64" + elif ( + versions.is_win10_17763_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win10-17763-x86" elif ( versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) and not is_64bit ): symbol_filename = "services-win10-16299-x86" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win8-x64" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win8-x86" elif ( versions.is_win10_15063(context=context, symbol_table=symbol_table) and is_64bit @@ -136,6 +156,16 @@ class SvcScan(interfaces.plugins.PluginInterface): and not is_64bit ): symbol_filename = "services-win10-15063-x86" + elif ( + versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win8-x64" + elif ( + versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win8-x86" elif ( versions.is_windows_8_or_later(context=context, symbol_table=symbol_table) and is_64bit diff --git a/volatility3/framework/symbols/windows/services/services-win10-17763-x86.json b/volatility3/framework/symbols/windows/services/services-win10-17763-x86.json new file mode 100644 index 000000000..8f2854721 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-17763-x86.json @@ -0,0 +1,248 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 4 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 12 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 20 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 12 + }, + "_SERVICE_RECORD": { + "fields": { + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 48 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 160 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 24 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 56 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 44 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 160 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 52 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 156 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-18362-x64.json b/volatility3/framework/symbols/windows/services/services-win10-18362-x64.json new file mode 100644 index 000000000..a6a80c1d3 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-18362-x64.json @@ -0,0 +1,255 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 16 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 24 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 40 + } + }, + "kind": "struct", + "size": 40 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_RECORD": { + "fields": { + "ServiceList": { + "type": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + }, + "offset": 0 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 32 + }, + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 64 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 240 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 36 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 76 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 56 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 240 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 72 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 248 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-18362-x86.json b/volatility3/framework/symbols/windows/services/services-win10-18362-x86.json new file mode 100644 index 000000000..4684dfe5b --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-18362-x86.json @@ -0,0 +1,248 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 4 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 12 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 20 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 12 + }, + "_SERVICE_RECORD": { + "fields": { + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 48 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 164 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 24 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 56 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 44 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 164 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 52 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 156 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-19041-x64.json b/volatility3/framework/symbols/windows/services/services-win10-19041-x64.json new file mode 100644 index 000000000..e44dbbd37 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-19041-x64.json @@ -0,0 +1,255 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 16 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 24 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 40 + } + }, + "kind": "struct", + "size": 40 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_RECORD": { + "fields": { + "ServiceList": { + "type": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + }, + "offset": 0 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 32 + }, + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 64 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 296 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 36 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 76 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 56 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 296 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 72 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 296 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "David McDonald", + "datetime": "2023-11-16T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-19041-x86.json b/volatility3/framework/symbols/windows/services/services-win10-19041-x86.json new file mode 100644 index 000000000..cc5ed9a73 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-19041-x86.json @@ -0,0 +1,248 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 4 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 12 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 20 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 12 + }, + "_SERVICE_RECORD": { + "fields": { + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 48 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 192 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 24 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 56 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 44 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 192 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 52 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 192 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-25398-x64.json b/volatility3/framework/symbols/windows/services/services-win10-25398-x64.json new file mode 100644 index 000000000..cd29abc43 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-25398-x64.json @@ -0,0 +1,255 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 16 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 24 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 40 + } + }, + "kind": "struct", + "size": 40 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_RECORD": { + "fields": { + "ServiceList": { + "type": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + }, + "offset": 0 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 32 + }, + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 64 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 336 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 36 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 84 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 56 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 296 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 80 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 336 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "David McDonald", + "datetime": "2023-11-16T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/versions.py b/volatility3/framework/symbols/windows/versions.py index 84ce65432..6e2c2e9bd 100644 --- a/volatility3/framework/symbols/windows/versions.py +++ b/volatility3/framework/symbols/windows/versions.py @@ -151,11 +151,45 @@ is_win10_16299_or_later = OsDistinguisher( ], ) +is_win10_17763_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 17763), + fallback_checks=[ + ("_EPROCESS", "TrustletIdentity", False), + ("ParentSecurityDomain", None, False), + ], +) + +is_win10_18362_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 18362), + fallback_checks=[ + ("ObHeaderCookie", None, True), + ("_CM_CACHED_VALUE_INDEX", None, False), + ("_WNF_PROCESS_CONTEXT", None, True), + ], +) + is_win10_18363_or_later = OsDistinguisher( version_check=lambda x: x >= (10, 0, 18363), fallback_checks=[("_KQOS_GROUPING_SETS", None, True)], ) +is_win10_19041_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 19041), + fallback_checks=[ + ("_EPROCESS", "TimerResolutionIgnore", True), + ("_EPROCESS", "VmProcessorHostTransition", True), + ("_KQOS_GROUPING_SETS", None, True), + ], +) + +is_win10_25398_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 25398), + fallback_checks=[ + ("_EPROCESS", "MmSlabIdentity", True), + ("_EPROCESS", "EnableProcessImpersonationLogging", True), + ], +) + is_windows_10 = OsDistinguisher( version_check=lambda x: x >= (10, 0), fallback_checks=[("ObHeaderCookie", None, True)], From 86d6b4f7245a553bb2b7db6a3d844375a57f8cd4 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:08:14 -0600 Subject: [PATCH 056/979] Bump plugin version --- volatility3/framework/plugins/windows/svcscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 74fe2d802..7688594d2 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -32,7 +32,7 @@ class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 4b400b6df612fe0f3a796c06dbfb7fa272e5d0fd Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:09:36 -0600 Subject: [PATCH 057/979] Refactor out code duplication in OS version checks --- .../framework/plugins/windows/svcscan.py | 132 ++++++------------ 1 file changed, 39 insertions(+), 93 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 7688594d2..4f20ed346 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -4,9 +4,16 @@ import logging import os -from typing import Dict, List, NamedTuple, Union +from typing import Dict, List, NamedTuple, Optional, Tuple, Union, cast -from volatility3.framework import constants, exceptions, interfaces, renderers, symbols +from volatility3.framework import ( + constants, + exceptions, + interfaces, + objects, + renderers, + symbols, +) from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints @@ -76,6 +83,28 @@ class SvcScan(interfaces.plugins.PluginInterface): binary_info.dll, ) + # These checks must be completed from newest -> oldest OS version. + _win_version_file_map: List[Tuple[versions.OsDistinguisher, bool, str]] = [ + (versions.is_win10_25398_or_later, True, "services-win10-25398-x64"), + (versions.is_win10_19041_or_later, True, "services-win10-19041-x64"), + (versions.is_win10_19041_or_later, False, "services-win10-19041-x86"), + (versions.is_win10_18362_or_later, True, "services-win10-18362-x64"), + (versions.is_win10_18362_or_later, False, "services-win10-18362-x86"), + (versions.is_win10_17763_or_later, False, "services-win10-17763-x86"), + (versions.is_win10_16299_or_later, True, "services-win10-16299-x64"), + (versions.is_win10_16299_or_later, False, "services-win10-16299-x86"), + (versions.is_win10_15063, True, "services-win10-15063-x64"), + (versions.is_win10_15063, False, "services-win10-15063-x86"), + (versions.is_win10_up_to_15063, True, "services-win8-x64"), + (versions.is_win10_up_to_15063, False, "services-win8-x86"), + (versions.is_windows_8_or_later, True, "services-win8-x64"), + (versions.is_windows_8_or_later, True, "services-win8-x86"), + (versions.is_vista_or_later, True, "services-vista-x64"), + (versions.is_vista_or_later, False, "services-vista-x86"), + (versions.is_windows_xp, False, "services-xp-x86"), + (versions.is_xp_or_2003, True, "services-xp-2003-x64"), + ] + @staticmethod def create_service_table( context: interfaces.context.ContextInterface, @@ -96,97 +125,14 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types = context.symbol_space[symbol_table].natives is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) - if ( - versions.is_windows_xp(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-xp-x86" - elif ( - versions.is_xp_or_2003(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-xp-2003-x64" - elif ( - versions.is_win10_25398_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-25398-x64" - elif ( - versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-19041-x64" - elif ( - versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-19041-x86" - elif ( - versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-18362-x64" - elif ( - versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-18362-x86" - elif ( - versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-16299-x64" - elif ( - versions.is_win10_17763_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-17763-x86" - elif ( - versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-16299-x86" - elif ( - versions.is_win10_15063(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-15063-x64" - elif ( - versions.is_win10_15063(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-15063-x86" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win8-x64" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win8-x86" - elif ( - versions.is_windows_8_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win8-x64" - elif ( - versions.is_windows_8_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win8-x86" - elif ( - versions.is_vista_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-vista-x64" - elif ( - versions.is_vista_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-vista-x86" - else: + try: + symbol_filename = next( + filename + for version_check, for_64bit, filename in SvcScan._win_version_file_map + if is_64bit == for_64bit + and version_check(context=context, symbol_table=symbol_table) + ) + except StopIteration: raise NotImplementedError("This version of Windows is not supported!") return intermed.IntermediateSymbolTable.create( From 5541378afe86d41734c05f6b84bbafcd26d46644 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:09:59 -0600 Subject: [PATCH 058/979] Create unique config path --- volatility3/framework/plugins/windows/svcscan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 4f20ed346..d16bee113 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -147,7 +147,9 @@ class SvcScan(interfaces.plugins.PluginInterface): def _get_service_key(self, kernel): for hive in hivelist.HiveList.list_hives( context=self.context, - base_config_path=self.config_path, + base_config_path=interfaces.configuration.path_join( + self.config_path, "hivelist" + ), layer_name=kernel.layer_name, symbol_table=kernel.symbol_table_name, filter_string="machine\\system", From f5c1bf0f1f5c177b2dcee9d09d3ab522ba83f41e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:10:20 -0600 Subject: [PATCH 059/979] Remove argument that matches default --- volatility3/framework/plugins/windows/svcscan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index d16bee113..b77491a65 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -153,7 +153,6 @@ class SvcScan(interfaces.plugins.PluginInterface): layer_name=kernel.layer_name, symbol_table=kernel.symbol_table_name, filter_string="machine\\system", - hive_offsets=None, ): # Get ControlSet\Services. try: From c517a44cde3cb87d48da3a72d685630eca7acaa1 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:10:54 -0600 Subject: [PATCH 060/979] Add type hints/casts to _get_service_key method --- volatility3/framework/plugins/windows/svcscan.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index b77491a65..dd2f76890 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -144,7 +144,7 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types=native_types, ) - def _get_service_key(self, kernel): + def _get_service_key(self, kernel) -> Optional[objects.StructType]: for hive in hivelist.HiveList.list_hives( context=self.context, base_config_path=interfaces.configuration.path_join( @@ -156,10 +156,14 @@ class SvcScan(interfaces.plugins.PluginInterface): ): # Get ControlSet\Services. try: - return hive.get_key(r"CurrentControlSet\Services") + return cast( + objects.StructType, hive.get_key(r"CurrentControlSet\Services") + ) except (KeyError, exceptions.InvalidAddressException): try: - return hive.get_key(r"ControlSet001\Services") + return cast( + objects.StructType, hive.get_key(r"ControlSet001\Services") + ) except (KeyError, exceptions.InvalidAddressException): pass From 42c2a86a37ace33cf93d28efd491b00e8c29dd53 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:11:16 -0600 Subject: [PATCH 061/979] Add logging for when a Services key cannot be retrieved --- volatility3/framework/plugins/windows/svcscan.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index dd2f76890..095661880 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -165,6 +165,10 @@ class SvcScan(interfaces.plugins.PluginInterface): objects.StructType, hive.get_key(r"ControlSet001\Services") ) except (KeyError, exceptions.InvalidAddressException): + vollog.log( + constants.LOGLEVEL_VVVV, + "Could not retrieve any control set from SYSTEM hive", + ) pass return None From 4192e8a711aff985352f3347dc8cd289894a812f Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 18 Jan 2024 12:22:23 -0600 Subject: [PATCH 062/979] Fix condition in OsDistinguisher --- volatility3/framework/symbols/windows/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/versions.py b/volatility3/framework/symbols/windows/versions.py index 6e2c2e9bd..e1e74afc0 100644 --- a/volatility3/framework/symbols/windows/versions.py +++ b/volatility3/framework/symbols/windows/versions.py @@ -155,7 +155,7 @@ is_win10_17763_or_later = OsDistinguisher( version_check=lambda x: x >= (10, 0, 17763), fallback_checks=[ ("_EPROCESS", "TrustletIdentity", False), - ("ParentSecurityDomain", None, False), + ("ParentSecurityDomain", None, True), ], ) From 43e8e01daff8dc1ff88b9edfc6d5d6b2fac33417 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 30 Jan 2024 09:26:17 -0600 Subject: [PATCH 063/979] Removes unnecessary pass and deindents return stmt --- volatility3/framework/plugins/windows/svcscan.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 095661880..10de46e2a 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -169,9 +169,8 @@ class SvcScan(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVVV, "Could not retrieve any control set from SYSTEM hive", ) - pass - return None + return None @staticmethod def _get_service_dll( From 470c750d82ee0208f673472bcf2bf6632ae4190f Mon Sep 17 00:00:00 2001 From: hsarkey Date: Wed, 31 Jan 2024 16:00:29 -0500 Subject: [PATCH 064/979] Updated to use dictionary for refined criteria and BaseAbsentValue for when criteria is not met --- .../framework/plugins/windows/malfind.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 284144077..8c0cb68ac 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -142,15 +142,20 @@ class Malfind(interfaces.plugins.PluginInterface): kernel = self.context.modules[self.config["kernel"]] # set refined criteria to know when to add to "Notes" column - refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] + refined_criteria = { + b"MZ": "MZ header", + b"\x55\x8B": "PE header", + b"\x55\x48": "Function prologue", + b"\x55\x89": "Function prologue", + } is_32bit_arch = not symbols.symbol_table_is_64bit( self.context, kernel.symbol_table_name ) for proc in procs: - # by default, "Notes" column will be set to none - notes = "None" + # by default, "Notes" column will be set to N/A + notes = renderers.NotApplicableValue() process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( @@ -158,12 +163,7 @@ class Malfind(interfaces.plugins.PluginInterface): ): # Check for unique headers and update "Notes" column if criteria is met if data[0:2] in refined_criteria: - if data[0:2] == b"MZ": - notes = "MZ header" - elif data[0:2] == b"\x55\x8B": - notes = "PE header" - else: - notes = "Function prologue" + notes = refined_criteria[data[0:2]] # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 if is_32bit_arch or proc.get_is_wow64(): From 6d093a4e4338aaee5b69384f1652cda994a3e1d3 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 31 Jan 2024 21:04:16 +0000 Subject: [PATCH 065/979] Documentation: Update copyright in README.md for 2024 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 471735af8..f69acb3bb 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 31 Jan 2024 21:04:16 +0000 Subject: [PATCH 066/979] Documentation: Update copyright in README.md for 2024 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 471735af8..f69acb3bb 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 31 Jan 2024 21:09:48 +0000 Subject: [PATCH 067/979] Core: Apply black 24.1.0 across the whole codebase --- volatility3/cli/text_renderer.py | 8 +++--- volatility3/framework/__init__.py | 6 ++--- volatility3/framework/automagic/mac.py | 6 ++--- volatility3/framework/automagic/module.py | 6 ++--- .../framework/automagic/symbol_finder.py | 12 ++++----- volatility3/framework/automagic/windows.py | 20 +++++++------- .../framework/configuration/requirements.py | 6 ++--- volatility3/framework/layers/avml.py | 6 ++--- volatility3/framework/layers/elf.py | 6 ++--- volatility3/framework/layers/intel.py | 6 ++--- volatility3/framework/layers/lime.py | 6 ++--- volatility3/framework/layers/qemu.py | 6 ++--- volatility3/framework/layers/xen.py | 6 ++--- volatility3/framework/objects/__init__.py | 6 ++--- volatility3/framework/plugins/mac/pslist.py | 4 +-- .../framework/plugins/windows/crashinfo.py | 6 ++--- .../framework/plugins/windows/mftscan.py | 1 - .../framework/plugins/windows/netscan.py | 10 ++++--- .../plugins/windows/registry/printkey.py | 6 ++--- volatility3/framework/renderers/conversion.py | 6 ++--- volatility3/framework/symbols/__init__.py | 26 +++++++++---------- .../symbols/windows/extensions/__init__.py | 6 ++--- volatility3/plugins/windows/statistics.py | 10 +++---- 23 files changed, 88 insertions(+), 93 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ffb8d516b..6e58ee68d 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -389,9 +389,11 @@ class JsonRenderer(CLIRenderer): interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), - datetime.datetime: lambda x: x.isoformat() - if not isinstance(x, interfaces.renderers.BaseAbsentValue) - else None, + datetime.datetime: lambda x: ( + x.isoformat() + if not isinstance(x, interfaces.renderers.BaseAbsentValue) + else None + ), "default": lambda x: x, } diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 9c17846a8..1565b2267 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -206,9 +206,9 @@ def _zipwalk(path: str): if not file.is_dir(): dirlist = zip_results.get(os.path.dirname(file.filename), []) dirlist.append(os.path.basename(file.filename)) - zip_results[ - os.path.join(path, os.path.dirname(file.filename)) - ] = dirlist + zip_results[os.path.join(path, os.path.dirname(file.filename))] = ( + dirlist + ) for value in zip_results: yield value, zip_results[value] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index aa75fbc3d..e51753139 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -138,9 +138,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): config_path = join("automagic", "MacIntelHelper", new_layer_name) context.config[join(config_path, "memory_layer")] = layer_name context.config[join(config_path, "page_map_offset")] = dtb - context.config[ - join(config_path, MacSymbolFinder.banner_config_key) - ] = str(banner, "latin-1") + context.config[join(config_path, MacSymbolFinder.banner_config_key)] = ( + str(banner, "latin-1") + ) new_layer = intel.Intel32e( context, diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index ee56a040c..ff13db905 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -34,9 +34,9 @@ class KernelModule(interfaces.automagic.AutomagicInterface): return None # The requirement is unfulfilled and is a ModuleRequirement - context.config[ - interfaces.configuration.path_join(new_config_path, "class") - ] = "volatility3.framework.contexts.Module" + context.config[interfaces.configuration.path_join(new_config_path, "class")] = ( + "volatility3.framework.contexts.Module" + ) for req in requirement.requirements: if ( diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index bf1c8ff16..21e594549 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -150,12 +150,12 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join - context.config[ - path_join(config_path, requirement.name, "class") - ] = clazz - context.config[ - path_join(config_path, requirement.name, "isf_url") - ] = isf_path + context.config[path_join(config_path, requirement.name, "class")] = ( + clazz + ) + context.config[path_join(config_path, requirement.name, "isf_url")] = ( + isf_path + ) context.config[ path_join(config_path, requirement.name, "symbol_mask") ] = layer.address_mask diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index a8530829b..52296f5ad 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -402,19 +402,19 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): if swap_location: context.config[current_layer_path] = current_layer_name try: - context.config[ - layer_loc_path - ] = requirements.URIRequirement.location_from_file( - swap_location + context.config[layer_loc_path] = ( + requirements.URIRequirement.location_from_file( + swap_location + ) ) except ValueError: vollog.warning( f"Volatility swap_location {swap_location} could not be validated - swap layer disabled" ) continue - context.config[ - layer_class_path - ] = "volatility3.framework.layers.physical.FileLayer" + context.config[layer_class_path] = ( + "volatility3.framework.layers.physical.FileLayer" + ) # Add the requirement new_req = requirements.TranslationLayerRequirement( @@ -424,9 +424,9 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): ) swap_req.add_requirement(new_req) - context.config[ - path_join(swap_sub_config, "number_of_elements") - ] = counter + context.config[path_join(swap_sub_config, "number_of_elements")] = ( + counter + ) context.config[swap_sub_config] = True swap_req.construct(context, swap_config) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index abdffdbe4..1c0622574 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -550,9 +550,9 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} - context.config[ - interfaces.configuration.path_join(config_path, self.name) - ] = True + context.config[interfaces.configuration.path_join(config_path, self.name)] = ( + True + ) return {} @classmethod diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index c825464cc..2e5572192 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -224,7 +224,7 @@ class AVMLStacker(interfaces.automagic.StackerLayerInterface): except exceptions.LayerException: return None new_name = context.layers.free_layer_name("AVMLLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return AVMLLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index a10d36592..b2fd6d4d1 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -115,9 +115,9 @@ class Elf64Stacker(interfaces.automagic.StackerLayerInterface): vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("Elf64Layer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) try: return Elf64Layer(context, new_name, new_name) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 7d3b86a12..ae477854d 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -277,9 +277,9 @@ class Intel(linear.LinearlyMappedLayer): This allows translation layers to provide maps of contiguous regions in one layer """ - stashed_offset = ( - stashed_mapped_offset - ) = stashed_size = stashed_mapped_size = stashed_map_layer = None + stashed_offset = stashed_mapped_offset = stashed_size = stashed_mapped_size = ( + stashed_map_layer + ) = None for offset, size, mapped_offset, mapped_size, map_layer in self._mapping( offset, length, ignore_errors ): diff --git a/volatility3/framework/layers/lime.py b/volatility3/framework/layers/lime.py index 28d646640..8b93932ab 100644 --- a/volatility3/framework/layers/lime.py +++ b/volatility3/framework/layers/lime.py @@ -104,7 +104,7 @@ class LimeStacker(interfaces.automagic.StackerLayerInterface): except LimeFormatException: return None new_name = context.layers.free_layer_name("LimeLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return LimeLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 501b8655e..ff483291c 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -486,9 +486,9 @@ class QemuStacker(interfaces.automagic.StackerLayerInterface): except exceptions.LayerException: return None new_name = context.layers.free_layer_name("QemuSuspendLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) layer = QemuSuspendLayer(context, new_name, new_name) cls.stacker_slow_warning() return layer diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index f7881a091..927b30430 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -173,8 +173,8 @@ class XenCoreDumpStacker(elf.Elf64Stacker): vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("XenCoreDumpLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return XenCoreDumpLayer(context, new_name, new_name) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 831856e3d..316a30bec 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -768,12 +768,10 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): raise IndexError(f"Member not present in array template: {child}") @overload - def __getitem__(self, i: int) -> interfaces.objects.Template: - ... + def __getitem__(self, i: int) -> interfaces.objects.Template: ... @overload - def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: - ... + def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: ... def __getitem__(self, i): """Returns the i-th item from the array.""" diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 88045a277..9b570f3f9 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -49,9 +49,7 @@ class PsList(interfaces.plugins.PluginInterface): ] @classmethod - def get_list_tasks( - cls, method: str - ) -> Callable[ + def get_list_tasks(cls, method: str) -> Callable[ [interfaces.context.ContextInterface, str, Callable[[int], bool]], Iterable[interfaces.objects.ObjectInterface], ]: diff --git a/volatility3/framework/plugins/windows/crashinfo.py b/volatility3/framework/plugins/windows/crashinfo.py index 4ecd85087..862eb6080 100644 --- a/volatility3/framework/plugins/windows/crashinfo.py +++ b/volatility3/framework/plugins/windows/crashinfo.py @@ -46,9 +46,9 @@ class Crashinfo(interfaces.plugins.PluginInterface): bitmap_size = format_hints.Hex(summary_header.BitmapSize) bitmap_pages = format_hints.Hex(summary_header.Pages) else: - bitmap_header_size = ( - bitmap_size - ) = bitmap_pages = renderers.NotApplicableValue() + bitmap_header_size = bitmap_size = bitmap_pages = ( + renderers.NotApplicableValue() + ) yield ( 0, diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 7e4e1ca18..a266fd864 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -176,7 +176,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): class ADS(interfaces.plugins.PluginInterface): - """Scans for Alternate Data Stream""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index d0bbd5cbd..62ead3ab7 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -487,10 +487,12 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if not isinstance(row_data[9], datetime.datetime): continue row_data = [ - "N/A" - if isinstance(i, renderers.UnreadableValue) - or isinstance(i, renderers.UnparsableValue) - else i + ( + "N/A" + if isinstance(i, renderers.UnreadableValue) + or isinstance(i, renderers.UnparsableValue) + else i + ) for i in row_data ] description = ( diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index e248c19bc..180f8f9d9 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -193,9 +193,9 @@ class PrintKey(interfaces.plugins.PluginInterface): vollog.debug( "Couldn't read registry value type, so data is unreadable" ) - value_data: Union[ - interfaces.renderers.BaseAbsentValue, bytes - ] = renderers.UnreadableValue() + value_data: Union[interfaces.renderers.BaseAbsentValue, bytes] = ( + renderers.UnreadableValue() + ) else: try: value_data = node.decode_data() diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index bf7da9ecb..bb18fcc8a 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -28,9 +28,9 @@ def wintime_to_datetime( def unixtime_to_datetime( unixtime: int, ) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: - ret: Union[ - interfaces.renderers.BaseAbsentValue, datetime.datetime - ] = renderers.UnparsableValue() + ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = ( + renderers.UnparsableValue() + ) if unixtime > 0: with contextlib.suppress(ValueError): diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index 10cf39cf1..d1e7a104d 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -35,9 +35,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): def __init__(self) -> None: super().__init__() - self._dict: Dict[ - str, interfaces.symbols.BaseSymbolTableInterface - ] = collections.OrderedDict() + self._dict: Dict[str, interfaces.symbols.BaseSymbolTableInterface] = ( + collections.OrderedDict() + ) # Permanently cache all resolved symbols self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} @@ -73,9 +73,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self, offset: int, size: int = 0, table_name: str = None ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" - table_list: Iterable[ - interfaces.symbols.BaseSymbolTableInterface - ] = self._dict.values() + table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = ( + self._dict.values() + ) if table_name is not None: if table_name in self._dict: table_list = [self._dict[table_name]] @@ -179,15 +179,15 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): if child.vol.type_name not in self._resolved: traverse_list.append(child.vol.type_name) try: - self._resolved[ - child.vol.type_name - ] = self._weak_resolve( - SymbolType.TYPE, child.vol.type_name + self._resolved[child.vol.type_name] = ( + self._weak_resolve( + SymbolType.TYPE, child.vol.type_name + ) ) except exceptions.SymbolError: - self._resolved[ - child.vol.type_name - ] = self.UnresolvedTemplate(child.vol.type_name) + self._resolved[child.vol.type_name] = ( + self.UnresolvedTemplate(child.vol.type_name) + ) # Stash the replacement replacements.add((traverser, child)) elif child.children: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d435851d7..846e5bd90 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -452,9 +452,9 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): ].is_valid(self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name: Union[ - str, interfaces.renderers.BaseAbsentValue - ] = renderers.UnreadableValue() + name: Union[str, interfaces.renderers.BaseAbsentValue] = ( + renderers.UnreadableValue() + ) # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 9915312e3..7f56b75f8 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -31,13 +31,9 @@ class Statistics(plugins.PluginInterface): # Do mass mapping and determine the number of different layers and how many pages go to each one layer = self.context.layers[self.config["primary"]] - page_count = ( - swap_count - ) = ( - invalid_page_count - ) = ( - large_page_count - ) = large_swap_count = large_invalid_count = other_invalid = 0 + page_count = swap_count = invalid_page_count = large_page_count = ( + large_swap_count + ) = large_invalid_count = other_invalid = 0 if isinstance(layer, intel.Intel): page_addr = 0 From ba488c78cd02478a60ba94a72d1c6a712f02c365 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 31 Jan 2024 21:13:00 +0000 Subject: [PATCH 068/979] Documentation: Bump the copyright here to 2024 --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index d601c1eee..cabfdc327 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -169,7 +169,7 @@ master_doc = "index" # General information about the project. project = "Volatility 3" -copyright = "2012-2022, Volatility Foundation" +copyright = "2012-2024, Volatility Foundation" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From f95be04305d7fe1bedf599849d40996f82d2032b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 31 Jan 2024 21:13:00 +0000 Subject: [PATCH 069/979] Documentation: Bump the copyright here to 2024 --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index d601c1eee..cabfdc327 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -169,7 +169,7 @@ master_doc = "index" # General information about the project. project = "Volatility 3" -copyright = "2012-2022, Volatility Foundation" +copyright = "2012-2024, Volatility Foundation" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From d1c7f14a75dc36c32523caa907aa49375c772160 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 2 Feb 2024 13:28:04 +0000 Subject: [PATCH 070/979] Linux: attempt to fix issue 1089 --- .../symbols/linux/extensions/__init__.py | 75 ++++++++++++++----- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d8a2867cc..f67e3962b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -26,42 +26,79 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): + def _get_mod_mem_type(self): + """Attempt to get the mod_mem_type enum once to allow repeated access from other functions""" + if not self.has_member("mod_mem_type"): + try: + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type( + self._context, self + ) + # mod_mem_type and module_memory were added in kernel 6.4 which replaces + # module_layout for storing the information around core_layout etc. + # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information + self.mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices + except exceptions.SymbolError: + vollog.debug( + f"Unable to find mod_mem_type enum. This is expected on kernels <6.4 but may cause issues with later kernels" + ) + self.mod_mem_type = None + def get_module_base(self): - if self.has_member("core_layout"): - return self.core_layout.base + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base else: - return self.module_core + if self.has_member("core_layout"): + return self.core_layout.base + else: + return self.module_core def get_init_size(self): - if self.has_member("init_layout"): - return self.init_layout.size - elif self.has_member("init_size"): - return self.init_size + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + else: + if self.has_member("init_layout"): + return self.init_layout.size + elif self.has_member("init_size"): + return self.init_size raise AttributeError( "module -> get_init_size: Unable to determine .init section size of module" ) def get_core_size(self): - if self.has_member("core_layout"): - return self.core_layout.size - elif self.has_member("core_size"): - return self.core_size + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_TEXT"]].size + else: + if self.has_member("core_layout"): + return self.core_layout.size + elif self.has_member("core_size"): + return self.core_size raise AttributeError( "module -> get_core_size: Unable to determine core size of module" ) def get_module_core(self): - if self.has_member("core_layout"): - return self.core_layout.base - elif self.has_member("module_core"): - return self.module_core + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base + else: + if self.has_member("core_layout"): + return self.core_layout.base + elif self.has_member("module_core"): + return self.module_core raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - if self.has_member("init_layout"): - return self.init_layout.base - elif self.has_member("module_init"): - return self.module_init + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base + else: + if self.has_member("init_layout"): + return self.init_layout.base + elif self.has_member("module_init"): + return self.module_init raise AttributeError("module -> get_module_core: Unable to get module init") def get_name(self): From ecf20d99b4ddc3d28c577ee383902877c3090d45 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 2 Feb 2024 13:28:57 +0000 Subject: [PATCH 071/979] Linux: fix typo in error message in module.get_module_init extension --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f67e3962b..9c15acef5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -99,7 +99,7 @@ class module(generic.GenericIntelProcess): return self.init_layout.base elif self.has_member("module_init"): return self.module_init - raise AttributeError("module -> get_module_core: Unable to get module init") + raise AttributeError("module -> get_module_init: Unable to get module init") def get_name(self): """Get the name of the module as a string""" From 261d7ffbf2e6837980989292b3966ddd2042d277 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 2 Feb 2024 14:03:55 +0000 Subject: [PATCH 072/979] Linux: include all mod mem types in size calculations --- .../framework/symbols/linux/extensions/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9c15acef5..e3ff4bdd5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -56,7 +56,11 @@ class module(generic.GenericIntelProcess): def get_init_size(self): self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + return ( + self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size + + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size + ) else: if self.has_member("init_layout"): return self.init_layout.size @@ -69,7 +73,12 @@ class module(generic.GenericIntelProcess): def get_core_size(self): self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_TEXT"]].size + return ( + self.mem[self.mod_mem_type["MOD_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_DATA"]].size + + self.mem[self.mod_mem_type["MOD_RODATA"]].size + + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size + ) else: if self.has_member("core_layout"): return self.core_layout.size From 130621a2e7655015f220ac561de974e99f571fca Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Feb 2024 20:04:36 +0100 Subject: [PATCH 073/979] Changing comment when Type error is occuring --- volatility3/framework/plugins/windows/iat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 11c273859..73976fb86 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -50,7 +50,7 @@ class IAT(interfaces.plugins.PluginInterface): ) if proc_layer_name is None: - raise TypeError("Layer must be a string not None") + raise TypeError("add_process_layer failed") pe_table_name = intermed.IntermediateSymbolTable.create( self.context, From ab8ed049a4c4bddbc4c2a2129cb425b9113b5d37 Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sat, 3 Feb 2024 11:32:00 +0300 Subject: [PATCH 074/979] Windows: add TrueCrypt plugin --- .../framework/plugins/windows/truecrypt.py | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 volatility3/framework/plugins/windows/truecrypt.py diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py new file mode 100644 index 000000000..988c900ea --- /dev/null +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -0,0 +1,141 @@ +from typing import Iterable, Generator, List, Tuple + +from volatility3.framework import constants, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces.configuration import RequirementInterface +from volatility3.framework.interfaces.objects import ObjectInterface +from volatility3.framework.objects import Bytes, DataFormatInfo, Integer, StructType +from volatility3.framework.objects.templates import ObjectTemplate +from volatility3.framework.objects.utility import array_to_string +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import pe + +from volatility3.plugins.windows import modules + +class Passphrase(interfaces.plugins.PluginInterface): + """TrueCrypt Cached Passphrase Finder""" + + _version = (0, 1, 0) + _required_framework_version = (2, 5, 2) + + @classmethod + def get_requirements(cls) -> List[RequirementInterface]: + return [ + requirements.ModuleRequirement( + 'kernel', + description='Windows kernel', + architectures=['Intel32', 'Intel64'] + ), + requirements.VersionRequirement( + name='modules', + component=modules.Modules, + version=(1, 1, 0) + ), + requirements.IntRequirement( + name="min-length", + description="Minimum length of passphrases to identify", + default=5, + optional=True + ), + ] + + def scan_module(self, module_base: int, layer_name: str) -> Generator[Tuple[int, str], None, None]: + """Scans the TrueCrypt kernel module for cached passphrases. + + Args: + module_base: the module's DLL base + layer_name: the name of the layer in which the module resides + + Generates: + A tuple of the offset at which a password is found, and the password + """ + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "windows", + "pe", + class_types=pe.class_types + ) + dos_header: pe.IMAGE_DOS_HEADER = self.context.object( + pe_table_name + constants.BANG + '_IMAGE_DOS_HEADER', + layer_name, + module_base, + ) + data_section: StructType = next( + sec for sec in dos_header.get_nt_header().get_sections() + if array_to_string(sec.Name) == '.data' + ) + base: int = data_section.VirtualAddress + module_base + size: int = data_section.Misc.VirtualSize + # Looking at `Length` in TrueCrypt/Common/Password.h::Password struct + DWORD_SIZE_BYTES: int = 4 + format = DataFormatInfo(length=DWORD_SIZE_BYTES, byteorder="little", signed=True) + int32 = ObjectTemplate( + Integer, + pe_table_name + constants.BANG + 'int', + data_format=format + ) + count, not_aligned = divmod(size, DWORD_SIZE_BYTES) + if not_aligned: + raise ValueError("PE data section not DWORD-aligned!") + lengths = self.context.object( + pe_table_name + constants.BANG + 'array', + layer_name, + base, + count=count, + subtype=int32, + ) + min_length = self.config.get('min-length') + for length in lengths: + # TrueCrypt maximum password length is 64 + # (see TrueCrypt/Common/Password.h) + if not min_length <= length <= 64: + continue + offset = length.vol['offset'] + DWORD_SIZE_BYTES + passphrase: Bytes = self.context.object( + pe_table_name + constants.BANG + 'bytes', + layer_name, + offset, + length=length, + ) + # TrueCrypt/Common/Password.c permits chars in the range + # [0x20, 0x7F). + if not all(0x20 <= c < 0x7F for c in passphrase): + continue + # TrueCrypt/Common/Password.h::Password struct is padded with + # 3 zero bytes to keep 64-byte alignment. + buf: Bytes = self.context.object( + pe_table_name + constants.BANG + 'bytes', + layer_name, + offset + length + 1, # +1 for '\0'-terminated password string + length=3 + ) + if any(buf): + continue + # Password found. + yield offset, passphrase.decode(encoding='ascii') + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + mods: Iterable[ObjectInterface] = modules.Modules.list_modules( + self.context, + kernel.layer_name, + kernel.symbol_table_name + ) + truecrypt_module_base = next( + mod.DllBase for mod in mods + if mod.BaseDllName.get_string().lower() == 'truecrypt.sys' + ) + for offset, password in self.scan_module(truecrypt_module_base, kernel.layer_name): + yield (0, (format_hints.Hex(offset), len(password), password)) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Length", int), + ("Password", str), + ], + self._generator() + ) From 41cbfbfa340ef8949d20e1717aa5baaee02ef984 Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sat, 3 Feb 2024 11:58:25 +0300 Subject: [PATCH 075/979] add license --- volatility3/framework/plugins/windows/truecrypt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py index 988c900ea..feba56965 100644 --- a/volatility3/framework/plugins/windows/truecrypt.py +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -1,3 +1,7 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from typing import Iterable, Generator, List, Tuple from volatility3.framework import constants, interfaces, renderers From cf9029fd85ae090034a253e3dcd819a0381b6442 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 4 Feb 2024 00:43:33 +0100 Subject: [PATCH 076/979] Fixing the year in the Copyright --- volatility3/framework/plugins/windows/iat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 73976fb86..d2fdc0ad8 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,4 +1,4 @@ -# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 import logging, io, pefile From d44c0fef36dfe1f7a83cc25211073489a56820be Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sun, 4 Feb 2024 11:21:14 +0300 Subject: [PATCH 077/979] run Black formatter --- .../framework/plugins/windows/truecrypt.py | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py index feba56965..81250a749 100644 --- a/volatility3/framework/plugins/windows/truecrypt.py +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -17,6 +17,7 @@ from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import modules + class Passphrase(interfaces.plugins.PluginInterface): """TrueCrypt Cached Passphrase Finder""" @@ -27,78 +28,75 @@ class Passphrase(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[RequirementInterface]: return [ requirements.ModuleRequirement( - 'kernel', - description='Windows kernel', - architectures=['Intel32', 'Intel64'] + "kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name='modules', - component=modules.Modules, - version=(1, 1, 0) + name="modules", component=modules.Modules, version=(1, 1, 0) ), requirements.IntRequirement( name="min-length", description="Minimum length of passphrases to identify", default=5, - optional=True + optional=True, ), ] - - def scan_module(self, module_base: int, layer_name: str) -> Generator[Tuple[int, str], None, None]: + + def scan_module( + self, module_base: int, layer_name: str + ) -> Generator[Tuple[int, str], None, None]: """Scans the TrueCrypt kernel module for cached passphrases. - + Args: module_base: the module's DLL base layer_name: the name of the layer in which the module resides - + Generates: A tuple of the offset at which a password is found, and the password """ pe_table_name = intermed.IntermediateSymbolTable.create( - self.context, - self.config_path, - "windows", - "pe", - class_types=pe.class_types + self.context, self.config_path, "windows", "pe", class_types=pe.class_types ) dos_header: pe.IMAGE_DOS_HEADER = self.context.object( - pe_table_name + constants.BANG + '_IMAGE_DOS_HEADER', + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", layer_name, module_base, ) data_section: StructType = next( - sec for sec in dos_header.get_nt_header().get_sections() - if array_to_string(sec.Name) == '.data' + sec + for sec in dos_header.get_nt_header().get_sections() + if array_to_string(sec.Name) == ".data" ) base: int = data_section.VirtualAddress + module_base size: int = data_section.Misc.VirtualSize # Looking at `Length` in TrueCrypt/Common/Password.h::Password struct DWORD_SIZE_BYTES: int = 4 - format = DataFormatInfo(length=DWORD_SIZE_BYTES, byteorder="little", signed=True) + format = DataFormatInfo( + length=DWORD_SIZE_BYTES, byteorder="little", signed=True + ) int32 = ObjectTemplate( - Integer, - pe_table_name + constants.BANG + 'int', - data_format=format + Integer, pe_table_name + constants.BANG + "int", data_format=format ) count, not_aligned = divmod(size, DWORD_SIZE_BYTES) if not_aligned: raise ValueError("PE data section not DWORD-aligned!") lengths = self.context.object( - pe_table_name + constants.BANG + 'array', + pe_table_name + constants.BANG + "array", layer_name, base, count=count, subtype=int32, ) - min_length = self.config.get('min-length') + min_length = self.config.get("min-length") for length in lengths: # TrueCrypt maximum password length is 64 # (see TrueCrypt/Common/Password.h) if not min_length <= length <= 64: continue - offset = length.vol['offset'] + DWORD_SIZE_BYTES + offset = length.vol["offset"] + DWORD_SIZE_BYTES passphrase: Bytes = self.context.object( - pe_table_name + constants.BANG + 'bytes', + pe_table_name + constants.BANG + "bytes", layer_name, offset, length=length, @@ -110,30 +108,31 @@ class Passphrase(interfaces.plugins.PluginInterface): # TrueCrypt/Common/Password.h::Password struct is padded with # 3 zero bytes to keep 64-byte alignment. buf: Bytes = self.context.object( - pe_table_name + constants.BANG + 'bytes', + pe_table_name + constants.BANG + "bytes", layer_name, - offset + length + 1, # +1 for '\0'-terminated password string - length=3 + offset + length + 1, # +1 for '\0'-terminated password string + length=3, ) if any(buf): continue # Password found. - yield offset, passphrase.decode(encoding='ascii') - + yield offset, passphrase.decode(encoding="ascii") + def _generator(self): kernel = self.context.modules[self.config["kernel"]] mods: Iterable[ObjectInterface] = modules.Modules.list_modules( - self.context, - kernel.layer_name, - kernel.symbol_table_name + self.context, kernel.layer_name, kernel.symbol_table_name ) truecrypt_module_base = next( - mod.DllBase for mod in mods - if mod.BaseDllName.get_string().lower() == 'truecrypt.sys' + mod.DllBase + for mod in mods + if mod.BaseDllName.get_string().lower() == "truecrypt.sys" ) - for offset, password in self.scan_module(truecrypt_module_base, kernel.layer_name): + for offset, password in self.scan_module( + truecrypt_module_base, kernel.layer_name + ): yield (0, (format_hints.Hex(offset), len(password), password)) - + def run(self) -> renderers.TreeGrid: return renderers.TreeGrid( [ @@ -141,5 +140,5 @@ class Passphrase(interfaces.plugins.PluginInterface): ("Length", int), ("Password", str), ], - self._generator() + self._generator(), ) From 6a384c197c7cc7429ea14417cec8703a5f759395 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 7 Feb 2024 09:04:55 +0000 Subject: [PATCH 078/979] Linux: update mm_struct extension so that _get_maple_tree_iter and _get_mmap_iter are more clearly identified, showing that get_vma_iter should be used instead --- .../symbols/linux/extensions/__init__.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d73d0cfb9..b57803d20 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -447,12 +447,14 @@ class maple_tree(objects.StructType): class mm_struct(objects.StructType): - def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns an iterator for the mmap list member of an mm_struct.""" + def _get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns an iterator for the mmap list member of an mm_struct. Use this only if + required, get_vma_iter() will choose the correct _get_maple_tree_iter() or + _get_mmap_iter() automatically as required.""" if not self.has_member("mmap"): raise AttributeError( - "get_mmap_iter called on mm_struct where no mmap member exists." + "_get_mmap_iter called on mm_struct where no mmap member exists." ) if not self.mmap: return None @@ -466,12 +468,14 @@ class mm_struct(objects.StructType): seen.add(link.vol.offset) link = link.vm_next - def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns an iterator for the mm_mt member of an mm_struct.""" + def _get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns an iterator for the mm_mt member of an mm_struct. Use this only if + required, get_vma_iter() will choose the correct _get_maple_tree_iter() or + get_mmap_iter() automatically as required.""" if not self.has_member("mm_mt"): raise AttributeError( - "get_maple_tree_iter called on mm_struct where no mm_mt member exists." + "_get_maple_tree_iter called on mm_struct where no mm_mt member exists." ) symbol_table_name = self.get_symbol_table_name() for vma_pointer in self.mm_mt.get_slot_iter(): @@ -487,9 +491,9 @@ class mm_struct(objects.StructType): """Returns an iterator for the VMAs in an mm_struct. Automatically choosing the mmap or mm_mt as required.""" if self.has_member("mmap"): - yield from self.get_mmap_iter() + yield from self._get_mmap_iter() elif self.has_member("mm_mt"): - yield from self.get_maple_tree_iter() + yield from self._get_maple_tree_iter() else: raise AttributeError("Unable to find mmap or mm_mt in mm_struct") From 18220498ae572fadf6e47c3e38ffe65c24447183 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 7 Feb 2024 09:07:19 +0000 Subject: [PATCH 079/979] Linux: update linux.pslist plugin to use task.mm.get_vma_iter() so that it works correctly on newer kernels --- volatility3/framework/plugins/linux/pslist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 9afd13e5a..771040dc0 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -17,7 +17,7 @@ class PsList(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 2, 0) + _version = (2, 2, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -128,7 +128,7 @@ class PsList(interfaces.plugins.PluginInterface): else: # Find the vma that belongs to the main ELF of the process file_output = "Error outputting file" - for v in task.mm.get_mmap_iter(): + for v in task.mm.get_vma_iter(): if v.vm_start == task.mm.start_code: file_handle = elfs.Elfs.elf_dump( self.context, From 9127509db9a8bcfffc971e7b959d429ffcecea6e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:13:14 +0000 Subject: [PATCH 080/979] CLI: Reblack after updating PR --- volatility3/cli/__init__.py | 45 ++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 0e7ba41ee..9393c0805 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -106,7 +106,7 @@ class CommandLine: ) # Load up system defaults - delayed_logs, default_config = self.load_system_defaults('vol.json') + delayed_logs, default_config = self.load_system_defaults("vol.json") parser = volargparse.HelpfulArgParser( add_help=False, @@ -280,7 +280,7 @@ class CommandLine: if partial_args.cache_path: constants.CACHE_PATH = partial_args.cache_path - + vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") @@ -473,28 +473,47 @@ class CommandLine: ) return requirements.URIRequirement.location_from_file(filename) - def load_system_defaults(self, filename: str) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: + def load_system_defaults( + self, filename: str + ) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: """Modify the main configuration based on the default configuration override""" # Build the config path - default_config_path = os.path.join(os.path.expanduser("~"), ".config", "volatility3", filename) - if sys.platform == 'win32': - default_config_path = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3", - filename) + default_config_path = os.path.join( + os.path.expanduser("~"), ".config", "volatility3", filename + ) + if sys.platform == "win32": + default_config_path = os.path.join( + os.environ.get("APPDATA", os.path.expanduser("~")), + "volatility3", + filename, + ) delayed_logs = [] # Process it if the files exist if os.path.exists(default_config_path): - with open(default_config_path, 'rb') as config_json: + with open(default_config_path, "rb") as config_json: result = json.load(config_json) if not isinstance(result, dict): - delayed_logs.append((logging.INFO, - f'Default configuration file {default_config_path} does not contain a dictionary')) + delayed_logs.append( + ( + logging.INFO, + f"Default configuration file {default_config_path} does not contain a dictionary", + ) + ) else: delayed_logs.append( - (logging.INFO, f"Loading default configuration options from {default_config_path}")) - delayed_logs.append((logging.DEBUG, - f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}")) + ( + logging.INFO, + f"Loading default configuration options from {default_config_path}", + ) + ) + delayed_logs.append( + ( + logging.DEBUG, + f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}", + ) + ) return delayed_logs, result return delayed_logs, {} From a66a4461906e7661b957559634559a56ee6356f6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:16:35 +0000 Subject: [PATCH 081/979] CLI: Reblack after updating PR - take 2 --- volatility3/cli/volshell/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 40f42f3af..9e74acfec 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -56,7 +56,7 @@ class VolShell(cli.CommandLine): framework.require_interface_version(2, 0, 0) # Load up system defaults - delayed_logs, default_config = self.load_system_defaults('volshell.json') + delayed_logs, default_config = self.load_system_defaults("volshell.json") parser = argparse.ArgumentParser( prog=self.CLI_NAME, @@ -151,10 +151,12 @@ class VolShell(cli.CommandLine): default=constants.CACHE_PATH, type=str, ) - parser.add_argument("--offline", - help = "Do not search online for additional JSON files", - default = False, - action = 'store_true') + parser.add_argument( + "--offline", + help="Do not search online for additional JSON files", + default=False, + action="store_true", + ) # Volshell specific flags os_specific = parser.add_mutually_exclusive_group(required=False) From 7b42ef0a9f23ecb6b4fff2425c0ac33c85b30d04 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:59:50 +0000 Subject: [PATCH 082/979] Documentation: Add in more information about overriding configuration values --- doc/source/vol-cli.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index cc6f7fe6a..7b91e815d 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -143,3 +143,18 @@ Options `hivescan` would match `windows.registry.hivescan.HiveScan`, but `pslist` is ambiguous because it could match `windows.pslist` or `linux.pslist`. + +Overriding options +------------------ + +The default values for the command line interface are defined by constants within the code, +but can be overridden by creating a JSON file (`%APPDATA%/volatility3/vol.json` for Windows +systems, or `~/.config/volatility3/vol.json` or `volshell.json` for all others). + +The format of this file is a JSON dictionary, containing the options above and their value. +It should be noted that the ordering is (`<` means is overridden by): + +`in-built default value < config file value < command line parameter` + +It should also be noted that boolean flags (such as `offline`) that are overridden as true will +not be unset by not specifying the command line flag. From 63a325c5157845d607d119971928657fb02ecdc2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 23:44:12 +0000 Subject: [PATCH 083/979] CLI: Add support for filtering lines from output --- volatility3/cli/__init__.py | 12 +++++++++++- volatility3/cli/text_renderer.py | 9 +++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 91bda7c66..2c6dde00a 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -22,6 +22,7 @@ import traceback from typing import Any, Dict, Type, Union from urllib import parse, request +from volatility3.cli import text_filter import volatility3.plugins import volatility3.symbols from volatility3 import framework @@ -230,6 +231,12 @@ class CommandLine: default=False, action="store_true", ) + parser.add_argument( + "--filters", + help="List of filters to apply to the output (in the form of [+-]columname,pattern[!])", + default=[], + action="append", + ) # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. @@ -444,7 +451,10 @@ class CommandLine: try: # Construct and run the plugin if constructed: - renderers[args.renderer]().render(constructed.run()) + grid = constructed.run() + renderer = renderers[args.renderer]() + renderer.filter = text_filter.CLIFilter(grid, args.filters) + renderer.render(grid) except exceptions.VolatilityException as excp: self.process_exceptions(excp) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 6e58ee68d..b0ba6baa7 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -10,6 +10,7 @@ import string import sys from functools import wraps from typing import Any, Callable, Dict, List, Tuple +from volatility3.cli import text_filter from volatility3.framework import interfaces, renderers from volatility3.framework.renderers import format_hints @@ -134,6 +135,7 @@ class CLIRenderer(interfaces.renderers.Renderer): name = "unnamed" structured_output = False + filter: text_filter.CLIFilter = None class QuickTextRenderer(CLIRenderer): @@ -172,6 +174,9 @@ class QuickTextRenderer(CLIRenderer): outfd.write("\n{}\n".format("\t".join(line))) def visitor(node: interfaces.renderers.TreeNode, accumulator): + if self.filter and self.filter.filter(node.values): + return accumulator + accumulator.write("\n") # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case accumulator.write( @@ -306,6 +311,10 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths[tree_indent_column] = max( max_column_widths.get(tree_indent_column, 0), node.path_depth ) + + if self.filter and self.filter.filter(node.values): + return accumulator + line = {} for column_index in range(len(grid.columns)): column = grid.columns[column_index] From d8d3e157afab853fbffff394d7d1bc371025664e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 23:46:30 +0000 Subject: [PATCH 084/979] CLI: Don't forget the core text filter class --- volatility3/cli/text_filter.py | 96 ++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 volatility3/cli/text_filter.py diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py new file mode 100644 index 000000000..314af9a7a --- /dev/null +++ b/volatility3/cli/text_filter.py @@ -0,0 +1,96 @@ +import logging +from typing import Any, List, Optional +from volatility3.framework import constants, interfaces +import re + +vollog = logging.getLogger(__name__) + + +class CLIFilter: + def __init__(self, treegrid, filters: List[str]): + self._filters = self._prepare(treegrid, filters) + + def _prepare(self, treegrid: interfaces.renderers.TreeGrid, filters: List[str]): + """Runs through the filter strings and creates the necessary filter objects""" + output = [] + + for filter in filters: + exclude = False + regex = False + pattern = None + column_name = None + if filter.startswith("-"): + exclude = True + filter = filter[1:] + elif filter.startswith("+"): + filter = filter[1:] + components = filter.split(",") + if len(components) < 2: + pattern = components[0] + else: + column_name = components[0] + pattern = ",".join(components[1:]) + if pattern and pattern.endswith("!"): + regex = True + pattern = pattern[:-1] + column_num = None + if column_name: + for num, column in enumerate(treegrid.columns): + if column_name.lower() in column.name.lower(): + column_num = num + break + if pattern: + output.append(ColumnFilter(column_num, pattern, regex, exclude)) + + vollog.log(constants.LOGLEVEL_VVV, "Filters:\n" + repr(output)) + + return output + + def filter( + self, + row: List[Any], + ) -> bool: + """Filters the row based on each of the column_filters""" + found = any([column_filter.found(row) for column_filter in self._filters]) + return not found + + +class ColumnFilter: + def __init__( + self, + column_num: Optional[int], + pattern: str, + regex: bool = False, + exclude: bool = False, + ) -> None: + self.column_num = column_num + self.pattern = pattern + self.exclude = exclude + self.regex = regex + + def find(self, item) -> bool: + """Identifies whether an item is found in the appropriate column""" + try: + if self.regex: + return re.search(self.pattern, f"{item}") + return self.pattern in f"{item}" + except IOError: + return False + + def found(self, row: List[Any]) -> bool: + """Determines whether a row should be filtered + + If the classes exclude value is false, and the necessary pattern is found, the row is not filtered, + otherwise it is filtered. + """ + if self.column_num is None: + found = any([self.find(x) for x in row]) + else: + found = self.find(row[self.column_num]) + if self.exclude: + return not found + return found + + def __repr__(self) -> str: + """Returns a display of a column filter""" + return f"ColumnFilter(column={self.column_num},exclude={self.exclude},regex={self.regex},pattern={self.pattern})" From 865f004f5c6628c230c3dedde0e04573aa4e1339 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 9 Feb 2024 00:00:21 +0000 Subject: [PATCH 085/979] CLI: Ensure no filters still returns results --- volatility3/cli/text_filter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 314af9a7a..948e969df 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -51,6 +51,8 @@ class CLIFilter: row: List[Any], ) -> bool: """Filters the row based on each of the column_filters""" + if not self._filters: + return False found = any([column_filter.found(row) for column_filter in self._filters]) return not found From 683319bd7f96cef33e4b1cc4333d4c0835310113 Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 12 Feb 2024 08:47:02 +0000 Subject: [PATCH 086/979] Linux: add deprecation warning for get_maple_tree_iter and get_mmap_iter mm_struct extensions. --- .../symbols/linux/extensions/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b57803d20..f9db30498 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -447,6 +447,16 @@ class maple_tree(objects.StructType): class mm_struct(objects.StructType): + + def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """ + Deprecated: Use either get_vma_iter() or _get_mmap_iter(). + """ + vollog.warning( + "This method has been deprecated in favour of using the get_vma_iter() method." + ) + yield from self.get_vma_iter() + def _get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or @@ -468,6 +478,15 @@ class mm_struct(objects.StructType): seen.add(link.vol.offset) link = link.vm_next + def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """ + Deprecated: Use either get_vma_iter() or _get_maple_tree_iter(). + """ + vollog.warning( + "This method has been deprecated in favour of using the get_vma_iter() method." + ) + yield from self.get_vma_iter() + def _get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mm_mt member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or From 0e75140ec5a36e8f9933eee90cc74f0e06789242 Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 12 Feb 2024 09:35:06 +0000 Subject: [PATCH 087/979] Linux: update module extension with mod_mem_type property and add try/except for get methods --- .../symbols/linux/extensions/__init__.py | 75 ++++++++++++------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3557f3a17..fb60e3b97 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -15,6 +15,7 @@ from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATE from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES from volatility3.framework.constants.linux import CAPABILITIES from volatility3.framework import exceptions, objects, interfaces, symbols +from volatility3.framework.renderers import UnparsableValue from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed @@ -26,9 +27,10 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): - def _get_mod_mem_type(self): - """Attempt to get the mod_mem_type enum once to allow repeated access from other functions""" - if not self.has_member("mod_mem_type"): + @property + def mod_mem_type(self): + """Return the mod_mem_type enum choices if available or None if not""" + if not self.has_member("_mod_mem_type"): try: vmlinux = linux.LinuxUtilities.get_module_from_volobj_type( self._context, self @@ -36,31 +38,41 @@ class module(generic.GenericIntelProcess): # mod_mem_type and module_memory were added in kernel 6.4 which replaces # module_layout for storing the information around core_layout etc. # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information - self.mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices + self._mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices except exceptions.SymbolError: vollog.debug( f"Unable to find mod_mem_type enum. This is expected on kernels <6.4 but may cause issues with later kernels" ) - self.mod_mem_type = None + self._mod_mem_type = None + return self._mod_mem_type def get_module_base(self): - self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base + try: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base + except: + raise AttributeError( + "module -> get_module_base: Unable to get module base. Cannot read base from MOD_TEXT." + ) else: if self.has_member("core_layout"): return self.core_layout.base - else: + elif self.has_member("module_core"): return self.module_core + raise AttributeError("module -> get_module_base: Unable to get module base") def get_init_size(self): - self._get_mod_mem_type() if self.mod_mem_type: - return ( - self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size - + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size - ) + try: + return ( + self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size + + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size + ) + except: + raise AttributeError( + "module -> get_init_size: Unable to determine .init section size of module. Cannot read size of MOD_INIT_TEXT, MOD_INIT_DATA, and MOD_INIT_RODATA" + ) else: if self.has_member("init_layout"): return self.init_layout.size @@ -71,14 +83,19 @@ class module(generic.GenericIntelProcess): ) def get_core_size(self): - self._get_mod_mem_type() if self.mod_mem_type: - return ( - self.mem[self.mod_mem_type["MOD_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_DATA"]].size - + self.mem[self.mod_mem_type["MOD_RODATA"]].size - + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size - ) + try: + return ( + self.mem[self.mod_mem_type["MOD_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_DATA"]].size + + self.mem[self.mod_mem_type["MOD_RODATA"]].size + + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size + ) + except KeyError: + raise AttributeError( + "module -> get_core_size: Unable to determine core size of module. Cannot read size of MOD_TEXT, MOD_DATA, MOD_RODATA, and MOD_RO_AFTER_INIT." + ) + else: if self.has_member("core_layout"): return self.core_layout.size @@ -89,9 +106,13 @@ class module(generic.GenericIntelProcess): ) def get_module_core(self): - self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base + try: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base + except KeyError: + raise AttributeError( + "module -> get_module_core: Unable to get module core. Cannot read base from MOD_TEXT." + ) else: if self.has_member("core_layout"): return self.core_layout.base @@ -100,9 +121,13 @@ class module(generic.GenericIntelProcess): raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base + try: + return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base + except KeyError: + raise AttributeError( + "module -> get_module_core: Unable to get module init. Cannot read base from MOD_INIT_TEXT." + ) else: if self.has_member("init_layout"): return self.init_layout.base From 53d4ec66ebb774db76ef1999832662ecc541e56b Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 12 Feb 2024 09:36:33 +0000 Subject: [PATCH 088/979] Linux: Remove unnecessary import from linux extension. --- volatility3/framework/symbols/linux/extensions/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fb60e3b97..c259d16aa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -15,7 +15,6 @@ from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATE from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES from volatility3.framework.constants.linux import CAPABILITIES from volatility3.framework import exceptions, objects, interfaces, symbols -from volatility3.framework.renderers import UnparsableValue from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed From cc35fecf91b1d2704c87031427c623e066ac968a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 19:40:37 +1100 Subject: [PATCH 089/979] Linux: Add library_list plugin and other ELF related code enhacements. - Add library_list plugin - Add ELF dynamic table enum types in elf.json - Update missing program header enum types in elf.json - Add PAGE constants - Add ELF ident and class enums - Replace ELF hardcoded type numbers for enum description matching - Fix unmanaged ValueError exception issue in Elf64Layer::_load_segments() --- .../framework/constants/linux/__init__.py | 26 +++ volatility3/framework/layers/elf.py | 11 +- volatility3/framework/plugins/linux/elfs.py | 34 ++-- .../framework/plugins/linux/library_list.py | 169 ++++++++++++++++++ volatility3/framework/symbols/linux/elf.json | 138 +++++++++++++- .../symbols/linux/extensions/__init__.py | 3 +- .../framework/symbols/linux/extensions/elf.py | 122 +++++++++++-- 7 files changed, 464 insertions(+), 39 deletions(-) create mode 100644 volatility3/framework/plugins/linux/library_list.py diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 6e8883f19..5e82e580e 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -5,11 +5,15 @@ Linux-specific values that aren't found in debug symbols """ +from enum import IntEnum KERNEL_NAME = "__kernel__" # arch/x86/include/asm/page_types.h PAGE_SHIFT = 12 +PAGE_SIZE = 1 << PAGE_SHIFT +PAGE_MASK = ~(PAGE_SIZE - 1) + """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" # include/linux/sched.h @@ -281,3 +285,25 @@ CAPABILITIES = ( ) ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1 + + +class ELF_IDENT(IntEnum): + """ELF header e_ident indexes""" + + EI_MAG0 = 0 + EI_MAG1 = 1 + EI_MAG2 = 2 + EI_MAG3 = 3 + EI_CLASS = 4 + EI_DATA = 5 + EI_VERSION = 6 + EI_OSABI = 7 + EI_PAD = 8 + + +class ELF_CLASS(IntEnum): + """ELF header class types""" + + ELFCLASSNONE = 0 + ELFCLASS32 = 1 + ELFCLASS64 = 2 diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index b2fd6d4d1..6bd5c2d63 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -6,9 +6,11 @@ import struct from typing import Optional from volatility3.framework import exceptions, interfaces, constants +from volatility3.framework.constants.linux import ELF_CLASS from volatility3.framework.layers import segmented from volatility3.framework.symbols import intermed + vollog = logging.getLogger(__name__) @@ -21,7 +23,7 @@ class Elf64Layer(segmented.SegmentedLayer): _header_struct = struct.Struct(" 0 ): diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index e688ecb42..7171a6616 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -14,8 +14,14 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux.extensions import elf +from volatility3.framework.constants.linux import ( + PAGE_SIZE, + PAGE_MASK, + ELF_MAX_EXTRACTION_SIZE, +) from volatility3.plugins.linux import pslist + vollog = logging.getLogger(__name__) @@ -23,7 +29,7 @@ class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -87,7 +93,10 @@ class Elfs(plugins.PluginInterface): sections = {} # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ? for phdr in elf_object.get_program_headers(): - if phdr.p_type != 1: # PT_LOAD = 1 + try: + if phdr.p_type.description != "PT_LOAD": + continue + except ValueError: continue start = phdr.p_vaddr @@ -95,18 +104,18 @@ class Elfs(plugins.PluginInterface): end = start + size # Use complete memory pages for dumping - # If start isn't a multiple of 4096, stick to the highest multiple < start - # If end isn't a multiple of 4096, stick to the lowest multiple > end - if start % 4096: - start = start & ~0xFFF + # If start isn't a multiple of a page, stick to the highest multiple < start + # If end isn't a multiple of a page, stick to the lowest multiple > end + if start % PAGE_SIZE: + start = start & PAGE_MASK - if end % 4096: - end = (end & ~0xFFF) + 4096 + if end % PAGE_SIZE: + end = (end & PAGE_MASK) + PAGE_SIZE real_size = end - start # Check if ELF has a legitimate size - if real_size < 0 or real_size > constants.linux.ELF_MAX_EXTRACTION_SIZE: + if real_size < 0 or real_size > ELF_MAX_EXTRACTION_SIZE: raise ValueError(f"The claimed size of the ELF is invalid: {real_size}") sections[start] = real_size @@ -140,12 +149,7 @@ class Elfs(plugins.PluginInterface): for vma in task.mm.get_vma_iter(): hdr = proc_layer.read(vma.vm_start, 4, pad=True) - if not ( - hdr[0] == 0x7F - and hdr[1] == 0x45 - and hdr[2] == 0x4C - and hdr[3] == 0x46 - ): + if hdr != b"\x7fELF": continue path = vma.get_name(self.context, task) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py new file mode 100644 index 000000000..ed5545347 --- /dev/null +++ b/volatility3/framework/plugins/linux/library_list.py @@ -0,0 +1,169 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import Iterable, Tuple + +from volatility3.framework import interfaces, renderers, constants, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.framework.objects import utility +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.linux.extensions import elf +from volatility3.plugins.linux import pslist + + +vollog = logging.getLogger(__name__) + + +class LibraryList(interfaces.plugins.PluginInterface): + """Enumerate libraries loaded into processes""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + ), + requirements.ListRequirement( + name="pids", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + ] + + def get_libdl_libraries( + self, proc_layer_name: str, vma_start: int + ) -> interfaces.objects.ObjectInterface: + """Get the ELF link map objects for the given VMA address + + Args: + proc_layer_name (str): Name of the process layer + vma_start (int): VMA start address + + Yields: + ELF link map objects for the given VMA address + """ + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "linux", + "elf", + class_types=elf.class_types, + ) + elf_object = self.context.object( + elf_table_name + constants.BANG + "Elf", + offset=vma_start, + layer_name=proc_layer_name, + ) + + if not elf_object or not elf_object.is_valid(): + return None + + kernel = self.context.modules[self.config["kernel"]] + + try: + for link_map in elf_object.get_link_maps(kernel.symbol_table_name): + if link_map.l_addr and link_map.l_name: + yield link_map + except exceptions.InvalidAddressException: + # Protection against memory smear in this VMA + pass + + def get_libdl_maps( + self, task: interfaces.objects.ObjectInterface, proc_layer_name: str + ) -> interfaces.objects.ObjectInterface: + """Get the ELF link maps objects for a task + + Args: + task (task_struct): A reference task + proc_layer_name (str): Name of the process layer + + Yields: + ELF link map objects + """ + + link_map_seen = set() + for vma in task.mm.get_vma_iter(): + for link_map in self.get_libdl_libraries(proc_layer_name, vma.vm_start): + if link_map.l_addr in link_map_seen: + continue + + yield link_map + link_map_seen.add(link_map.l_addr) + + def get_task_libraries( + self, task: interfaces.objects.ObjectInterface + ) -> Tuple[int, str]: + """Get the task libraries from the ELF headers found within the memory maps + + Args: + task (task_struct): The reference task + + Yields: + Tuples with a ELF link map address and name + """ + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + return + + for elf_link_map in self.get_libdl_maps(task, proc_layer_name): + name = elf_link_map.get_name() + if not name: + continue + yield elf_link_map.l_addr, name + + def get_tasks_libraries( + self, + tasks: Iterable[interfaces.objects.ObjectInterface], + ) -> Iterable[Tuple[str, int, int, str]]: + """Get the task libraries from the ELF headers found within the memory maps for + all the tasks. + + Args: + tasks: An iterable of tasks + + Yields: + Tuples with a task name, task tgid, an ELF link map address and name + """ + for task in tasks: + task_name = utility.array_to_string(task.comm) + for linkmap_addr, linkmap_name in self.get_task_libraries(task): + yield task_name, task.tgid, linkmap_addr, linkmap_name + + def _format_fields(self, fields): + task_name, task_pid, addr, name = fields + return task_name, task_pid, format_hints.Hex(addr), name + + def _generator( + self, tasks: Iterable[interfaces.objects.ObjectInterface] + ) -> Iterable[Tuple[int, Tuple]]: + for fields in self.get_tasks_libraries(tasks): + yield 0, self._format_fields(fields) + + def run(self): + pids = self.config.get("pids") + pid_filter = pslist.PsList.create_pid_filter(pids) + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=pid_filter + ) + + headers = [ + ("Name", str), + ("Pid", int), + ("LoadAddress", format_hints.Hex), + ("Path", str), + ] + + return renderers.TreeGrid(headers, self._generator(tasks)) diff --git a/volatility3/framework/symbols/linux/elf.json b/volatility3/framework/symbols/linux/elf.json index 76cd8a2ec..e0a95bbba 100644 --- a/volatility3/framework/symbols/linux/elf.json +++ b/volatility3/framework/symbols/linux/elf.json @@ -270,8 +270,8 @@ "d_tag": { "offset": 0, "type": { - "kind": "base", - "name": "long long" + "kind": "enum", + "name": "DtypeEnum64" } }, "d_ptr": { @@ -699,8 +699,8 @@ "d_tag": { "offset": 0, "type": { - "kind": "base", - "name": "long" + "kind": "enum", + "name": "DtypeEnum32" } }, "d_ptr": { @@ -905,11 +905,139 @@ "PT_PHDR": 6, "PT_TLS": 7, "PT_LOOS": 1610612736, + "PT_GNU_EH_FRAME": 1685382480, + "PT_GNU_STACK": 1685382481, + "PT_GNU_RELRO": 1685382482, + "PT_GNU_PROPERTY": 1685382483, "PT_HIOS": 1879048191, "PT_LOWPROC": 1879048192, "PT_HIPROC": 2147483647 }, "size": 4 + }, + "DtypeEnum32": { + "base": "long", + "constants": { + "DT_NULL": 0, + "DT_NEEDED": 1, + "DT_PLTRELSZ": 2, + "DT_PLTGOT": 3, + "DT_HASH": 4, + "DT_STRTAB": 5, + "DT_SYMTAB": 6, + "DT_RELA": 7, + "DT_RELASZ": 8, + "DT_RELAENT": 9, + "DT_STRSZ": 10, + "DT_SYMENT": 11, + "DT_INIT": 12, + "DT_FINI": 13, + "DT_SONAME": 14, + "DT_RPATH": 15, + "DT_SYMBOLIC": 16, + "DT_REL": 17, + "DT_RELSZ": 18, + "DT_RELENT": 19, + "DT_PLTREL": 20, + "DT_DEBUG": 21, + "DT_TEXTREL": 22, + "DT_JMPREL": 23, + "DT_BIND_NOW": 24, + "DT_INIT_ARRAY": 25, + "DT_FINI_ARRAY": 26, + "DT_INIT_ARRAYSZ": 27, + "DT_FINI_ARRAYSZ": 28, + "DT_RUNPATH": 29, + "DT_FLAGS": 30, + "DT_ENCODING": 32, + "DT_PREINIT_ARRAYSZ": 33, + "DT_SYMTAB_SHNDX": 34, + "DT_RELRSZ": 35, + "DT_RELR": 36, + "DT_RELRENT": 37, + "DT_NUM": 38, + "OLD_DT_LOOS": 1610612736, + "DT_LOOS": 1610612749, + "DT_HIOS": 1879044096, + "DT_VALRNGLO": 1879047424, + "DT_VALRNGHI": 1879047679, + "DT_ADDRRNGLO": 1879047680, + "DT_GNU_HASH": 1879047925, + "DT_ADDRRNGHI": 1879047935, + "DT_VERSYM": 1879048176, + "DT_RELACOUNT": 1879048185, + "DT_RELCOUNT": 1879048186, + "DT_FLAGS_1": 1879048187, + "DT_VERDEF": 1879048188, + "DT_VERDEFNUM": 1879048189, + "DT_VERNEED": 1879048190, + "DT_VERNEEDNUM": 1879048191, + "DT_LOPROC": 1879048192, + "DT_HIPROC": 2147483647 + }, + "size": 4 + }, + "DtypeEnum64": { + "base": "long long", + "constants": { + "DT_NULL": 0, + "DT_NEEDED": 1, + "DT_PLTRELSZ": 2, + "DT_PLTGOT": 3, + "DT_HASH": 4, + "DT_STRTAB": 5, + "DT_SYMTAB": 6, + "DT_RELA": 7, + "DT_RELASZ": 8, + "DT_RELAENT": 9, + "DT_STRSZ": 10, + "DT_SYMENT": 11, + "DT_INIT": 12, + "DT_FINI": 13, + "DT_SONAME": 14, + "DT_RPATH": 15, + "DT_SYMBOLIC": 16, + "DT_REL": 17, + "DT_RELSZ": 18, + "DT_RELENT": 19, + "DT_PLTREL": 20, + "DT_DEBUG": 21, + "DT_TEXTREL": 22, + "DT_JMPREL": 23, + "DT_BIND_NOW": 24, + "DT_INIT_ARRAY": 25, + "DT_FINI_ARRAY": 26, + "DT_INIT_ARRAYSZ": 27, + "DT_FINI_ARRAYSZ": 28, + "DT_RUNPATH": 29, + "DT_FLAGS": 30, + "DT_ENCODING": 32, + "DT_PREINIT_ARRAYSZ": 33, + "DT_SYMTAB_SHNDX": 34, + "DT_RELRSZ": 35, + "DT_RELR": 36, + "DT_RELRENT": 37, + "DT_NUM": 38, + "OLD_DT_LOOS": 1610612736, + "DT_LOOS": 1610612749, + "DT_HIOS": 1879044096, + "DT_VALRNGLO": 1879047424, + "DT_VALRNGHI": 1879047679, + "DT_ADDRRNGLO": 1879047680, + "DT_GNU_HASH": 1879047925, + "DT_ADDRRNGHI": 1879047935, + "DT_VERSYM": 1879048176, + "DT_RELACOUNT": 1879048185, + "DT_RELCOUNT": 1879048186, + "DT_FLAGS_1": 1879048187, + "DT_VERDEF": 1879048188, + "DT_VERDEFNUM": 1879048189, + "DT_VERNEED": 1879048190, + "DT_VERNEEDNUM": 1879048191, + "DT_LOPROC": 1879048192, + "DT_HIPROC": 2147483647 + }, + "size": 8 } }, "base_types": { @@ -958,7 +1086,7 @@ }, "metadata": { "producer": { - "version": "0.0.1", + "version": "0.0.2", "name": "ikelos-by-hand", "datetime": "2019-10-21T22:52:00" }, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d73d0cfb9..0a3db9fcd 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -7,14 +7,13 @@ import logging import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple, List -from volatility3.framework import constants +from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES from volatility3.framework.constants.linux import CAPABILITIES -from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index fe85b194f..4b2b29b54 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -6,6 +6,10 @@ from typing import Dict, Tuple import logging from volatility3.framework import constants +from volatility3.framework.constants.linux import ( + ELF_IDENT, + ELF_CLASS, +) from volatility3.framework import objects, interfaces, exceptions vollog = logging.getLogger(__name__) @@ -59,13 +63,15 @@ class elf(objects.StructType): ei_class = self._context.object( symbol_table_name + constants.BANG + "unsigned char", layer_name=layer_name, - offset=object_info.offset + 0x4, + offset=object_info.offset + ELF_IDENT.EI_CLASS, ) - if ei_class == 1: + if ei_class == ELF_CLASS.ELFCLASS32: self._type_prefix = "Elf32_" - elif ei_class == 2: + self._ei_class_size = 32 + elif ei_class == ELF_CLASS.ELFCLASS64: self._type_prefix = "Elf64_" + self._ei_class_size = 64 else: raise ValueError(f"Unsupported ei_class value {ei_class}") @@ -140,36 +146,103 @@ class elf(objects.StructType): ) return section_headers + def get_link_maps(self, kernel_symbol_table_name): + """Get the ELF link map objects for the given VMA address + + Args: + kernel_symbol_table_name (str): Kernel symbol table name + + Yields: + The ELF link map objects + """ + got_entry_size = self._ei_class_size // 8 + + elf_symbol_table = self.get_symbol_table_name() + + link_maps_seen = set() + for phdr in self.get_program_headers(): + try: + if phdr.p_type.description != "PT_DYNAMIC": + continue + except ValueError: + continue + + for dsec in phdr.dynamic_sections(): + try: + if dsec.d_tag.description != "DT_PLTGOT": + continue + except ValueError: + continue + + got_start = dsec.d_ptr + + # link_map is stored at the second GOT entry + link_map_addr = got_start + got_entry_size + + # It needs the kernel symbol table to create a pointer + link_map_ptr = self._context.object( + kernel_symbol_table_name + constants.BANG + "pointer", + offset=link_map_addr, + layer_name=self.vol.layer_name, + ) + if not link_map_ptr: + continue + + linkmap_symname = ( + elf_symbol_table + constants.BANG + self._type_prefix + "LinkMap" + ) + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map_ptr, + layer_name=self.vol.layer_name, + ) + + while link_map and link_map.vol.offset != 0: + if link_map.vol.offset in link_maps_seen: + break + link_maps_seen.add(link_map.vol.offset) + + yield link_map + + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map.l_next, + layer_name=self.vol.layer_name, + ) + def _find_symbols(self): dt_strtab = None dt_symtab = None dt_strent = None for phdr in self.get_program_headers(): + # Find PT_DYNAMIC segment try: - # Find PT_DYNAMIC segment - if str(phdr.p_type.description) != "PT_DYNAMIC": + if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: - # If the p_type value is outside the ones declared in the enumeration, an - # exception is raised - return None + continue # This section contains pointers to the strtab, symtab, and strent sections for dsec in phdr.dynamic_sections(): - if dsec.d_tag == 5: + try: + dtag = dsec.d_tag.description + except ValueError: + continue + + if dtag == "DT_STRTAB": dt_strtab = dsec.d_ptr - elif dsec.d_tag == 6: + elif dtag == "DT_SYMTAB": dt_symtab = dsec.d_ptr - elif dsec.d_tag == 11: + elif dtag == "DT_SYMENT": # Size of the symtab symbol entry dt_strent = dsec.d_ptr break - if dt_strtab is None or dt_symtab is None or dt_strent is None: + if not (dt_strtab and dt_symtab and dt_strent): return None self._cached_symtab = dt_symtab @@ -274,15 +347,18 @@ class elf_phdr(objects.StructType): def get_vaddr(self): offset = self.__getattr__("p_vaddr") - if self._parent_e_type == 3: # ET_DYN - offset = self._parent_offset + offset + try: + if self._parent_e_type.description == "ET_DYN": + offset = self._parent_offset + offset + except ValueError: + pass return offset def dynamic_sections(self): # sanity check try: - if str(self.p_type.description) != "PT_DYNAMIC": + if self.p_type.description != "PT_DYNAMIC": return None except ValueError: # If the value is outside the ones declared in the enumeration, an @@ -314,10 +390,26 @@ class elf_phdr(objects.StructType): break +class elf_linkmap(objects.StructType): + def get_name(self): + try: + buf = self._context.layers.read(self.vol.layer_name, self.l_name, 256) + except exceptions.PagedInvalidAddressException: + # Protection against memory smear + return None + + idx = buf.find(b"\x00") + if idx != -1: + buf = buf[:idx] + return buf.decode() + + class_types = { "Elf": elf, "Elf64_Phdr": elf_phdr, "Elf32_Phdr": elf_phdr, "Elf32_Sym": elf_sym, "Elf64_Sym": elf_sym, + "Elf32_LinkMap": elf_linkmap, + "Elf64_LinkMap": elf_linkmap, } From 6e7b5b59416a0f0830660d6415144ec8ee0039ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 19:41:30 +1100 Subject: [PATCH 090/979] Linux: Add linux_library_list test case --- test/test_volatility.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index aaad615bc..7b151fd6c 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -6,6 +6,7 @@ # import os +import re import subprocess import sys import shutil @@ -331,6 +332,32 @@ def test_linux_tty_check(image, volatility, python): assert rc == 0 +def test_linux_library_list(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.library_list.LibraryList", image, volatility, python + ) + + assert re.search( + rb"NetworkManager\s2363\s0x7f52cdda0000\s/lib/x86_64-linux-gnu/libnss_files.so.2", + out, + ) + assert re.search( + rb"gnome-settings-\s3807\s0x7f7e660b5000\s/lib/x86_64-linux-gnu/libbz2.so.1.0", + out, + ) + assert re.search( + rb"gdu-notificatio\s3878\s0x7f25ce33e000\s/usr/lib/x86_64-linux-gnu/libXau.so.6", + out, + ) + assert re.search( + rb"bash\s8600\s0x7fe78a85f000\s/lib/x86_64-linux-gnu/libnss_files.so.2", + out, + ) + + assert out.count(b"\n") >= 2677 + assert rc == 0 + + # MAC From cf81ceda8262b1bdfc528277cd405c24d572e320 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 20:20:25 +1100 Subject: [PATCH 091/979] Fix CodeQL suggestion --- volatility3/framework/symbols/linux/extensions/elf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 4b2b29b54..828370fe8 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -351,6 +351,8 @@ class elf_phdr(objects.StructType): if self._parent_e_type.description == "ET_DYN": offset = self._parent_offset + offset except ValueError: + # Unknown ELF object file type. Anyway, if the ELF object file type is not a + # shared object (ET_DYN), the virtual address is 'p_vaddr'. pass return offset From cd08cb95fdf9d2503162020694fd4e3eddaf51d3 Mon Sep 17 00:00:00 2001 From: Alejandro Diego Date: Tue, 6 Feb 2024 09:26:49 -0500 Subject: [PATCH 092/979] Windows: Add filtering by offset to psscan Add capability to the psscan plugin to filter by specific offset. The filter would be used by other plugins to find specific offset using the psscan capabilities, like find hidden processes as a result of some dkom for example. The `--offset` flag argument should represent a physical address space. The flag is included in the following plugins: - dlllist - handles --- .../framework/plugins/windows/dlllist.py | 37 +++++++++++++---- .../framework/plugins/windows/handles.py | 41 ++++++++++++++----- .../framework/plugins/windows/psscan.py | 40 +++++++++++++++++- 3 files changed, 97 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index d73cea652..27eddc991 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -13,7 +13,7 @@ from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins import timeliner -from volatility3.plugins.windows import info, pslist +from volatility3.plugins.windows import info, pslist, psscan vollog = logging.getLogger(__name__) @@ -36,6 +36,9 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="psscan", component=psscan.PsScan, version=(1, 1, 0) + ), requirements.VersionRequirement( name="info", component=info.Info, version=(1, 0, 0) ), @@ -45,6 +48,11 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): description="Process IDs to include (all other processes are excluded)", optional=True, ), + requirements.IntRequirement( + name="offset", + description="Process offset in the physical address space", + optional=True, + ), requirements.BooleanRequirement( name="dump", description="Extract listed DLLs", @@ -221,6 +229,24 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) kernel = self.context.modules[self.config["kernel"]] + if self.config["offset"]: + procs = psscan.PsScan.scan_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=psscan.PsScan.create_offset_filter( + self.context.layers[kernel.layer_name], + self.config["offset"], + ), + ) + else: + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + return renderers.TreeGrid( [ ("PID", int), @@ -232,12 +258,5 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ("LoadTime", datetime.datetime), ("File output", str), ], - self._generator( - pslist.PsList.list_processes( - context=self.context, - layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name, - filter_func=filter_func, - ) - ), + self._generator(procs=procs), ) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index ddd9cb78e..2c5fde1c9 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -9,7 +9,7 @@ from volatility3.framework import constants, exceptions, renderers, interfaces, from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist +from volatility3.plugins.windows import pslist, psscan vollog = logging.getLogger(__name__) @@ -43,14 +43,22 @@ class Handles(interfaces.plugins.PluginInterface): description="Windows kernel", architectures=["Intel32", "Intel64"], ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="psscan", component=psscan.PsScan, version=(1, 1, 0) + ), requirements.ListRequirement( name="pid", element_type=int, description="Process IDs to include (all other processes are excluded)", optional=True, ), - requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + requirements.IntRequirement( + name="offset", + description="Process offset in the physical address space", + optional=True, ), ] @@ -416,6 +424,24 @@ class Handles(interfaces.plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) kernel = self.context.modules[self.config["kernel"]] + if self.config["offset"]: + procs = psscan.PsScan.scan_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=psscan.PsScan.create_offset_filter( + self.context.layers[kernel.layer_name], + self.config["offset"], + ), + ) + else: + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + return renderers.TreeGrid( [ ("PID", int), @@ -426,12 +452,5 @@ class Handles(interfaces.plugins.PluginInterface): ("GrantedAccess", format_hints.Hex), ("Name", str), ], - self._generator( - pslist.PsList.list_processes( - self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func=filter_func, - ) - ), + self._generator(procs=procs), ) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 3d9ae5c1e..8b3afaf7c 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Iterable, Callable, Optional, Tuple +from typing import Iterable, Callable, List, Optional, Tuple from volatility3.framework import renderers, interfaces, layers, exceptions from volatility3.framework.configuration import requirements @@ -59,6 +59,44 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] + @classmethod + def create_offset_filter( + cls, + memory: interfaces.layers.DataLayerInterface, + offset: int = None, + exclude: bool = False, + ) -> Callable[[interfaces.objects.ObjectInterface], bool]: + """A factory for producing filter functions that filter based on the physical offset of the process. + + Args: + offset: A number that is the physical offset to be filtered out + memory: Memory object needed to do the offset mapping to physical. + exclude: Accept only tasks that are not the offset argument + Returns: + Filter function to be passed to the list of processes. + """ + if not isinstance(memory, interfaces.layers.DataLayerInterface): + raise TypeError("memory object requires an instance of DataLayerInterface") + + filter_func = lambda _: False + + # return physical offset in tuple -> (_, _, physical_offset, _, _) + # from the first item in the memory mapping list + virtual_to_physical_offset = lambda virtual_offset, memory: list( + memory.mapping(offset=virtual_offset, length=0) + )[0][2] + + if offset: + if exclude: + filter_func = ( + lambda x: virtual_to_physical_offset(x.vol.offset, memory) == offset + ) + else: + filter_func = ( + lambda x: virtual_to_physical_offset(x.vol.offset, memory) != offset + ) + return filter_func + @classmethod def scan_processes( cls, From 437a91375db004de5d0df26443d1bd71ee9f2b94 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 20 Feb 2024 06:39:42 +0000 Subject: [PATCH 093/979] Linux: add TODO for mm_struct for methods that should be removed when moving to version 3.0.0 --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f9db30498..27113965f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -448,6 +448,7 @@ class maple_tree(objects.StructType): class mm_struct(objects.StructType): + # TODO: As of version 3.0.0 this method should be removed def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """ Deprecated: Use either get_vma_iter() or _get_mmap_iter(). @@ -478,6 +479,7 @@ class mm_struct(objects.StructType): seen.add(link.vol.offset) link = link.vm_next + # TODO: As of version 3.0.0 this method should be removed def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """ Deprecated: Use either get_vma_iter() or _get_maple_tree_iter(). From a597ee17768b1513f8587642c0d00c9170007f46 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 20 Feb 2024 06:48:16 +0000 Subject: [PATCH 094/979] Linux: update minor version number due to API changes with linux extension for mm_struct --- volatility3/framework/constants/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09dded076..9aaafb933 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_MINOR = 6 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 5e38ffa811bcf385c2b470351ed2518118c7d875 Mon Sep 17 00:00:00 2001 From: Alejandro Diego Date: Tue, 20 Feb 2024 15:07:15 -0500 Subject: [PATCH 095/979] Added support for virtual address filtering and validation of ph/v space --- .../framework/plugins/windows/dlllist.py | 3 +- .../framework/plugins/windows/handles.py | 3 +- .../framework/plugins/windows/psscan.py | 67 ++++++++++++++----- 3 files changed, 53 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 27eddc991..f876dc1a2 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -235,7 +235,8 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.layer_name, kernel.symbol_table_name, filter_func=psscan.PsScan.create_offset_filter( - self.context.layers[kernel.layer_name], + self.context, + kernel.layer_name, self.config["offset"], ), ) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 2c5fde1c9..d43d26ef1 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -430,7 +430,8 @@ class Handles(interfaces.plugins.PluginInterface): kernel.layer_name, kernel.symbol_table_name, filter_func=psscan.PsScan.create_offset_filter( - self.context.layers[kernel.layer_name], + self.context, + kernel.layer_name, self.config["offset"], ), ) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 8b3afaf7c..82a481070 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -59,42 +59,73 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] + @classmethod + def physical_offset_from_virtual(cls, context, layer_name, proc): + """Calculate the physical offset from the virtual offset of a process. + + Args: + context: The context containing layers and modules information. + layer_name: The name of the layer containing the process memory. + proc: The process object for which to calculate the physical offset. + + Returns: + int: The physical offset of the process. + Raises: + TypeError: If the primary layer is not an Intel layer. + """ + memory = context.layers[layer_name] + + if not isinstance(memory, layers.intel.Intel): + raise TypeError("Primary layer is not an intel layer") + + (_, _, ph_offset, _, _) = list( + memory.mapping(offset=proc.vol.offset, length=0) + )[0] + + return ph_offset + @classmethod def create_offset_filter( cls, - memory: interfaces.layers.DataLayerInterface, + context: interfaces.context.ContextInterface, + layer_name: str, offset: int = None, + physical: bool = True, exclude: bool = False, ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on the physical offset of the process. Args: offset: A number that is the physical offset to be filtered out - memory: Memory object needed to do the offset mapping to physical. exclude: Accept only tasks that are not the offset argument + Returns: Filter function to be passed to the list of processes. """ - if not isinstance(memory, interfaces.layers.DataLayerInterface): - raise TypeError("memory object requires an instance of DataLayerInterface") - filter_func = lambda _: False - # return physical offset in tuple -> (_, _, physical_offset, _, _) - # from the first item in the memory mapping list - virtual_to_physical_offset = lambda virtual_offset, memory: list( - memory.mapping(offset=virtual_offset, length=0) - )[0][2] - if offset: - if exclude: - filter_func = ( - lambda x: virtual_to_physical_offset(x.vol.offset, memory) == offset - ) + if physical: + if exclude: + filter_func = ( + lambda proc: cls.physical_offset_from_virtual( + context, layer_name, proc + ) + == offset + ) + else: + filter_func = ( + lambda proc: cls.physical_offset_from_virtual( + context, layer_name, proc + ) + != offset + ) else: - filter_func = ( - lambda x: virtual_to_physical_offset(x.vol.offset, memory) != offset - ) + if exclude: + lambda proc: proc.vol.offset == offset + else: + lambda proc: proc.vol.offset != offset + return filter_func @classmethod From 42c6a9ede6bbe3ff52b868144516679753c9fc8c Mon Sep 17 00:00:00 2001 From: Alejandro Diego Date: Tue, 20 Feb 2024 20:07:17 -0500 Subject: [PATCH 096/979] Remove unnecessary import and correctly filter_func var --- volatility3/framework/plugins/windows/psscan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 82a481070..5634298a8 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Iterable, Callable, List, Optional, Tuple +from typing import Iterable, Callable, Optional, Tuple from volatility3.framework import renderers, interfaces, layers, exceptions from volatility3.framework.configuration import requirements @@ -122,9 +122,9 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) else: if exclude: - lambda proc: proc.vol.offset == offset + filter_func = lambda proc: proc.vol.offset == offset else: - lambda proc: proc.vol.offset != offset + filter_func = lambda proc: proc.vol.offset != offset return filter_func From e10cff979af3917b3d0a67ec050c181e37bffed8 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 21 Feb 2024 12:50:05 +0000 Subject: [PATCH 097/979] Linux: update module extension with memoization for _mod_mem_type --- .../symbols/linux/extensions/__init__.py | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3673b66df..8d11374c9 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -26,27 +26,33 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._mod_mem_type = None # Initialize _mod_mem_type to None for memoization + @property def mod_mem_type(self): - """Return the mod_mem_type enum choices if available or None if not""" - if not self.has_member("_mod_mem_type"): + """Return the mod_mem_type enum choices if available or an empty dict if not""" + # mod_mem_type and module_memory were added in kernel 6.4 which replaces + # module_layout for storing the information around core_layout etc. + # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information + + if self._mod_mem_type is None: try: - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type( - self._context, self - ) - # mod_mem_type and module_memory were added in kernel 6.4 which replaces - # module_layout for storing the information around core_layout etc. - # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information - self._mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices + self._mod_mem_type = self._context.symbol_space.get_enumeration( + self.get_symbol_table_name() + constants.BANG + "mod_mem_type" + ).choices except exceptions.SymbolError: vollog.debug( - f"Unable to find mod_mem_type enum. This is expected on kernels <6.4 but may cause issues with later kernels" + f"Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" ) - self._mod_mem_type = None + # set to empty dict to show that the enum was not found, and so shouldn't be searched for again + self._mod_mem_type = {} return self._mod_mem_type def get_module_base(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base except: @@ -61,7 +67,7 @@ class module(generic.GenericIntelProcess): raise AttributeError("module -> get_module_base: Unable to get module base") def get_init_size(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size @@ -82,7 +88,7 @@ class module(generic.GenericIntelProcess): ) def get_core_size(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_TEXT"]].size @@ -105,7 +111,7 @@ class module(generic.GenericIntelProcess): ) def get_module_core(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base except KeyError: @@ -120,7 +126,7 @@ class module(generic.GenericIntelProcess): raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base except KeyError: From 3b5fde94208bbfc98c7b2a147cdd2341427b7097 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 21 Feb 2024 12:58:24 +0000 Subject: [PATCH 098/979] Linux: update module extension logic for choosing the correct values for different kernel versions --- .../symbols/linux/extensions/__init__.py | 60 +++++++++---------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 8d11374c9..751c80cb5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -52,43 +52,41 @@ class module(generic.GenericIntelProcess): return self._mod_mem_type def get_module_base(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base - except: + except KeyError: raise AttributeError( "module -> get_module_base: Unable to get module base. Cannot read base from MOD_TEXT." ) - else: - if self.has_member("core_layout"): - return self.core_layout.base - elif self.has_member("module_core"): - return self.module_core + elif self.has_member("core_layout"): + return self.core_layout.base + elif self.has_member("module_core"): + return self.module_core raise AttributeError("module -> get_module_base: Unable to get module base") def get_init_size(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size ) - except: + except KeyError: raise AttributeError( "module -> get_init_size: Unable to determine .init section size of module. Cannot read size of MOD_INIT_TEXT, MOD_INIT_DATA, and MOD_INIT_RODATA" ) - else: - if self.has_member("init_layout"): - return self.init_layout.size - elif self.has_member("init_size"): - return self.init_size + elif self.has_member("init_layout"): + return self.init_layout.size + elif self.has_member("init_size"): + return self.init_size raise AttributeError( "module -> get_init_size: Unable to determine .init section size of module" ) def get_core_size(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_TEXT"]].size @@ -100,44 +98,40 @@ class module(generic.GenericIntelProcess): raise AttributeError( "module -> get_core_size: Unable to determine core size of module. Cannot read size of MOD_TEXT, MOD_DATA, MOD_RODATA, and MOD_RO_AFTER_INIT." ) - - else: - if self.has_member("core_layout"): - return self.core_layout.size - elif self.has_member("core_size"): - return self.core_size + elif self.has_member("core_layout"): + return self.core_layout.size + elif self.has_member("core_size"): + return self.core_size raise AttributeError( "module -> get_core_size: Unable to determine core size of module" ) def get_module_core(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base except KeyError: raise AttributeError( "module -> get_module_core: Unable to get module core. Cannot read base from MOD_TEXT." ) - else: - if self.has_member("core_layout"): - return self.core_layout.base - elif self.has_member("module_core"): - return self.module_core + elif self.has_member("core_layout"): + return self.core_layout.base + elif self.has_member("module_core"): + return self.module_core raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base except KeyError: raise AttributeError( "module -> get_module_core: Unable to get module init. Cannot read base from MOD_INIT_TEXT." ) - else: - if self.has_member("init_layout"): - return self.init_layout.base - elif self.has_member("module_init"): - return self.module_init + elif self.has_member("init_layout"): + return self.init_layout.base + elif self.has_member("module_init"): + return self.module_init raise AttributeError("module -> get_module_init: Unable to get module init") def get_name(self): From 77675d584d2e07df733c0700c8a3b7dd974252c0 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 21 Feb 2024 14:26:48 +0000 Subject: [PATCH 099/979] Update patch version --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 9aaafb933..7764dc44c 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 6 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From a82b93523de768b68fed529064113340920e52fe Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 22 Feb 2024 09:47:23 +0000 Subject: [PATCH 100/979] Linux: psscan tiny update so that memory_layer_name is reused --- volatility3/framework/plugins/linux/psscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 462577e58..40784a647 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -28,7 +28,7 @@ class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -139,7 +139,7 @@ class PsScan(interfaces.plugins.PluginInterface): kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" ) memory_layer_name = kernel_layer.dependencies[0] - memory_layer = context.layers[kernel_layer.dependencies[0]] + memory_layer = context.layers[memory_layer_name] # scan the memory_layer for these needles for address, _ in memory_layer.scan( From 0899ba8d7fedece67b520c8a2a31c45868177ce1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 28 Feb 2024 23:09:46 +0000 Subject: [PATCH 101/979] Documentation: Fix up syntax errors involving * character --- volatility3/framework/interfaces/plugins.py | 2 +- volatility3/framework/plugins/linux/kmsg.py | 4 +++- volatility3/framework/plugins/linux/pslist.py | 6 +++--- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- volatility3/framework/plugins/windows/mftscan.py | 4 ++-- .../framework/symbols/linux/extensions/__init__.py | 8 ++++---- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 29395aadf..697e4cdc3 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -60,7 +60,7 @@ class FileHandlerInterface(io.RawIOBase): @staticmethod def sanitize_filename(filename: str) -> str: """Sanititizes the filename to ensure only a specific whitelist of characters is allowed through""" - allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]\{\}!$%^:#~?<>,|" + allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]{}!$%^:#~?<>,|" result = "" for char in filename: if char in allowed: diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index dd707a7ff..c5e0fc302 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -198,7 +198,9 @@ class ABCKmsg(ABC): class Kmsg_pre_3_5(ABCKmsg): """The kernel ring buffer (log_buf) is a char array that sequentially stores log lines, each separated by newline (LF) characters. i.e: - <6>[ 9565.250411] line1!\n<6>[ 9565.250412] line2\n... + + <6>[ 9565.250411] line1!\\n<6>[ 9565.250412] line2\\n... + """ @classmethod diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 771040dc0..046ee43d8 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -83,11 +83,11 @@ class PsList(interfaces.plugins.PluginInterface): cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False ) -> Tuple[int, int, int, str]: """Extract the fields needed for the final output + Args: task: A task object from where to get the fields. - decorate_comm: If True, it decorates the comm string of - - User threads: in curly brackets, - - Kernel threads: in square brackets + decorate_comm: If True, it decorates the comm string of user threads in curly brackets, + and of Kernel threads in square brackets. Defaults to False. Returns: A tuple with the fields to show in the plugin output. diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index e9c98a227..78217fbec 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -83,7 +83,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock: Kernel generic `sock` object Returns a tuple with: - sock: The respective kernel's \*_sock object for that socket family + sock: The respective kernel's \\*_sock object for that socket family sock_stat: A tuple with the source and destination (address and port) along with its state string socket_filter: A dictionary with information about the socket filter """ @@ -501,7 +501,7 @@ class Sockstat(plugins.PluginInterface): family: Socket family string (AF_UNIX, AF_INET, etc) sock_type: Socket type string (STREAM, DGRAM, etc) protocol: Protocol string (UDP, TCP, etc) - sock_fields: A tuple with the \*_sock object, the sock stats and the extended info dictionary + sock_fields: A tuple with the \\*_sock object, the sock stats and the extended info dictionary """ vmlinux = context.modules[symbol_table] diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 91a2e9152..85c072037 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -38,7 +38,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\*|BAAD/"} + {"yara_rules": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File @@ -197,7 +197,7 @@ class ADS(interfaces.plugins.PluginInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\*|BAAD/"} + {"yara_rules": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 751c80cb5..588ce3874 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1137,17 +1137,17 @@ class vfsmount(objects.StructType): """Helper to make sure it is comparing two pointers to 'vfsmount'. Depending on the kernel version, the calling object (self) could be - a 'vfsmount \*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust + a 'vfsmount \\*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust in the framework "auto" dereferencing ability to assure that when we reach this point 'self' will be a 'vfsmount' already and self.vol.offset - a 'vfsmount \*' and not a 'vfsmount \*\*'. The argument must be a 'vfsmount \*'. + a 'vfsmount \\*' and not a 'vfsmount \\*\\*'. The argument must be a 'vfsmount \\*'. Typically, it's called from do_get_path(). Args: - vfsmount_ptr (vfsmount \*): A pointer to a 'vfsmount' + vfsmount_ptr (vfsmount *): A pointer to a 'vfsmount' Raises: - exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \*' + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \\*' Returns: bool: 'True' if the given argument points to the the same 'vfsmount' From 338106dfe8e0667a07b0ab0ba4d52fbf5f4d51e7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 14:46:35 +1100 Subject: [PATCH 102/979] Move the memory page parameters to the Intel layer --- volatility3/framework/constants/linux/__init__.py | 5 ----- volatility3/framework/layers/intel.py | 12 ++++++++++++ volatility3/framework/plugins/linux/elfs.py | 14 +++++--------- .../framework/symbols/linux/extensions/__init__.py | 2 +- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 5e82e580e..3eabc2341 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -9,11 +9,6 @@ from enum import IntEnum KERNEL_NAME = "__kernel__" -# arch/x86/include/asm/page_types.h -PAGE_SHIFT = 12 -PAGE_SIZE = 1 << PAGE_SHIFT -PAGE_MASK = ~(PAGE_SIZE - 1) - """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" # include/linux/sched.h diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index ae477854d..75e561b33 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -67,6 +67,12 @@ class Intel(linear.LinearlyMappedLayer): math.ceil(math.log2(struct.calcsize(self._entry_format))) ) + @classproperty + @functools.lru_cache() + def page_shift(cls) -> int: + """Page shift for the intel memory layers.""" + return cls._page_size_in_bits + @classproperty @functools.lru_cache() def page_size(cls) -> int: @@ -76,6 +82,12 @@ class Intel(linear.LinearlyMappedLayer): """ return 1 << cls._page_size_in_bits + @classproperty + @functools.lru_cache() + def page_mask(cls) -> int: + """Page mask for the intel memory layers.""" + return ~(cls.page_size - 1) + @classproperty @functools.lru_cache() def bits_per_register(cls) -> int: diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 7171a6616..43cd6bb8b 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -14,11 +14,7 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux.extensions import elf -from volatility3.framework.constants.linux import ( - PAGE_SIZE, - PAGE_MASK, - ELF_MAX_EXTRACTION_SIZE, -) +from volatility3.framework.constants.linux import ELF_MAX_EXTRACTION_SIZE from volatility3.plugins.linux import pslist @@ -106,11 +102,11 @@ class Elfs(plugins.PluginInterface): # Use complete memory pages for dumping # If start isn't a multiple of a page, stick to the highest multiple < start # If end isn't a multiple of a page, stick to the lowest multiple > end - if start % PAGE_SIZE: - start = start & PAGE_MASK + if start % proc_layer.page_size: + start = start & proc_layer.page_mask - if end % PAGE_SIZE: - end = (end & PAGE_MASK) + PAGE_SIZE + if end % proc_layer.page_size: + end = (end & proc_layer.page_mask) + proc_layer.page_size real_size = end - start diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0a3db9fcd..1faafc267 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -640,7 +640,7 @@ class vm_area_struct(objects.StructType): elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True elif proclayer and "x" in flags_str: - for i in range(self.vm_start, self.vm_end, 1 << constants.linux.PAGE_SHIFT): + for i in range(self.vm_start, self.vm_end, proclayer.page_size): try: if proclayer.is_dirty(i): vollog.warning( From d5a0543a2d2d950c6743455f3f0ffa0b5afc2097 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 15:34:20 +1100 Subject: [PATCH 103/979] Use the ELF class constant instead of hardcoding a value. Fixed some f-strings --- volatility3/framework/layers/xen.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index 927b30430..e7aa0ccec 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -5,6 +5,7 @@ from typing import Optional from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.layers import elf from volatility3.framework.symbols import intermed +from volatility3.framework.constants.linux import ELF_CLASS vollog = logging.getLogger(__name__) @@ -14,7 +15,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): _header_struct = struct.Struct(" Date: Thu, 29 Feb 2024 15:34:48 +1100 Subject: [PATCH 104/979] Update author and modification time --- volatility3/framework/symbols/linux/elf.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/elf.json b/volatility3/framework/symbols/linux/elf.json index e0a95bbba..79a96e07a 100644 --- a/volatility3/framework/symbols/linux/elf.json +++ b/volatility3/framework/symbols/linux/elf.json @@ -1087,8 +1087,8 @@ "metadata": { "producer": { "version": "0.0.2", - "name": "ikelos-by-hand", - "datetime": "2019-10-21T22:52:00" + "name": "gcmoreira-by-hand", + "datetime": "2024-02-19T14:37:00" }, "format": "6.1.0" } From 9b0915dc85470df78bc739148093bb72d659297c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 15:36:46 +1100 Subject: [PATCH 105/979] Add logging for unknown ELF types --- volatility3/framework/layers/elf.py | 4 ++ volatility3/framework/plugins/linux/elfs.py | 4 ++ .../framework/symbols/linux/extensions/elf.py | 66 ++++++++++++++++--- 3 files changed, 64 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index 6bd5c2d63..81f3c3634 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -55,6 +55,10 @@ class Elf64Layer(segmented.SegmentedLayer): try: ptype = phdr.p_type.description except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue if ( diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 43cd6bb8b..6820576dc 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -93,6 +93,10 @@ class Elfs(plugins.PluginInterface): if phdr.p_type.description != "PT_LOAD": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue start = phdr.p_vaddr diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 828370fe8..2cf5c3d4e 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -165,6 +165,10 @@ class elf(objects.StructType): if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue for dsec in phdr.dynamic_sections(): @@ -172,6 +176,10 @@ class elf(objects.StructType): if dsec.d_tag.description != "DT_PLTGOT": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF dynamic section type: {dsec.d_tag}", + ) continue got_start = dsec.d_ptr @@ -186,16 +194,27 @@ class elf(objects.StructType): layer_name=self.vol.layer_name, ) if not link_map_ptr: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid ELF link map pointer at 0x{link_map_addr:x}", + ) continue linkmap_symname = ( elf_symbol_table + constants.BANG + self._type_prefix + "LinkMap" ) - link_map = self._context.object( - object_type=linkmap_symname, - offset=link_map_ptr, - layer_name=self.vol.layer_name, - ) + try: + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map_ptr, + layer_name=self.vol.layer_name, + ) + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid ELF link map address at 0x{link_map_ptr:x}", + ) + continue while link_map and link_map.vol.offset != 0: if link_map.vol.offset in link_maps_seen: @@ -204,11 +223,18 @@ class elf(objects.StructType): yield link_map - link_map = self._context.object( - object_type=linkmap_symname, - offset=link_map.l_next, - layer_name=self.vol.layer_name, - ) + try: + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map.l_next, + layer_name=self.vol.layer_name, + ) + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + f"ELF link map linked list is corrupt at 0x{self.vol.offset:x}", + ) + break def _find_symbols(self): dt_strtab = None @@ -221,6 +247,10 @@ class elf(objects.StructType): if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue # This section contains pointers to the strtab, symtab, and strent sections @@ -228,6 +258,10 @@ class elf(objects.StructType): try: dtag = dsec.d_tag.description except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF dynamic section type: {dsec.d_tag}", + ) continue if dtag == "DT_STRTAB": @@ -353,6 +387,10 @@ class elf_phdr(objects.StructType): except ValueError: # Unknown ELF object file type. Anyway, if the ELF object file type is not a # shared object (ET_DYN), the virtual address is 'p_vaddr'. + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF object type: {self._parent_e_type}", + ) pass return offset @@ -365,6 +403,10 @@ class elf_phdr(objects.StructType): except ValueError: # If the value is outside the ones declared in the enumeration, an # exception is raised + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {self.p_type}", + ) return None # the buffer of array starts at elf_base + our virtual address ( offset ) @@ -398,6 +440,10 @@ class elf_linkmap(objects.StructType): buf = self._context.layers.read(self.vol.layer_name, self.l_name, 256) except exceptions.PagedInvalidAddressException: # Protection against memory smear + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid l_name address for ELF link map at 0x{self.vol.offset:x}", + ) return None idx = buf.find(b"\x00") From 3c9af096e15402a403b41b29a79ab3c74cc617f1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 16:17:47 +1100 Subject: [PATCH 106/979] Add missing PAGE_SHIFT replacement --- volatility3/framework/symbols/linux/extensions/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1faafc267..04cfa099f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -611,7 +611,8 @@ class vm_area_struct(objects.StructType): def get_page_offset(self) -> int: if self.vm_file == 0: return 0 - return self.vm_pgoff << constants.linux.PAGE_SHIFT + parent_layer = self._context.layers[self.vol.layer_name] + return self.vm_pgoff << parent_layer.page_shift def get_name(self, context, task): if self.vm_file != 0: From b681438ddcfdc33defa34d15cf48bd9ce4c9ca58 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 16:21:34 +1100 Subject: [PATCH 107/979] Remove unnecessary 'pass' statement. --- volatility3/framework/symbols/linux/extensions/elf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 2cf5c3d4e..eadcbbae0 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -391,7 +391,6 @@ class elf_phdr(objects.StructType): constants.LOGLEVEL_VVVV, f"Skipping unknown ELF object type: {self._parent_e_type}", ) - pass return offset From a8e10828273fb8cfa74743137ba70fd730e42619 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 29 Feb 2024 14:36:46 +0100 Subject: [PATCH 108/979] restore pslist, add queue_head_t type class --- volatility3/framework/plugins/mac/pslist.py | 8 ++------ volatility3/framework/symbols/mac/__init__.py | 4 +++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 9835644b8..9b570f3f9 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -49,9 +49,7 @@ class PsList(interfaces.plugins.PluginInterface): ] @classmethod - def get_list_tasks( - cls, method: str - ) -> Callable[ + def get_list_tasks(cls, method: str) -> Callable[ [interfaces.context.ContextInterface, str, Callable[[int], bool]], Iterable[interfaces.objects.ObjectInterface], ]: @@ -188,9 +186,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object( - object_type="queue_entry", offset=kernel.get_symbol("tasks").address - ) + queue_entry = kernel.object_from_symbol(symbol_name="tasks") seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index 56ac96633..be4fe8cd1 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -21,12 +21,14 @@ class MacKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("vm_map_object", extensions.vm_map_object) self.set_type_class("socket", extensions.socket) self.set_type_class("inpcb", extensions.inpcb) - self.set_type_class("queue_entry", extensions.queue_entry) self.set_type_class("ifnet", extensions.ifnet) self.set_type_class("sockaddr_dl", extensions.sockaddr_dl) self.set_type_class("sockaddr", extensions.sockaddr) self.set_type_class("sysctl_oid", extensions.sysctl_oid) self.set_type_class("kauth_scope", extensions.kauth_scope) + # https://developer.apple.com/documentation/kernel/queue_head_t + self.set_type_class("queue_entry", extensions.queue_entry) + self.set_type_class("queue_head_t", extensions.queue_entry) class MacUtilities(interfaces.configuration.VersionableInterface): From 850b377dcf8b087d5296774b0803965271cda9b7 Mon Sep 17 00:00:00 2001 From: hsarkey Date: Fri, 9 Feb 2024 15:21:41 -0500 Subject: [PATCH 109/979] Updated windows.dlllist plugin to have --name and --base flags for filtering on dll name and base address. --- .../framework/plugins/windows/dlllist.py | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index f876dc1a2..4f50bdda5 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -5,6 +5,7 @@ import contextlib import datetime import logging import ntpath +import re from typing import List, Optional, Type from volatility3.framework import constants, exceptions, interfaces, renderers @@ -22,7 +23,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the loaded modules in a particular windows memory image.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -53,6 +54,22 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): description="Process offset in the physical address space", optional=True, ), + requirements.StringRequirement( + name="name", + description="Specify a regular expression to match dll name(s)", + optional=True, + ), + requirements.IntRequirement( + name="base", + description="Specify a base address", + optional=True, + ), + requirements.BooleanRequirement( + name="ignore-case", + description="Specify case insensitivity for the regular expression name matching", + default=False, + optional=True, + ), requirements.BooleanRequirement( name="dump", description="Extract listed DLLs", @@ -144,9 +161,35 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): BaseDllName = FullDllName = renderers.UnreadableValue() with contextlib.suppress(exceptions.InvalidAddressException): BaseDllName = entry.BaseDllName.get_string() - # We assume that if the BaseDllName points to an invalid buffer, so will FullDllName + # We assume that if BaseDllName points to invalid buffer, so will FullDllName FullDllName = entry.FullDllName.get_string() + # Check if a name regex was passed and apply it to only show matches + if self.config["name"]: + try: + flags = re.I if self.config["ignore-case"] else 0 + mod_re = re.compile(self.config["name"], flags) + except re.error: + vollog.debug( + "Error parsing regular expression: %s", self.config["name"] + ) + return None + + # If Base or Full Dll Name are invalid, move on + if isinstance(BaseDllName, renderers.UnreadableValue) or isinstance( + FullDllName, renderers.UnreadableValue + ): + continue + + # If regex does not match, move on + if not mod_re.search(BaseDllName) and not mod_re.search( + FullDllName + ): + continue + + if self.config["base"] and self.config["base"] != entry.DllBase: + continue + if dll_load_time_field: # Versions prior to 6.1 won't have the LoadTime attribute # and 32bit version shouldn't have the Quadpart according to MSDN From d56ccbfd0b749d8f2ad69764658bf8f3856b8d21 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 14:18:30 +0100 Subject: [PATCH 110/979] allow specifying integers as 0x in ListRequirement --- volatility3/cli/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 3de7d8f8f..a67762a31 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,7 +827,11 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - additional["type"] = requirement.element_type + # Allow a list of integers, specified with convenient 0x hexadecimal format + if requirement.element_type == int: + additional["type"] = lambda x: int(x, 0) + else: + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 3040bc5fc21f31e27f75ac8c3ec359e43454d098 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 14:20:13 +0100 Subject: [PATCH 111/979] add --dump mechanism, similarly to linux.proc --- .../framework/plugins/mac/proc_maps.py | 173 +++++++++++++++++- 1 file changed, 169 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 781b3ed66..204d414e4 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -2,17 +2,23 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.mac import pslist +from typing import Callable, Generator, Type, Optional +import logging + +vollog = logging.getLogger(__name__) class Maps(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" _required_framework_version = (2, 0, 0) + _version = (1, 1, 0) + MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @classmethod def get_requirements(cls): @@ -31,14 +37,155 @@ class Maps(interfaces.plugins.PluginInterface): element_type=int, optional=True, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed memory segments", + default=False, + optional=True, + ), + requirements.ListRequirement( + name="address", + description="Process virtual memory addresses to include " + "(all other VMA sections are excluded). This can be any " + "virtual address within the VMA section. Virtual addresses " + "must be separated by a space.", + element_type=int, + optional=True, + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size for dumped VMA sections " + "(all the bigger sections will be ignored)", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), ] + @classmethod + def list_vmas( + cls, + task: interfaces.objects.ObjectInterface, + filter_func: Callable[ + [interfaces.objects.ObjectInterface], bool + ] = lambda _: True, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Lists the Virtual Memory Areas of a specific process. + + Args: + task: task object from which to list the vma + filter_func: Function to take a vma and return False if it should be filtered out + + Returns: + Yields vmas based on the task and filtered based on the filter function + """ + for vma in task.get_map_iter(): + if filter_func(vma): + yield vma + else: + vollog.debug( + f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.p_pid} due to filter_func" + ) + + @classmethod + def vma_dump( + cls, + context: interfaces.context.ContextInterface, + task: interfaces.objects.ObjectInterface, + vm_start: int, + vm_end: int, + open_method: Type[interfaces.plugins.FileHandlerInterface], + maxsize: int = MAXSIZE_DEFAULT, + ) -> Optional[interfaces.plugins.FileHandlerInterface]: + """Extracts the complete data for VMA as a FileInterface. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + task: an task_struct instance + vm_start: The start virtual address from the vma to dump + vm_end: The end virtual address from the vma to dump + open_method: class to provide context manager for opening the file + maxsize: Max size of VMA section (default MAXSIZE_DEFAULT) + + Returns: + An open FileInterface object containing the complete data for the task or None in the case of failure + """ + pid = task.p_pid + + try: + proc_layer_name = task.add_process_layer() + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + pid, excp.invalid_address, excp.layer_name + ) + ) + return None + vm_size = vm_end - vm_start + + # check if vm_size is negative, this should never happen. + if vm_size < 0: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is negative." + ) + return None + # check if vm_size is larger than the maxsize limit, and therefore is not saved out. + if maxsize <= vm_size: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is larger than maxsize limit of {maxsize}" + ) + return None + proc_layer = context.layers[proc_layer_name] + file_name = f"pid.{pid}.vma.{vm_start:#x}-{vm_end:#x}.dmp" + try: + file_handle = open_method(file_name) + chunk_size = 1024 * 1024 * 10 + offset = vm_start + while offset < vm_start + vm_size: + to_read = min(chunk_size, vm_start + vm_size - offset) + data = proc_layer.read(offset, to_read, pad=True) + file_handle.write(data) + offset += to_read + except Exception as excp: + vollog.debug(f"Unable to dump virtual memory {file_name}: {excp}") + return None + return file_handle + def _generator(self, tasks): + address_list = self.config.get("address", None) + if not address_list: + # do not filter as no address_list was supplied + vma_filter_func = lambda _: True + else: + # filter for any vm_start that matches the supplied address config + def vma_filter_function(task: interfaces.objects.ObjectInterface) -> bool: + addrs_in_vma = [ + addr + for addr in address_list + if task.links.start <= addr <= task.links.end + ] + + # if any of the user supplied addresses would fall within this vma return true + if addrs_in_vma: + return True + else: + return False + + vma_filter_func = vma_filter_function + for task in tasks: process_name = utility.array_to_string(task.p_comm) process_pid = task.p_pid - for vma in task.get_map_iter(): + for vma in self.list_vmas(task, filter_func=vma_filter_func): + try: + vm_start = vma.links.start + vm_end = vma.links.end + except AttributeError: + vollog.debug( + f"Unable to find the vm_start and vm_end for vma at {vma.vol.offset:#x} for pid {process_pid}" + ) + continue + path = vma.get_path( self.context, self.context.modules[self.config["kernel"]].symbol_table_name, @@ -46,15 +193,32 @@ class Maps(interfaces.plugins.PluginInterface): if path == "": path = vma.get_special_path() + file_output = "Disabled" + if self.config["dump"]: + file_output = "Error outputting file" + file_handle = self.vma_dump( + self.context, + task, + vm_start, + vm_end, + self.open, + self.config["maxsize"], + ) + + if file_handle: + file_handle.close() + file_output = file_handle.preferred_filename + yield ( 0, ( process_pid, process_name, - format_hints.Hex(vma.links.start), - format_hints.Hex(vma.links.end), + format_hints.Hex(vm_start), + format_hints.Hex(vm_end), vma.get_perms(), path, + file_output, ), ) @@ -72,6 +236,7 @@ class Maps(interfaces.plugins.PluginInterface): ("End", format_hints.Hex), ("Protection", str), ("Map Name", str), + ("File output", str), ], self._generator( list_tasks(self.context, self.config["kernel"], filter_func=filter_func) From 965ddcb674d30528792a404b819635c60c207f8e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 19:30:46 +0100 Subject: [PATCH 112/979] make queue_head_t optional, as it might not exist in all ISF --- volatility3/framework/symbols/mac/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index be4fe8cd1..83aebb13b 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -28,7 +28,7 @@ class MacKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("kauth_scope", extensions.kauth_scope) # https://developer.apple.com/documentation/kernel/queue_head_t self.set_type_class("queue_entry", extensions.queue_entry) - self.set_type_class("queue_head_t", extensions.queue_entry) + self.optional_set_type_class("queue_head_t", extensions.queue_entry) class MacUtilities(interfaces.configuration.VersionableInterface): From 2e835099b58b54af2f988355ae04cbfeff9cd13e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 20:19:31 +0100 Subject: [PATCH 113/979] fix wrong list_head comparison + better naming --- .../symbols/mac/extensions/__init__.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index c89b527e6..0a6bfbb90 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -490,22 +490,24 @@ class queue_entry(objects.StructType): for attr in ["next", "prev"]: with contextlib.suppress(exceptions.InvalidAddressException): - n = getattr(self, attr).dereference().cast(type_name) - - while n is not None and n.vol.offset != list_head: - if n.vol.offset in seen: + queue_element = getattr(self, attr).dereference().cast(type_name) + while ( + queue_element is not None + and queue_element.vol.offset != list_head.vol.offset + ): + if queue_element.vol.offset in seen: break - yield n + yield queue_element - seen.add(n.vol.offset) + seen.add(queue_element.vol.offset) yielded = yielded + 1 if yielded == max_size: - return + return None - n = ( - getattr(n.member(attr=member_name), attr) + queue_element = ( + getattr(queue_element.member(attr=member_name), attr) .dereference() .cast(type_name) ) From caf108b604be6925a2e5e1a4afc627da1fa943a5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 17:17:48 +0100 Subject: [PATCH 114/979] macOS dmesg plugin support --- volatility3/framework/plugins/mac/dmesg.py | 79 ++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 volatility3/framework/plugins/mac/dmesg.py diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py new file mode 100644 index 000000000..a006ff854 --- /dev/null +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -0,0 +1,79 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility + +vollog = logging.getLogger(__name__) + + +class Dmesg(interfaces.plugins.PluginInterface): + """Prints the kernel log buffer.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + ] + + @classmethod + def get_kernel_log_buffer( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str + ): + """ + Online documentation : + - https://github.com/apple-open-source/macos/blob/master/xnu/bsd/sys/msgbuf.h + - https://github.com/apple-open-source/macos/blob/ea4cd5a06831aca49e33df829d2976d6de5316ec/xnu/bsd/kern/subr_log.c#L751 + Volatility 2 plugin : + - https://github.com/volatilityfoundation/volatility/blob/master/volatility/plugins/mac/dmesg.py + """ + + kernel = context.modules[kernel_module_name] + if not kernel.has_symbol("msgbufp"): + vollog.error( + 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' + ) + return [] + + msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") + msgbufp = msgbufp_ptr.dereference() + msg_size = msgbufp.msg_size # max buffer size + msg_bufx = msgbufp.msg_bufx # write pointer + msg_bufc = msgbufp.msg_bufc + # msg_bufc is circular, meaning that if its size exceeds msg_size, + # msg_bufx will point to the beginning of the buffer and start overwriting. + msg_bufc_data: str = utility.pointer_to_string(msg_bufc, msg_size) + # Avoid OOB reads + msg_bufx = msg_bufx if msg_bufx <= msg_size else 0 + # We directly take into account the case where the write buffer did a loop, + # as older messages will start at msg_bufx offset (not overwritten yet). + dmesg = msg_bufc_data[msg_bufx:] + dmesg += msg_bufc_data[:msg_bufx] + + # Yield each line + for dmesg_line in dmesg.splitlines(): + yield (dmesg_line.strip(),) + + def _generator(self): + for value in self.get_kernel_log_buffer( + context=self.context, kernel_module_name=self.config["kernel"] + ): + yield (0, value) + + def run(self): + return renderers.TreeGrid( + [ + ("line", str), + ], + self._generator(), + ) From 8d79e3211ba2e8ea751c657b5da9f99a8b3a90c2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 17:30:51 +0100 Subject: [PATCH 115/979] prefer TypeError to vollog.error --- volatility3/framework/plugins/mac/dmesg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index a006ff854..b837b9db6 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -40,10 +40,9 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): - vollog.error( + raise TypeError( 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) - return [] msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") msgbufp = msgbufp_ptr.dereference() From 2f0bb6b297ffe601c563d646f09c80b8b0a4864f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 18:39:48 +0100 Subject: [PATCH 116/979] do not strip each line --- volatility3/framework/plugins/mac/dmesg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index b837b9db6..d4f2c869d 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -61,7 +61,7 @@ class Dmesg(interfaces.plugins.PluginInterface): # Yield each line for dmesg_line in dmesg.splitlines(): - yield (dmesg_line.strip(),) + yield (dmesg_line,) def _generator(self): for value in self.get_kernel_log_buffer( From cb6f8c507268dd399f24e762766b3ba42498799a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 4 Mar 2024 20:47:49 +1100 Subject: [PATCH 117/979] Rename methods to be private --- .../framework/plugins/linux/library_list.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py index ed5545347..062ed078e 100644 --- a/volatility3/framework/plugins/linux/library_list.py +++ b/volatility3/framework/plugins/linux/library_list.py @@ -43,7 +43,7 @@ class LibraryList(interfaces.plugins.PluginInterface): ), ] - def get_libdl_libraries( + def _get_libdl_libraries( self, proc_layer_name: str, vma_start: int ) -> interfaces.objects.ObjectInterface: """Get the ELF link map objects for the given VMA address @@ -81,7 +81,7 @@ class LibraryList(interfaces.plugins.PluginInterface): # Protection against memory smear in this VMA pass - def get_libdl_maps( + def _get_libdl_maps( self, task: interfaces.objects.ObjectInterface, proc_layer_name: str ) -> interfaces.objects.ObjectInterface: """Get the ELF link maps objects for a task @@ -96,14 +96,14 @@ class LibraryList(interfaces.plugins.PluginInterface): link_map_seen = set() for vma in task.mm.get_vma_iter(): - for link_map in self.get_libdl_libraries(proc_layer_name, vma.vm_start): + for link_map in self._get_libdl_libraries(proc_layer_name, vma.vm_start): if link_map.l_addr in link_map_seen: continue yield link_map link_map_seen.add(link_map.l_addr) - def get_task_libraries( + def _get_task_libraries( self, task: interfaces.objects.ObjectInterface ) -> Tuple[int, str]: """Get the task libraries from the ELF headers found within the memory maps @@ -118,13 +118,13 @@ class LibraryList(interfaces.plugins.PluginInterface): if not proc_layer_name: return - for elf_link_map in self.get_libdl_maps(task, proc_layer_name): + for elf_link_map in self._get_libdl_maps(task, proc_layer_name): name = elf_link_map.get_name() if not name: continue yield elf_link_map.l_addr, name - def get_tasks_libraries( + def _get_tasks_libraries( self, tasks: Iterable[interfaces.objects.ObjectInterface], ) -> Iterable[Tuple[str, int, int, str]]: @@ -139,7 +139,7 @@ class LibraryList(interfaces.plugins.PluginInterface): """ for task in tasks: task_name = utility.array_to_string(task.comm) - for linkmap_addr, linkmap_name in self.get_task_libraries(task): + for linkmap_addr, linkmap_name in self._get_task_libraries(task): yield task_name, task.tgid, linkmap_addr, linkmap_name def _format_fields(self, fields): @@ -149,7 +149,7 @@ class LibraryList(interfaces.plugins.PluginInterface): def _generator( self, tasks: Iterable[interfaces.objects.ObjectInterface] ) -> Iterable[Tuple[int, Tuple]]: - for fields in self.get_tasks_libraries(tasks): + for fields in self._get_tasks_libraries(tasks): yield 0, self._format_fields(fields) def run(self): From 373bf9dfaa466fdaba9635a78a6449679af502bf Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 4 Mar 2024 18:33:03 +0000 Subject: [PATCH 118/979] Windows: Fix driverirp black issue --- volatility3/framework/plugins/windows/driverirp.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index f7eca0359..433c61ca2 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -115,9 +115,17 @@ class DriverIrp(interfaces.plugins.PluginInterface): ) if not module_found: - yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], - format_hints.Hex(address), renderers.NotAvailableValue(), renderers.NotAvailableValue())) - + yield ( + 0, + ( + format_hints.Hex(driver.vol.offset), + driver_name, + MAJOR_FUNCTIONS[i], + format_hints.Hex(address), + renderers.NotAvailableValue(), + renderers.NotAvailableValue(), + ), + ) def run(self): return renderers.TreeGrid( From 403804431ae0ae5118b13cb0f894edfb9cc03d73 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:11:17 +0100 Subject: [PATCH 119/979] prefer SymbolError to TypeError --- volatility3/framework/plugins/mac/dmesg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index d4f2c869d..12c241641 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -3,7 +3,7 @@ # import logging -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility @@ -40,7 +40,7 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): - raise TypeError( + raise exceptions.SymbolError( 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) From 1bf18dbd8a9f68381dfbdb46c4d76ab92cbe0a68 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:11:50 +0100 Subject: [PATCH 120/979] implicit pointer dereference --- volatility3/framework/plugins/mac/dmesg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index 12c241641..daa6617ad 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -44,8 +44,7 @@ class Dmesg(interfaces.plugins.PluginInterface): 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) - msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") - msgbufp = msgbufp_ptr.dereference() + msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") msg_size = msgbufp.msg_size # max buffer size msg_bufx = msgbufp.msg_bufx # write pointer msg_bufc = msgbufp.msg_bufc From 98705e110df4691dc974210f0818621db2f27bbf Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:12:12 +0100 Subject: [PATCH 121/979] more specific msg_bufx comment --- volatility3/framework/plugins/mac/dmesg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index daa6617ad..7f817e605 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -46,7 +46,7 @@ class Dmesg(interfaces.plugins.PluginInterface): msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") msg_size = msgbufp.msg_size # max buffer size - msg_bufx = msgbufp.msg_bufx # write pointer + msg_bufx = msgbufp.msg_bufx # write index of the msg_bufc circular buffer msg_bufc = msgbufp.msg_bufc # msg_bufc is circular, meaning that if its size exceeds msg_size, # msg_bufx will point to the beginning of the buffer and start overwriting. From 69a6c7d5b7cbfb99e8ec70d05dac95a6dea71713 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:16:55 +0100 Subject: [PATCH 122/979] revert int as hex format commit --- volatility3/cli/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index a67762a31..3de7d8f8f 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,11 +827,7 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - # Allow a list of integers, specified with convenient 0x hexadecimal format - if requirement.element_type == int: - additional["type"] = lambda x: int(x, 0) - else: - additional["type"] = requirement.element_type + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 9914f339dfc1cdd511fc3fad676b92b06520243e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:19:15 +0100 Subject: [PATCH 123/979] simplify addrs_in_vma check --- volatility3/framework/plugins/mac/proc_maps.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 204d414e4..fe5179dfa 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -165,10 +165,7 @@ class Maps(interfaces.plugins.PluginInterface): ] # if any of the user supplied addresses would fall within this vma return true - if addrs_in_vma: - return True - else: - return False + return bool(addrs_in_vma) vma_filter_func = vma_filter_function From f4b12c6406e2f1accc147069403e363e7c6a01dc Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:26:18 +0100 Subject: [PATCH 124/979] allow ints in the 0x format in ListRequirement --- volatility3/cli/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 3de7d8f8f..35e977eba 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,7 +827,11 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - additional["type"] = requirement.element_type + # Allow a list of integers, specified with the convenient 0x hexadecimal format + if requirement.element_type == int: + additional["type"] = lambda x: int(x, 0) + else: + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 1f9a983f492ff7983030b8fa27b509106594f735 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:38:01 +0100 Subject: [PATCH 125/979] correct use of SymbolError --- volatility3/framework/plugins/mac/dmesg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index 7f817e605..f9f06a666 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -41,7 +41,9 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): raise exceptions.SymbolError( - 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' + "msgbufp", + kernel.symbol_table_name, + 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.', ) msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") From 9edf33b7212d46682b48dbb40b744f198f8741a8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 11 Mar 2024 21:20:29 +0000 Subject: [PATCH 126/979] Layers: Improve logging on crashdump layer --- volatility3/framework/layers/crash.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 8efd4f7c7..8fd0abbcc 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -261,11 +261,15 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): progress_callback: constants.ProgressCallback = None, ) -> Optional[interfaces.layers.DataLayerInterface]: for layer in [WindowsCrashDump32Layer, WindowsCrashDump64Layer]: - with contextlib.suppress(WindowsCrashDumpFormatException): + try: layer.check_header(context.layers[layer_name]) new_name = context.layers.free_layer_name(layer.__name__) context.config[ interfaces.configuration.path_join(new_name, "base_layer") ] = layer_name return layer(context, new_name, new_name) + except (WindowsCrashDump32Layer, WindowsCrashDump64Layer) as excp: + vollog.log( + constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" + ) return None From 084ea38f84b36e8db2594fc31f1c2d61b3b7c6ce Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 12 Mar 2024 00:13:02 +0000 Subject: [PATCH 127/979] Layers: Fix up typo in recent crashdump patch --- volatility3/framework/layers/crash.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 8fd0abbcc..5598fc2e4 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -268,8 +268,8 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): interfaces.configuration.path_join(new_name, "base_layer") ] = layer_name return layer(context, new_name, new_name) - except (WindowsCrashDump32Layer, WindowsCrashDump64Layer) as excp: + except WindowsCrashDumpFormatException as excp: vollog.log( constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" - ) + )\ return None From 8dbc64f4a8678455adbac80ac716dfa62b3aecb2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 12 Mar 2024 00:14:30 +0000 Subject: [PATCH 128/979] Layers: Fix up typo in recent crashdump patch - take 2 --- volatility3/framework/layers/crash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 5598fc2e4..042b18ddc 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -271,5 +271,5 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): except WindowsCrashDumpFormatException as excp: vollog.log( constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" - )\ + ) return None From f6495d3d986e01fb96789a88aa89cc2c8e40cb71 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 16 Mar 2024 08:42:53 +0000 Subject: [PATCH 129/979] Documentation: Improve logging level docstrings --- volatility3/framework/constants/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 921f602fd..3c5e0eb2f 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -59,14 +59,18 @@ PACKAGE_VERSION = ( AUTOMAGIC_CONFIG_PATH = "automagic" """The root section within the context configuration for automagic values""" +LOGLEVEL_INFO = 20 +"""Logging level for information data, showed when use the requests any logging: -v""" +LOGLEVEL_DEBUG = 10 +"""Logging level for debugging data, showed when the user requests more logging detail: -vv""" LOGLEVEL_V = 9 -"""Logging level for a single -v""" +"""Logging level for the lowest "extra" level of logging: -vvv""" LOGLEVEL_VV = 8 -"""Logging level for -vv""" +"""Logging level for two levels of detail: -vvvv""" LOGLEVEL_VVV = 7 -"""Logging level for -vvv""" +"""Logging level for three levels of detail: -vvvvv""" LOGLEVEL_VVVV = 6 -"""Logging level for -vvvv""" +"""Logging level for four levels of detail: -vvvvvv""" CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" From 436d355ef301d9cd73883b5129f8b9f744c2ecb7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 18:44:09 +0100 Subject: [PATCH 130/979] incrementally order extra log levels --- volatility3/cli/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 35e977eba..ec9918a65 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -264,6 +264,17 @@ class CommandLine: file_logger.setFormatter(file_formatter) rootlog.addHandler(file_logger) vollog.info("Logging started") + + for level, level_value in enumerate( + [ + constants.LOGLEVEL_V, + constants.LOGLEVEL_VV, + constants.LOGLEVEL_VVV, + constants.LOGLEVEL_VVVV, + ] + ): + logging.addLevelName(level_value, f"DETAIL {level+1}") + if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None From 33d653e7a66b8b96350a3568c6d156eadb0cf878 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 18:44:19 +0100 Subject: [PATCH 131/979] use logging explicit constants --- volatility3/cli/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index ec9918a65..116cde114 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -278,9 +278,9 @@ class CommandLine: if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None - console.setLevel(30 - (partial_args.verbosity * 10)) + console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: - console.setLevel(10 - (partial_args.verbosity - 2)) + console.setLevel(logging.DEBUG - (partial_args.verbosity - 2)) for level, msg in delayed_logs: vollog.log(level, msg) From 745491c846314fb37231c892e2cfab9eed4275f2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:02 +0100 Subject: [PATCH 132/979] put extra log level ordering in a function --- volatility3/cli/__init__.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 116cde114..457a49311 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -265,16 +265,7 @@ class CommandLine: rootlog.addHandler(file_logger) vollog.info("Logging started") - for level, level_value in enumerate( - [ - constants.LOGLEVEL_V, - constants.LOGLEVEL_VV, - constants.LOGLEVEL_VVV, - constants.LOGLEVEL_VVVV, - ] - ): - logging.addLevelName(level_value, f"DETAIL {level+1}") - + self.order_extra_verbose_levels() if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None @@ -706,6 +697,17 @@ class CommandLine: ) context.config[extended_path] = value + def order_extra_verbose_levels(self): + for level, level_value in enumerate( + [ + constants.LOGLEVEL_V, + constants.LOGLEVEL_VV, + constants.LOGLEVEL_VVV, + constants.LOGLEVEL_VVVV, + ] + ): + logging.addLevelName(level_value, f"DETAIL {level+1}") + def file_handler_class_factory(self, direct=True): output_dir = self.output_dir From 7ff5f57e5d4485ad2e5f2ad4dc78017880442c05 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:28 +0100 Subject: [PATCH 133/979] use logging explicit constants --- volatility3/cli/volshell/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 9e74acfec..998c8245b 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -198,9 +198,9 @@ class VolShell(cli.CommandLine): vollog.info("Logging started") if partial_args.verbosity < 3: - console.setLevel(30 - (partial_args.verbosity * 10)) + console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: - console.setLevel(10 - (partial_args.verbosity - 2)) + console.setLevel(logging.DEBUG - (partial_args.verbosity - 2)) for level, msg in delayed_logs: vollog.log(level, msg) From de4a3359982b67b74a354001d380502ff41804d1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:45 +0100 Subject: [PATCH 134/979] call extra verbose level ordering --- volatility3/cli/volshell/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 998c8245b..035ed9b2e 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -197,6 +197,7 @@ class VolShell(cli.CommandLine): vollog.addHandler(file_logger) vollog.info("Logging started") + self.order_extra_verbose_levels() if partial_args.verbosity < 3: console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: From 47891c3477fbd7bd008e8b3b8f119936289104de Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Apr 2024 21:17:42 +0100 Subject: [PATCH 135/979] Core: Develop nested requirement files --- requirements-dev.txt | 21 ++++----------------- requirements.txt | 4 ++-- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c9b615cd8..ae3482290 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,22 +1,9 @@ -# The following packages are required for core functionality. -pefile>=2023.2.7 - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 - -# This is required for several plugins that perform malware analysis and disassemble code. -# It can also improve accuracy of Windows 8 and later memory samples. -capstone>=3.0.5 - -# This is required by plugins that decrypt passwords, password hashes, etc. -pycryptodome +-r requirements.txt # This can improve error messages regarding improperly configured ISF files, # but is only recommended for development jsonschema>=2.3.0 -# This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0 +# Used to build executable file +pyinstaller>=6.5.0 +pyinstaller-hooks-contrib>=2024.3 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4d09ff82a..c63dc0b36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -# The following packages are required for core functionality. -pefile>=2023.2.7 +# Include the minimal requirements +-r requirements-minimal.txt # The following packages are optional. # If certain packages are not necessary, place a comment (#) at the start of the line. From c1f239b8d2171e83e331ad51bf7ff6e48ed04e52 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 7 Apr 2024 19:34:17 +0100 Subject: [PATCH 136/979] Linux: Improve debugging of pslist dump feature --- volatility3/framework/plugins/linux/elfs.py | 1 + volatility3/framework/plugins/linux/pslist.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 6820576dc..22e39d127 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -84,6 +84,7 @@ class Elfs(plugins.PluginInterface): ) if not elf_object.is_valid(): + vollog.debug("ELF object to be dumped is not valid") return None sections = {} diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 046ee43d8..1888bd7b8 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -142,6 +142,8 @@ class PsList(interfaces.plugins.PluginInterface): file_output = str(file_handle.preferred_filename) file_handle.close() break + else: + file_output = "VMA start matching task start_code not found" return file_output def _generator( From 974e6a4107cda567ce24a3d6384c28939ae64dd1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 21 Apr 2024 18:28:13 +0100 Subject: [PATCH 137/979] Core: When clearing the cache, actually clear cached files --- volatility3/framework/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 1565b2267..422c083d0 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -225,6 +225,12 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: def clear_cache(complete=False): try: + if complete: + glob_pattern = "*.cache" + for cache_filename in glob.glob( + os.path.join(constants.CACHE_PATH, glob_pattern) + ): + os.unlink(cache_filename) os.unlink(os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)) except FileNotFoundError: vollog.log(constants.LOGLEVEL_VVVV, "Attempting to clear a non-existant cache") From 8f0d1b37cc2f88646b5858509e1bc0d5bf4941e0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Apr 2024 20:51:25 +0100 Subject: [PATCH 138/979] Core: Improve error handling for proxy authentication issue --- volatility3/framework/layers/resources.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index a64fa7d7a..2dba7caa8 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -151,6 +151,12 @@ class ResourceAccessor(object): raise excp else: raise excp + except ValueError as excp: + # Reraise errors such as proxy auth errors as offline exception errors + # Example Proxy auth error - ValueError: AbstractDigestAuthHandler does not support the following scheme: 'Negotiate' + vollog.info(f"Cannot access {url} due to {excp} - Setting OFFLINE") + constants.OFFLINE = True + raise exceptions.OfflineException(url) except exceptions.OfflineException: vollog.info(f"Not accessing {url} in offline mode") raise From 35c583721892b84ae4f9b151ddccbb2e4ef60e9f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 30 Apr 2024 12:34:46 +0100 Subject: [PATCH 139/979] Core: Default to completely clearing the cache when requested --- volatility3/framework/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 422c083d0..feb97810b 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -223,7 +223,7 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: return plugin_list -def clear_cache(complete=False): +def clear_cache(complete=True): try: if complete: glob_pattern = "*.cache" From 314725f0c53811b6652f90c4b8050982b8329ab7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 1 May 2024 04:30:36 +0900 Subject: [PATCH 140/979] Fix: typo for CITATION.cff --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index c36c3b7d5..ac45dfc18 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -14,7 +14,7 @@ authors: identifiers: - type: url value: 'https://github.com/volatilityfoundation/volatility3' - description: Volatility 3 source code respository + description: Volatility 3 source code repository repository-code: 'https://github.com/volatilityfoundation/volatility3' url: 'https://github.com/volatilityfoundation/volatility3' abstract: >- From c7f29936dcfb81bb9ae3fe33e261db2bb7c3be85 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:15:45 +1000 Subject: [PATCH 141/979] CommandLine: Fix issue when --output filename doesn't contain an extension. If the argument is i.e. "--output aaa" ... it returned ".aaa" (hidden filename in linux) then "-1.aaa", "-2.aaa", etc. --- volatility3/cli/__init__.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 457a49311..c0eed20ee 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -716,19 +716,17 @@ class CommandLine: """Gets the final filename""" if output_dir is None: raise TypeError("Output directory is not a string") + os.makedirs(output_dir, exist_ok=True) - pref_name_array = self.preferred_filename.split(".") - filename, extension = ( - os.path.join(output_dir, ".".join(pref_name_array[:-1])), - pref_name_array[-1], - ) - output_filename = f"{filename}.{extension}" + output_filename = os.path.join(output_dir, self.preferred_filename) + filename, extension = os.path.splitext(output_filename) counter = 1 while os.path.exists(output_filename): - output_filename = f"{filename}-{counter}.{extension}" + output_filename = f"{filename}-{counter}{extension}" counter += 1 + return output_filename class CLIMemFileHandler(io.BytesIO, CLIFileHandler): From 32a5f131fd22d5351eee264fd6e58420a1458e9c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:17:51 +1000 Subject: [PATCH 142/979] LayerWriter plugin: Fix log wrong (non-existent) variable --- volatility3/framework/plugins/layerwriter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 1bee5f20d..a8664b3f1 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -115,9 +115,7 @@ class LayerWriter(plugins.PluginInterface): ) file_handle.close() except IOError as excp: - yield 0, ( - f"Layer cannot be written to {self.config['output_name']}: {excp}", - ) + yield 0, (f"Layer cannot be written to {output_name}: {excp}",) yield 0, (f"Layer has been written to {output_name}",) From 1246b20b6521e11110038fe091af912e2b5f4439 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:25:45 +1000 Subject: [PATCH 143/979] LayerWriter plugin: Code improvements --- volatility3/framework/plugins/layerwriter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index a8664b3f1..021c44248 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -95,7 +95,7 @@ class LayerWriter(plugins.PluginInterface): if not self.config["layers"]: self.config["layers"] = [] for name in self.context.layers: - if not self.context.layers[name].metadata.get("mapped", False): + if "mapped" not in self.context.layers[name].metadata: self.config["layers"] = [name] for name in self.config["layers"]: @@ -103,7 +103,8 @@ class LayerWriter(plugins.PluginInterface): if name not in self.context.layers: yield 0, (f"Layer Name {name} does not exist",) else: - output_name = self.config.get("output", ".".join([name, "raw"])) + default_output_name = f"{name}.raw" + output_name = self.config.get("output", default_output_name) try: file_handle = self.write_layer( self.context, From 6d22347ce6dd0152746564a2d77022ca1b4d9045 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 11:27:15 +1000 Subject: [PATCH 144/979] LayerWriter plugin: Fix --output argument. It's referenced in the code but never mentioned as a requirement --- volatility3/framework/plugins/layerwriter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 021c44248..d60ead682 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -38,6 +38,10 @@ class LayerWriter(plugins.PluginInterface): default=False, optional=True, ), + requirements.StringRequirement( + name="output", + description="Output filename", + ), requirements.ListRequirement( name="layers", element_type=str, From e5a5b895771b655d21c36689c33a534034c31e36 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:06:02 +1000 Subject: [PATCH 145/979] Intel layer: Fix. This if statement will never be executed unless "minimum_address > maximum_address" which doesn't make sense to me. --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 75e561b33..8589cdbb6 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -180,7 +180,7 @@ class Intel(linear.LinearlyMappedLayer): position = self._initial_position entry = self._initial_entry - if self.minimum_address > offset > self.maximum_address: + if not (self.minimum_address <= offset <= self.maximum_address): raise exceptions.PagedInvalidAddressException( self.name, offset, From f116c08a7ec60f62e3ef931de7639e402f421d65 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:07:23 +1000 Subject: [PATCH 146/979] NonLinearlySegmentedLayer: Fix maximum addresses in segmented layers --- volatility3/framework/layers/segmented.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/segmented.py b/volatility3/framework/layers/segmented.py index 0d29d8bff..e8c067072 100644 --- a/volatility3/framework/layers/segmented.py +++ b/volatility3/framework/layers/segmented.py @@ -152,7 +152,7 @@ class NonLinearlySegmentedLayer( raise ValueError("SegmentedLayer must contain some segments") if self._maxaddr is None: mapped, _, length, _ = self._segments[-1] - self._maxaddr = mapped + length + self._maxaddr = mapped + length - 1 return self._maxaddr @property From a6c77c488436c7b05040b5bf475be79cb59457f3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:09:45 +1000 Subject: [PATCH 147/979] LayerWriter: Fix - Last chunk size is wrongly calculated --- volatility3/framework/plugins/layerwriter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index d60ead682..178d86ba9 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -18,7 +18,7 @@ class LayerWriter(plugins.PluginInterface): default_block_size = 0x500000 _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -81,7 +81,7 @@ class LayerWriter(plugins.PluginInterface): file_handle = open_method(preferred_name) for i in range(0, layer.maximum_address, chunk_size): - current_chunk_size = min(chunk_size, layer.maximum_address - i) + current_chunk_size = min(chunk_size, layer.maximum_address + 1 - i) data = layer.read(i, current_chunk_size, pad=True) file_handle.write(data) if progress_callback: From d7aae3a9828ba03bd5531aa385724a954e48501a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 6 May 2024 13:11:52 +1000 Subject: [PATCH 148/979] Store the real/final output filename so that we can notify it correctly to the user --- volatility3/cli/__init__.py | 4 ++-- volatility3/framework/plugins/layerwriter.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index c0eed20ee..9873791e3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -790,8 +790,8 @@ class CommandLine: return None self._file.close() - output_filename = self._get_final_filename() - os.rename(self._name, output_filename) + self._output_filename = self._get_final_filename() + os.rename(self._name, self._output_filename) if direct: return CLIDirectFileHandler diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 178d86ba9..20d10d636 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -119,6 +119,7 @@ class LayerWriter(plugins.PluginInterface): progress_callback=self._progress_callback, ) file_handle.close() + output_name = file_handle._output_filename except IOError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) From 85052b6238414617cc87f6812894f14917e03393 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 7 May 2024 08:12:31 +1000 Subject: [PATCH 149/979] LayerWriter: Fix missing optional flag for the --output argument --- volatility3/framework/plugins/layerwriter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 20d10d636..6a06cc607 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -41,6 +41,7 @@ class LayerWriter(plugins.PluginInterface): requirements.StringRequirement( name="output", description="Output filename", + optional=True, ), requirements.ListRequirement( name="layers", From 3f3f1a9c0daae397a84a515e84b29ba9ef0a0300 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 8 May 2024 11:23:01 -0500 Subject: [PATCH 150/979] Prevent duplicate processing of the same file object --- volatility3/framework/plugins/windows/dumpfiles.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 48539c752..33d2d0d41 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -244,6 +244,8 @@ class DumpFiles(interfaces.plugins.PluginInterface): symbol_table=kernel.symbol_table_name, ) + dumped_files = set() + for proc in procs: try: object_table = proc.ObjectTable @@ -267,6 +269,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_re.search(name): continue + if file_obj.vol.offset in dumped_files: + continue + dumped_files.add(file_obj.vol.offset) + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): @@ -303,6 +309,10 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_re.search(name): continue + if file_obj.vol.offset in dumped_files: + continue + dumped_files.add(file_obj.vol.offset) + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): From 3933061551eb0f31b99229ac8346718f773f574c Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 9 May 2024 09:17:32 -0500 Subject: [PATCH 151/979] Add a default 0 value to macb columns to avoid mactime not reporting timeline entries --- volatility3/framework/plugins/timeliner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index d1c1c0f70..26a100f53 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -183,16 +183,16 @@ class Timeliner(interfaces.plugins.PluginInterface): plugin_name, self._sanitize_body_format(item), self._text_format( - times.get(TimeLinerType.ACCESSED, "") + times.get(TimeLinerType.ACCESSED, "0") ), self._text_format( - times.get(TimeLinerType.MODIFIED, "") + times.get(TimeLinerType.MODIFIED, "0") ), self._text_format( - times.get(TimeLinerType.CHANGED, "") + times.get(TimeLinerType.CHANGED, "0") ), self._text_format( - times.get(TimeLinerType.CREATED, "") + times.get(TimeLinerType.CREATED, "0") ), ) ) From 4c937a922d92175135e64be2192b94e8a232c4df Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 9 May 2024 14:47:56 -0500 Subject: [PATCH 152/979] Correctly check for a failed read --- volatility3/framework/plugins/windows/handles.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index d43d26ef1..93ca37c6b 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -161,8 +161,9 @@ class Handles(interfaces.plugins.PluginInterface): except exceptions.SymbolError: return None - data = self.context.layers.read(virtual_layer_name, kvo + func_addr, 0x200) - if data is None: + try: + data = self.context.layers.read(virtual_layer_name, kvo + func_addr, 0x200) + except exceptions.InvalidAddressException: return None md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) From 2b165a56be4d5c804ec6104e165cdf0f862b7204 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 9 May 2024 14:53:24 -0500 Subject: [PATCH 153/979] Fix pre-existing formatting issue from black checks --- volatility3/framework/plugins/windows/handles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 93ca37c6b..15f5f69df 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -162,7 +162,9 @@ class Handles(interfaces.plugins.PluginInterface): return None try: - data = self.context.layers.read(virtual_layer_name, kvo + func_addr, 0x200) + data = self.context.layers.read( + virtual_layer_name, kvo + func_addr, 0x200 + ) except exceptions.InvalidAddressException: return None From 5d5fa96e368f24e11f0860bdd43882da6677de25 Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 16 May 2024 09:19:59 +0100 Subject: [PATCH 154/979] Windows: add extra debugging messages to handles plugin, ref #1146 --- .../framework/plugins/windows/handles.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 15f5f69df..a19e7a397 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -25,7 +25,7 @@ class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -145,6 +145,9 @@ class Handles(interfaces.plugins.PluginInterface): if self._sar_value is None: if not has_capstone: + vollog.debug( + "capstone module is missing, unable to create disassembly of ObpCaptureHandleInformationEx" + ) return None kernel = self.context.modules[self.config["kernel"]] @@ -159,28 +162,46 @@ class Handles(interfaces.plugins.PluginInterface): try: func_addr = ntkrnlmp.get_symbol("ObpCaptureHandleInformationEx").address except exceptions.SymbolError: + vollog.debug("Unable to locate ObpCaptureHandleInformationEx symbol") return None try: + func_addr_to_read = kvo + func_addr + num_bytes_to_read = 0x200 + vollog.debug( + f"ObpCaptureHandleInformationEx symbol located at {hex(func_addr_to_read)}" + ) data = self.context.layers.read( - virtual_layer_name, kvo + func_addr, 0x200 + virtual_layer_name, func_addr_to_read, num_bytes_to_read ) except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}" + ) return None md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) + instruction_count = 0 for address, size, mnemonic, op_str in md.disasm_lite( data, kvo + func_addr ): # print("{} {} {} {}".format(address, size, mnemonic, op_str)) - + instruction_count += 1 if mnemonic.startswith("sar"): # if we don't want to parse op strings, we can disasm the # single sar instruction again, but we use disasm_lite for speed self._sar_value = int(op_str.split(",")[1].strip(), 16) + vollog.debug( + f"SAR located at {hex(address)} with value of {hex(self._sar_value)}" + ) break + if self._sar_value is None: + vollog.debug( + f"Failed to to locate SAR value having parsed {instruction_count} instructions" + ) + return self._sar_value @classmethod From c2b2321622ad1c170e10cf847b566796842a8020 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:09:14 -0500 Subject: [PATCH 155/979] Add a new getcellroutine plugin that reports hooked GetCellRoutine handlers of memory mapped Windows registry hives --- .../windows/registry/getcellroutine.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 volatility3/framework/plugins/windows/registry/getcellroutine.py diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py new file mode 100644 index 000000000..08523d2f1 --- /dev/null +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -0,0 +1,97 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List + +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import ssdt +from volatility3.plugins.windows.registry import hivelist + +vollog = logging.getLogger(__name__) + +class GetCellRoutine(interfaces.plugins.PluginInterface): + """ Reports registry hives with a hooked GetCellRoutine handler """ + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) + ), + ] + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + collection = ssdt.SSDT.build_module_collection( + self.context, kernel.layer_name, kernel.symbol_table_name + ) + + # walk each hive and validate that the GetCellRoutine handler + # is inside of the kernel (ntoskrnl) + for hive_object in hivelist.HiveList.list_hives( + context=self.context, + base_config_path=self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name + ): + hive = hive_object.hive + + try: + cellroutine = hive.GetCellRoutine + except exceptions.InvalidAddressException: + continue + + module_symbols = list( + collection.get_module_symbols_by_absolute_location(cellroutine) + ) + + if module_symbols: + for module_name, _ in module_symbols: + # GetCellRoutine handlers should only be in the kernel + if module_name not in constants.windows.KERNEL_MODULE_NAMES: + yield ( + 0, + ( + format_hints.Hex(hive.vol.offset), + hive_object.get_name() or "", + module_name, + format_hints.Hex(cellroutine) + ) + ) + # Doesn't map to any module... + else: + yield ( + 0, + ( + format_hints.Hex(hive.vol.offset), + hive_object.get_name() or "", + renderers.NotAvailableValue(), + format_hints.Hex(cellroutine) + ) + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Hive Offset", renderers.format_hints.Hex), + ("Hive Name", str), + ("GetCellRoutine Module", str), + ("GetCellRoutine Handler", renderers.format_hints.Hex) + ], + self._generator(), + ) From 47eb42204e19c482ff332ce9bb36fcbe54fa49d3 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:21:03 -0500 Subject: [PATCH 156/979] Fixes for black --- .../plugins/windows/registry/getcellroutine.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index 08523d2f1..2cbb87565 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -14,7 +14,7 @@ from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) class GetCellRoutine(interfaces.plugins.PluginInterface): - """ Reports registry hives with a hooked GetCellRoutine handler """ + """Reports registry hives with a hooked GetCellRoutine handler""" _required_framework_version = (2, 0, 0) @@ -32,7 +32,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) ), - ] + ] def _generator(self): kernel = self.context.modules[self.config["kernel"]] @@ -47,7 +47,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): context=self.context, base_config_path=self.config_path, layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name + symbol_table=kernel.symbol_table_name, ): hive = hive_object.hive @@ -70,7 +70,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): format_hints.Hex(hive.vol.offset), hive_object.get_name() or "", module_name, - format_hints.Hex(cellroutine) + format_hints.Hex(cellroutine), ) ) # Doesn't map to any module... @@ -81,7 +81,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): format_hints.Hex(hive.vol.offset), hive_object.get_name() or "", renderers.NotAvailableValue(), - format_hints.Hex(cellroutine) + format_hints.Hex(cellroutine), ) ) From 4ed7d40ecb61f1d4e671f3323a1b075ad2501bfc Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:23:21 -0500 Subject: [PATCH 157/979] Fixes for black --- .../framework/plugins/windows/registry/getcellroutine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index 2cbb87565..98b3c2517 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -71,7 +71,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): hive_object.get_name() or "", module_name, format_hints.Hex(cellroutine), - ) + ), ) # Doesn't map to any module... else: @@ -82,7 +82,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): hive_object.get_name() or "", renderers.NotAvailableValue(), format_hints.Hex(cellroutine), - ) + ), ) def run(self): @@ -91,7 +91,7 @@ class GetCellRoutine(interfaces.plugins.PluginInterface): ("Hive Offset", renderers.format_hints.Hex), ("Hive Name", str), ("GetCellRoutine Module", str), - ("GetCellRoutine Handler", renderers.format_hints.Hex) + ("GetCellRoutine Handler", renderers.format_hints.Hex), ], self._generator(), ) From bc8666b64a4b722918879eda5efcadeab633bfd2 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 14:24:53 -0500 Subject: [PATCH 158/979] Fixes for black --- volatility3/framework/plugins/windows/registry/getcellroutine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index 98b3c2517..ed54a135d 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -13,6 +13,7 @@ from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) + class GetCellRoutine(interfaces.plugins.PluginInterface): """Reports registry hives with a hooked GetCellRoutine handler""" From 901b0fd6baeaf9779a7111172b579f13688e8ec8 Mon Sep 17 00:00:00 2001 From: atcuno Date: Sun, 19 May 2024 15:11:06 -0500 Subject: [PATCH 159/979] Fix year in header --- .../framework/plugins/windows/registry/getcellroutine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/registry/getcellroutine.py b/volatility3/framework/plugins/windows/registry/getcellroutine.py index ed54a135d..200a45a82 100644 --- a/volatility3/framework/plugins/windows/registry/getcellroutine.py +++ b/volatility3/framework/plugins/windows/registry/getcellroutine.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # From 3eab0671b48818aeb4af1873214bbe506fd7b7de Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 20 May 2024 20:47:12 +0100 Subject: [PATCH 160/979] Generic: Add vmscan plugin --- volatility3/framework/plugins/vmscan.py | 217 ++++++++++++++++++ .../generic/vmcs/haswell-architecture.json | 131 +++++++++++ .../generic/vmcs/skylake-architecture.json | 131 +++++++++++ 3 files changed, 479 insertions(+) create mode 100644 volatility3/framework/plugins/vmscan.py create mode 100644 volatility3/symbols/generic/vmcs/haswell-architecture.json create mode 100644 volatility3/symbols/generic/vmcs/skylake-architecture.json diff --git a/volatility3/framework/plugins/vmscan.py b/volatility3/framework/plugins/vmscan.py new file mode 100644 index 000000000..9d82ba214 --- /dev/null +++ b/volatility3/framework/plugins/vmscan.py @@ -0,0 +1,217 @@ +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import enum +import logging +import os +import struct +from typing import Dict, List + +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import configuration, plugins +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed + +vollog = logging.getLogger(__name__) + + +class VMCSTest(enum.IntFlag): + VMCS_ABORT_INVALID = enum.auto() + VMCS_LINK_PTR_IS_NOT_FS = enum.auto() + VMCS_HOST_CR4_NO_VTX = enum.auto() + VMCS_CR3_IS_ZERO = enum.auto() + VMCS_GUEST_CR4_RESERVED = enum.auto() + + +class PageStartScanner(interfaces.layers.ScannerInterface): + def __init__(self, signatures: List[bytes], page_size: int = 0x1000): + super().__init__() + if not len(signatures): + raise ValueError("No signatures passed to constructor") + self._siglen = len(signatures[0]) + for item in signatures: + if len(item) != self._siglen: + raise ValueError( + "Signatures of different lengths passed to PageStartScanner" + ) + self._signatures = signatures + self._page_size = page_size + + def __call__(self, data: bytes, data_offset: int): + """Scans only the start of every page, to see whether a signature is present or not""" + for page_start in range( + data_offset % self._page_size, len(data), self._page_size + ): + if data[page_start : page_start + self._siglen] in self._signatures: + yield ( + page_start + data_offset, + data[page_start : page_start + self._siglen], + ) + + +class Vmscan(plugins.PluginInterface): + """Scans for Intel VT-d structues and generates VM volatility configs for them""" + + _required_framework_version = (2, 2, 0) + _version = (1, 0, 0) + + STRICTLY_REQUIRED_TESTS = { + VMCSTest.VMCS_ABORT_INVALID, + VMCSTest.VMCS_LINK_PTR_IS_NOT_FS, + VMCSTest.VMCS_HOST_CR4_NO_VTX, + } + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.TranslationLayerRequirement( + name="primary", description="Physical base memory layer" + ), + requirements.IntRequirement( + name="log-threshold", + description="Number of criteria failed to log to debug output", + default=2, + optional=True, + ), + ] + + # Scan for VMCS structures based on the known VMCS structures + # found in symbols/vmcs directory + + def _gather_vmcs_structures( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[bytes, str]: + """Enumerate all JSON files containing VMCS information and return the structures + Signatures can be generated using data extracted using the vmcs_layout tool at + https://github.com/google/rekall/tree/master/tools/linux/vmcs_layout + + Args: + context: The volatility context to work against + config_path: The location to store symbol table configurations under + + Returns: + A dictionary of pattern bytes to the string representation of the architecture + """ + filenames = intermed.IntermediateSymbolTable.file_symbol_url( + os.path.join("generic", "vmcs") + ) + table_names = [] + for filename in filenames: + base_name = os.path.basename(filename).split(".")[0] + table_name = intermed.IntermediateSymbolTable.create( + context, + configuration.path_join(config_path, "vmcs"), + os.path.join("generic", "vmcs"), + filename=base_name, + ) + table_names.append(table_name) + + result = {} + for table_name in table_names: + symbol_table = context.symbol_space[table_name] + revision_id = struct.pack( + " List[str]: + """Runs tests to verify whether a block of data is a VMCS page + Some tests based on the Hypervisor Memory Forensics paper by + Mariano Graziano, Andrea Lanzi and Davide Balzarotti + + Args: + context: The volatility context to be used for this call + vmcs: The instantiated VMCS object to verify + + Returns: + The list of failed criteria that the VMCS did not meet + """ + + # The VMCS should have been constructed on the physical layer (even a nested VMCS) + physical_layer_name = vmcs.vol.layer_name + + failed_tests: VMCSTest = VMCSTest(0) + # The abort field must be valid (generally 0, although other abort codes may exist) + if context.layers[physical_layer_name].read(vmcs.vol.offset + 4, 4) not in [ + b"\x00\x00\x00\x00" + ]: + failed_tests |= VMCSTest.VMCS_ABORT_INVALID + # The vmcs link pointer is supposed to always be set + if vmcs.vmcs_link_ptr != 0xFFFFFFFFFFFFFFFF: + failed_tests |= VMCSTest.VMCS_LINK_PTR_IS_NOT_FS + # To have a VMCS the host needs the VTx bit set in CR4, this can false positive often when all bits are set + if (vmcs.host_cr4 & 1 << 13) == 0: + failed_tests |= VMCSTest.VMCS_HOST_CR4_NO_VTX + # The guest CR3 is *exceptionally* unlikely to be 0 and the guest cr4 is likely to have some bits unset + if (vmcs.guest_cr3 == 0) or (vmcs.host_cr3 == 0): + failed_tests |= VMCSTest.VMCS_CR3_IS_ZERO + # CR4 registers have certain bits reserved that should not be set + if vmcs.guest_cr4 & 0xFFFFFFFFFF889000: + failed_tests |= VMCSTest.VMCS_GUEST_CR4_RESERVED + + if failed_tests and failed_tests.name: + failed_list = failed_tests.name.split("|") + return failed_list + + return [] + + def _generator(self): + # Gather VMCS structures + structures = self._gather_vmcs_structures(self.context, self.config_path) + # Scan memory for them + layer = self.context.layers[self.config["primary"]] + + # Try to move down to the highest physical layer + if layer.config.get("memory_layer"): + layer = self.context.layers[layer.config["memory_layer"]] + + # Run the scan + for offset, match in layer.scan( + self.context, + PageStartScanner(list(structures.keys())), + self._progress_callback, + ): + try: + vmcs = self.context.object( + structures[match] + constants.BANG + "_VMCS", + layer.name, + offset=offset, + ) + failed_list = self._verify_vmcs_page(self.context, vmcs) + if not failed_list: + yield ( + 0, + ( + structures[match], + format_hints.Hex(vmcs.vol.offset), + format_hints.Hex(vmcs.ept), + format_hints.Hex(vmcs.guest_cr3), + ), + ) + if len(failed_list) <= self.config["log-threshold"]: + vollog.debug( + f"Potential {structures[match]} VMCS found at {vmcs.vol.offset:x} with failed criteria: {failed_list}" + ) + except (exceptions.InvalidAddressException, AttributeError): + # Not what we're looking for + continue + + def run(self): + return renderers.TreeGrid( + [ + ("Architecture", str), + ("VMCS Physical offset", format_hints.Hex), + ("EPT", format_hints.Hex), + ("Guest CR3", format_hints.Hex), + ], + self._generator(), + ) diff --git a/volatility3/symbols/generic/vmcs/haswell-architecture.json b/volatility3/symbols/generic/vmcs/haswell-architecture.json new file mode 100644 index 000000000..33b2e55d8 --- /dev/null +++ b/volatility3/symbols/generic/vmcs/haswell-architecture.json @@ -0,0 +1,131 @@ +{ + "base_types": { + "pointer": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned char": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + } + }, + "enums": {}, + "metadata": { + "format": "6.1.0", + "producer": { + "datetime": "2021-07-31T17:37:28.313255", + "name": "vmextract-by-hand", + "version": "0.0.1" + } + }, + "symbols": { + "revision_id": { + "address": 0, + "constant_data": "MTg=" + } + }, + "user_types": { + "_VMCS": { + "fields": { + "ept": { + "offset": 320, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "executive_vmcs_ptr": { + "offset": 208, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr3": { + "offset": 528, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr4": { + "offset": 536, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_pdpte": { + "offset": 544, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "unsigned long long" + } + } + }, + "guest_physical_addr": { + "offset": 328, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr3": { + "offset": 816, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr4": { + "offset": 824, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vmcs_link_ptr": { + "offset": 248, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vpid": { + "offset": 206, + "type": { + "kind": "struct", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 4096 + } + } +} \ No newline at end of file diff --git a/volatility3/symbols/generic/vmcs/skylake-architecture.json b/volatility3/symbols/generic/vmcs/skylake-architecture.json new file mode 100644 index 000000000..78b110d94 --- /dev/null +++ b/volatility3/symbols/generic/vmcs/skylake-architecture.json @@ -0,0 +1,131 @@ +{ + "base_types": { + "pointer": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned char": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + } + }, + "enums": {}, + "metadata": { + "format": "6.1.0", + "producer": { + "datetime": "2021-07-16T16:21:01.062423", + "name": "vmextract-by-hand", + "version": "0.0.1" + } + }, + "symbols": { + "revision_id": { + "address": 0, + "constant_data": "NA==" + } + }, + "user_types": { + "_VMCS": { + "fields": { + "ept": { + "offset": 320, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "executive_vmcs_ptr": { + "offset": 208, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr3": { + "offset": 528, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr4": { + "offset": 536, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_pdpte": { + "offset": 544, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "unsigned long long" + } + } + }, + "guest_physical_addr": { + "offset": 328, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr3": { + "offset": 816, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr4": { + "offset": 824, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vmcs_link_ptr": { + "offset": 248, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vpid": { + "offset": 206, + "type": { + "kind": "struct", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 4096 + } + } +} From c5a45f91fc4968a46389781a4e21c34c771578a2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 20 May 2024 21:09:00 +0100 Subject: [PATCH 161/979] Linux: Replace uses of specific types with the more generic pointer --- volatility3/framework/automagic/linux.py | 2 +- volatility3/framework/plugins/linux/kmsg.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2eebcc2dc..7d7c563b6 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -159,7 +159,7 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): # This we get for free aslr_shift = ( - init_task.files.cast("long unsigned int") + init_task.files.cast("pointer") - module.get_symbol("init_files").address ) kaslr_shift = init_task_address - cls.virtual_to_physical_address( diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index c5e0fc302..e26d69543 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -66,7 +66,7 @@ class ABCKmsg(ABC): self._config = config self.vmlinux = context.modules[self._config["kernel"]] self.layer_name = self.vmlinux.layer_name # type: ignore - self.long_unsigned_int_size = self.vmlinux.get_type("long unsigned int").size + self.long_unsigned_int_size = self.vmlinux.get_type("pointer").size @classmethod def run_all( From c6d207d170c50ad460de4b9970e74405bd0e1d0d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 21 May 2024 09:06:01 +0100 Subject: [PATCH 162/979] Generic: Fix code scanning issue with vmscan --- volatility3/framework/plugins/vmscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/vmscan.py b/volatility3/framework/plugins/vmscan.py index 9d82ba214..64377d7d8 100644 --- a/volatility3/framework/plugins/vmscan.py +++ b/volatility3/framework/plugins/vmscan.py @@ -120,7 +120,7 @@ class Vmscan(plugins.PluginInterface): @classmethod def _verify_vmcs_page( - self, + cls, context: interfaces.context.ContextInterface, vmcs: interfaces.objects.ObjectInterface, ) -> List[str]: From 08493e9c3affda195c7c83a27c9e80bb020fe4c2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 21 May 2024 23:13:31 +0100 Subject: [PATCH 163/979] Automagic: Do some slight gymnastics to get the true value of the pointer --- volatility3/framework/automagic/linux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 7d7c563b6..52a73f45a 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -159,7 +159,10 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): # This we get for free aslr_shift = ( - init_task.files.cast("pointer") + int.from_bytes( + init_task.files.cast("bytes", length=init_task.files.vol.size), + byteorder=init_task.files.vol.data_format.byteorder, + ) - module.get_symbol("init_files").address ) kaslr_shift = init_task_address - cls.virtual_to_physical_address( From 57b3a99a1e42bfdb102f5038469999a815880330 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 May 2024 20:40:36 +0100 Subject: [PATCH 164/979] Infra: Upgrade deprecated githubs actions --- .github/workflows/build-pypi.yml | 6 +++--- .github/workflows/install.yml | 2 +- .github/workflows/test.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index f21898971..5a90364cd 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -20,9 +20,9 @@ jobs: matrix: python-version: ["3.7"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -37,7 +37,7 @@ jobs: python setup.py bdist_wheel - name: Archive dist - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: volatility3-pypi path: | diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index cc2a7fd3e..917685187 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b1a9dd31b..7ce6b13bb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,9 +8,9 @@ jobs: matrix: python-version: ["3.7"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From 35122df27d6d65c661c5fc43508e61609f262714 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 May 2024 20:42:25 +0100 Subject: [PATCH 165/979] Infra: Upgrade deprecated githubs actions 2 --- .github/workflows/black.yml | 2 +- .github/workflows/install.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 5f4523072..5adab3259 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -6,7 +6,7 @@ jobs: lint: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable with: options: "--check --diff --verbose" diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 917685187..e13161085 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -10,10 +10,10 @@ jobs: host: [ ubuntu-latest, windows-latest ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 771ed10b44573a7f8baa32822f3bc524195fe0c9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 29 May 2024 20:44:04 +0100 Subject: [PATCH 166/979] Core: Bump to 2.7.1 --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3c5e0eb2f..6963867f7 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 7 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 76c8067b21ccf4a596d06e6d2d470f01c39e1fa2 Mon Sep 17 00:00:00 2001 From: Branch Vincent Date: Sat, 1 Jun 2024 10:47:20 -0700 Subject: [PATCH 167/979] add full and dev extras --- requirements.txt | 2 +- setup.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index c63dc0b36..bd61edf14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ capstone>=3.0.5 pycryptodome # This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0 +leechcorepyc>=2.4.0; sys_platform != 'darwin' # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage gcsfs>=2023.1.0 diff --git a/setup.py b/setup.py index c2c55067d..6987c4359 100644 --- a/setup.py +++ b/setup.py @@ -10,12 +10,12 @@ with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() -def get_install_requires(): +def get_requires(filename): requirements = [] - with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: + with open(filename, "r", encoding="utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() - if stripped_line == "" or stripped_line.startswith("#"): + if stripped_line == "" or stripped_line.startswith(("#", "-r")): continue requirements.append(stripped_line) return requirements @@ -49,5 +49,9 @@ setuptools.setup( "volshell = volatility3.cli.volshell:main", ], }, - install_requires=get_install_requires(), + install_requires=get_requires("requirements-minimal.txt"), + extras_require={ + "dev": get_requires("requirements-dev.txt"), + "full": get_requires("requirements.txt"), + }, ) From 693eebc34b17fc067f8a91739b8aac884ce1481b Mon Sep 17 00:00:00 2001 From: Branch Vincent Date: Sat, 1 Jun 2024 15:29:01 -0700 Subject: [PATCH 168/979] migrate to pyproject.toml --- .github/workflows/build-pypi.yml | 5 ++- .github/workflows/test.yaml | 5 ++- .gitignore | 1 + README.md | 5 ++- pyproject.toml | 32 ++++++++++++++++++ setup.py | 33 ------------------- volatility3/framework/constants/__init__.py | 21 ++++-------- volatility3/framework/constants/_version.py | 13 ++++++++ volatility3/framework/interfaces/automagic.py | 2 +- 9 files changed, 60 insertions(+), 57 deletions(-) create mode 100644 pyproject.toml create mode 100644 volatility3/framework/constants/_version.py diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 5a90364cd..346d0337e 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -29,12 +29,11 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install build - name: Build PyPi packages run: | - python setup.py sdist --formats=gztar,zip - python setup.py bdist_wheel + python -m build - name: Archive dist uses: actions/upload-artifact@v4 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7ce6b13bb..55b2e4b60 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -18,13 +18,12 @@ jobs: run: | python -m pip install --upgrade pip pip install Cmake - pip install setuptools wheel + pip install build pip install -r ./test/requirements-testing.txt - name: Build PyPi packages run: | - python setup.py sdist --formats=gztar,zip - python setup.py bdist_wheel + python -m build - name: Download images run: | diff --git a/.gitignore b/.gitignore index 328ba5f83..c132736a9 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ config*.json # Pyinstaller files build dist +*.egg-info # Environments .env diff --git a/README.md b/README.md index f69acb3bb..d303d2dff 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,10 @@ Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of pip3 install -r requirements-minimal.txt ``` -Alternately, the minimal packages will be installed automatically when Volatility 3 is installed using setup.py. However, as noted in the Quick Start section below, Volatility 3 does not *need* to be installed via setup.py prior to using it. +Alternately, the minimal packages will be installed automatically when Volatility 3 is installed using pip. However, as noted in the Quick Start section below, Volatility 3 does not *need* to be installed prior to using it. ```shell -python3 setup.py build -python3 setup.py install +pip3 install . ``` To enable the full range of Volatility 3 functionality, use a command like the one below. For partial functionality, comment out any unnecessary packages in [requirements.txt](requirements.txt) prior to running the command. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..98c562a90 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "volatility3" +description = "Memory forensics framework" +keywords = ["volatility", "memory", "forensics", "framework", "windows", "linux", "volshell"] +readme = "README.md" +authors = [ + { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, +] +requires-python = ">=3.7.0" +license = { text = "VSL" } +dynamic = ["dependencies", "optional-dependencies", "version"] + +[project.urls] +Homepage = "https://github.com/volatilityfoundation/volatility3/" +"Bug Tracker" = "https://github.com/volatilityfoundation/volatility3/issues" +Documentation = "https://volatility3.readthedocs.io/" +"Source Code" = "https://github.com/volatilityfoundation/volatility3" + +[project.scripts] +vol = "volatility3.cli:main" +volshell = "volatility3.cli.volshell:main" + +[tool.setuptools.dynamic] +version = { attr = "volatility3.framework.constants._version.PACKAGE_VERSION" } +dependencies = { file = "requirements-minimal.txt" } + +[tool.setuptools.packages.find] +include = ["volatility3*"] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 6987c4359..3af033160 100644 --- a/setup.py +++ b/setup.py @@ -4,11 +4,6 @@ import setuptools -from volatility3.framework import constants - -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - def get_requires(filename): requirements = [] @@ -22,34 +17,6 @@ def get_requires(filename): setuptools.setup( - name="volatility3", - description="Memory forensics framework", - version=constants.PACKAGE_VERSION, - license="VSL", - keywords="volatility memory forensics framework windows linux volshell", - author="Volatility Foundation", - long_description=long_description, - long_description_content_type="text/markdown", - author_email="volatility@volatilityfoundation.org", - url="https://github.com/volatilityfoundation/volatility3/", - project_urls={ - "Bug Tracker": "https://github.com/volatilityfoundation/volatility3/issues", - "Documentation": "https://volatility3.readthedocs.io/", - "Source Code": "https://github.com/volatilityfoundation/volatility3", - }, - packages=setuptools.find_namespace_packages( - include=["volatility3", "volatility3.*"] - ), - package_dir={"volatility3": "volatility3"}, - python_requires=">=3.7.0", - include_package_data=True, - entry_points={ - "console_scripts": [ - "vol = volatility3.cli:main", - "volshell = volatility3.cli.volshell:main", - ], - }, - install_requires=get_requires("requirements-minimal.txt"), extras_require={ "dev": get_requires("requirements-dev.txt"), "full": get_requires("requirements.txt"), diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 6963867f7..8743a64b0 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -14,6 +14,13 @@ from typing import Callable, Optional import volatility3.framework.constants.linux import volatility3.framework.constants.windows +from volatility3.framework.constants._version import ( + PACKAGE_VERSION, + VERSION_MAJOR, + VERSION_MINOR, + VERSION_PATCH, + VERSION_SUFFIX, +) PLUGINS_PATH = [ os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "plugins")), @@ -42,20 +49,6 @@ if hasattr(sys, "frozen") and sys.frozen: BANG = "!" """Constant used to delimit table names from type names when referring to a symbol""" -# We use the SemVer 2.0.0 versioning scheme -VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 7 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface -VERSION_SUFFIX = "" - -# TODO: At version 2.0.0, remove the symbol_shift feature - -PACKAGE_VERSION = ( - ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) - + VERSION_SUFFIX -) -"""The canonical version of the volatility3 package""" - AUTOMAGIC_CONFIG_PATH = "automagic" """The root section within the context configuration for automagic values""" diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py new file mode 100644 index 000000000..37c115431 --- /dev/null +++ b/volatility3/framework/constants/_version.py @@ -0,0 +1,13 @@ +# We use the SemVer 2.0.0 versioning scheme +VERSION_MAJOR = 2 # Number of releases of the library with a breaking change +VERSION_MINOR = 7 # Number of changes that only add to the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_SUFFIX = "" + +# TODO: At version 2.0.0, remove the symbol_shift feature + +PACKAGE_VERSION = ( + ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + + VERSION_SUFFIX +) +"""The canonical version of the volatility3 package""" diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index fe1361b30..0867b1608 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -50,7 +50,7 @@ class AutomagicInterface( context: interfaces.context.ContextInterface, config_path: str, *args, - **kwargs + **kwargs, ) -> None: super().__init__(context, config_path) for requirement in self.get_requirements(): From 0defe0f07e3801e4a833c05e37148aceb93fe305 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 21:08:22 +1000 Subject: [PATCH 169/979] Rename symbol_table to kernel_module_name --- volatility3/framework/plugins/linux/netfilter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 33bb5c456..2b6b67268 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -75,13 +75,13 @@ class AbstractNetfilter(ABC): ): self._context = context self._config = config - symbol_table = self._config["kernel"] - self.vmlinux = context.modules[symbol_table] + kernel_module_name = self._config["kernel"] + self.vmlinux = context.modules[kernel_module_name] self.layer_name = self.vmlinux.layer_name - modules = lsmod.Lsmod.list_modules(context, symbol_table) + modules = lsmod.Lsmod.list_modules(context, kernel_module_name) self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( - context, symbol_table, modules + context, kernel_module_name, modules ) self._set_data_sizes() From fefc6c4a4be2e1f12c81d04103d1e783303473b6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 21:27:27 +1000 Subject: [PATCH 170/979] Pass just the kernel module name instead of the whole config --- volatility3/framework/plugins/linux/netfilter.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 2b6b67268..c2af660b1 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -69,13 +69,9 @@ class AbstractNetfilter(ABC): NF_MAX_HOOKS = LARGEST_HOOK_NUMBER + 1 def __init__( - self, - context: interfaces.context.ContextInterface, - config: interfaces.configuration.HierarchicalDict, + self, context: interfaces.context.ContextInterface, kernel_module_name: str ): self._context = context - self._config = config - kernel_module_name = self._config["kernel"] self.vmlinux = context.modules[kernel_module_name] self.layer_name = self.vmlinux.layer_name @@ -106,7 +102,8 @@ class AbstractNetfilter(ABC): Yields: The kmsg records. Same as _run() """ - vmlinux = context.modules[config["kernel"]] + kernel_module_name = config["kernel"] + vmlinux = context.modules[kernel_module_name] implementation_inst = None # type: ignore for subclass in class_subclasses(cls): @@ -123,7 +120,9 @@ class AbstractNetfilter(ABC): "Netfilter implementation '%s' matches!", subclass.__name__, ) - implementation_inst = subclass(context=context, config=config) + implementation_inst = subclass( + context=context, kernel_module_name=kernel_module_name + ) # More than one class could be executed for an specific kernel version # For instance: Netfilter Ingress hooks yield from implementation_inst._run() From 0962f2638243517b2f5ff0a76423dfaeb7796071 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 21:33:59 +1000 Subject: [PATCH 171/979] class_subclasses is a function, soit's better import the volatility.framework to avoid people accidentally thinking it's defined here and then importing it from here. --- volatility3/framework/plugins/linux/netfilter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index c2af660b1..ad446123f 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -6,8 +6,8 @@ from abc import ABC, abstractmethod import logging from typing import Iterator, List, Tuple +from volatility3 import framework from volatility3.framework import ( - class_subclasses, constants, interfaces, renderers, @@ -106,7 +106,7 @@ class AbstractNetfilter(ABC): vmlinux = context.modules[kernel_module_name] implementation_inst = None # type: ignore - for subclass in class_subclasses(cls): + for subclass in framework.class_subclasses(cls): if not subclass.symtab_checks(vmlinux=vmlinux): vollog.log( constants.LOGLEVEL_VVVV, From 3793510f59adda53f25395bddd358a97e8397054 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 21:36:20 +1000 Subject: [PATCH 172/979] Add VersionRequirement for LinuxUtilities --- volatility3/framework/plugins/linux/netfilter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index ad446123f..b01ec25a7 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -674,6 +674,9 @@ class Netfilter(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) + ), ] def _format_fields(self, fields): From 9d5636b03a7cbee926f88954e58284de30d78f1e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 21:43:11 +1000 Subject: [PATCH 173/979] Move the data size setter to the __init__() --- volatility3/framework/plugins/linux/netfilter.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index b01ec25a7..d27106118 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -75,17 +75,15 @@ class AbstractNetfilter(ABC): self.vmlinux = context.modules[kernel_module_name] self.layer_name = self.vmlinux.layer_name + # Set data sizes + self.ptr_size = self.vmlinux.get_type("pointer").size + self.list_head_size = self.vmlinux.get_type("list_head").size + modules = lsmod.Lsmod.list_modules(context, kernel_module_name) self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( context, kernel_module_name, modules ) - self._set_data_sizes() - - def _set_data_sizes(self): - self.ptr_size = self.vmlinux.get_type("pointer").size - self.list_head_size = self.vmlinux.get_type("list_head").size - @classmethod def run_all( cls, From 23b9ed8c0ca6241db3b5cad6213337ee620c4b7a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 9 Jun 2024 22:02:41 +1000 Subject: [PATCH 174/979] Extend the changes in fefc6c4a to the run_all() class method --- volatility3/framework/plugins/linux/netfilter.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index d27106118..60a71f798 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -86,21 +86,18 @@ class AbstractNetfilter(ABC): @classmethod def run_all( - cls, - context: interfaces.context.ContextInterface, - config: interfaces.configuration.HierarchicalDict, + cls, context: interfaces.context.ContextInterface, kernel_module_name: str ) -> Iterator[Tuple[int, str, str, int, int, str, bool]]: """It calls each subclass symtab_checks() to test the required conditions to that specific kernel implementation. Args: context: The volatility3 context on which to operate - config: Core configuration + kernel_module_name: The name of the table containing the kernel symbols Yields: The kmsg records. Same as _run() """ - kernel_module_name = config["kernel"] vmlinux = context.modules[kernel_module_name] implementation_inst = None # type: ignore @@ -698,8 +695,9 @@ class Netfilter(interfaces.plugins.PluginInterface): ) def _generator(self): + kernel_module_name = self.config["kernel"] for fields in AbstractNetfilter.run_all( - context=self.context, config=self.config + context=self.context, kernel_module_name=kernel_module_name ): yield (0, self._format_fields(fields)) From a72062f2889fdfe5d03e3974e4bc314a35e85700 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jun 2024 17:20:47 +1000 Subject: [PATCH 175/979] Revert "LayerWriter plugin: Fix --output argument. It's referenced in the code but never mentioned as a requirement" This reverts commit 6d22347ce6dd0152746564a2d77022ca1b4d9045. --- volatility3/framework/plugins/layerwriter.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 6a06cc607..319e4fd7a 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -38,11 +38,6 @@ class LayerWriter(plugins.PluginInterface): default=False, optional=True, ), - requirements.StringRequirement( - name="output", - description="Output filename", - optional=True, - ), requirements.ListRequirement( name="layers", element_type=str, From ab84070df30ddfaccc2e304e26fb7815a5555444 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jun 2024 17:25:05 +1000 Subject: [PATCH 176/979] Revert "Store the real/final output filename so that we can notify it correctly to the user" This reverts commit d7aae3a9828ba03bd5531aa385724a954e48501a. --- volatility3/cli/__init__.py | 4 ++-- volatility3/framework/plugins/layerwriter.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 9873791e3..c0eed20ee 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -790,8 +790,8 @@ class CommandLine: return None self._file.close() - self._output_filename = self._get_final_filename() - os.rename(self._name, self._output_filename) + output_filename = self._get_final_filename() + os.rename(self._name, output_filename) if direct: return CLIDirectFileHandler diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 319e4fd7a..61c3118b5 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -115,7 +115,6 @@ class LayerWriter(plugins.PluginInterface): progress_callback=self._progress_callback, ) file_handle.close() - output_name = file_handle._output_filename except IOError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) From 77fb0b7b26c2c52140380a2ce9f1a896a84c258a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 10 Jun 2024 18:37:38 +1000 Subject: [PATCH 177/979] Update the output filename (preferred_filename) so that we can notify it correctly to the user --- volatility3/cli/__init__.py | 10 +++++++++- volatility3/framework/plugins/layerwriter.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index c0eed20ee..6b17edac0 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -789,8 +789,16 @@ class CommandLine: if self._file.closed: return None - self._file.close() output_filename = self._get_final_filename() + + # Update the filename, which may have changed if a file with + # the same name already existed. This needs to be done before + # closing the file, otherwise FileHandlerInterface will raise + # an exception. Also, the preferred_filename setter only allows + # a specific set of characters, where '/' is not in that list + self.preferred_filename = os.path.basename(output_filename) + + self._file.close() os.rename(self._name, output_filename) if direct: diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 61c3118b5..24149a390 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -115,6 +115,10 @@ class LayerWriter(plugins.PluginInterface): progress_callback=self._progress_callback, ) file_handle.close() + + # Update the filename, which may have changed if a file + # with the same name already existed. + output_name = file_handle.preferred_filename except IOError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) From e4aac0c97634487db735f762bdc2f449fa8cd5ae Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 10 Jun 2024 11:30:15 +0100 Subject: [PATCH 178/979] Windows: ldrmodules update exception handling to InvalidAddressException --- .../framework/plugins/windows/ldrmodules.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index 4c8456fa9..ffd84ea4a 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -1,3 +1,9 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -5,12 +11,14 @@ from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import pslist, vadinfo +vollog = logging.getLogger(__name__) + class LdrModules(interfaces.plugins.PluginInterface): """Lists the loaded modules in a particular windows memory image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -71,7 +79,11 @@ class LdrModules(interfaces.plugins.PluginInterface): # Filter out VADs that do not start with a MZ header if dos_header.e_magic != 0x5A4D: continue - except exceptions.PagedInvalidAddressException: + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping vad at {hex(dos_header.vol.offset)} due to InvalidAddressException", + ) continue mapped_files[vad.get_start()] = vad.get_file_name() From 0ee3573be239966a888df644d2aca3a5dd18b02d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 12 Jun 2024 23:14:45 +0100 Subject: [PATCH 179/979] Revert "Intel layer: Fix. This if statement will never be executed unless "minimum_address > maximum_address" which doesn't make sense to me." This reverts commit e5a5b895771b655d21c36689c33a534034c31e36. --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 8589cdbb6..75e561b33 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -180,7 +180,7 @@ class Intel(linear.LinearlyMappedLayer): position = self._initial_position entry = self._initial_entry - if not (self.minimum_address <= offset <= self.maximum_address): + if self.minimum_address > offset > self.maximum_address: raise exceptions.PagedInvalidAddressException( self.name, offset, From 04ca0214eb264268bd368401f8d5eaa940f2524e Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 13 Jun 2024 15:24:32 -0500 Subject: [PATCH 180/979] Prevent backtrace in ADS scanning when contents cannot be recovered. Fix missing parantheses as well as format_hints call on bad value --- volatility3/framework/plugins/windows/mftscan.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 85c072037..1b1a1b45e 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -264,9 +264,10 @@ class ADS(interfaces.plugins.PluginInterface): disasm = interfaces.renderers.Disassembly( content, 0, architecture.lower() ) + content = format_hints.HexBytes(content) else: - content = renderers.NotAvailableValue - disasm = interfaces.renderers.BaseAbsentValue + content = renderers.NotAvailableValue() + disasm = interfaces.renderers.BaseAbsentValue() yield 0, ( format_hints.Hex(attr_data.vol.offset), @@ -275,7 +276,7 @@ class ADS(interfaces.plugins.PluginInterface): attr.Attr_Header.AttrType.lookup(), file_name, ads_name, - format_hints.HexBytes(content), + content, disasm, ) else: From 59f6a050688c30e2b05fec2b0efe54f57e2f46f3 Mon Sep 17 00:00:00 2001 From: atcuno Date: Fri, 14 Jun 2024 16:45:54 -0500 Subject: [PATCH 181/979] The notes variable is not reset for each VAD, allowing bleed through of a previously set note value to VADs enumerated afterwards --- volatility3/framework/plugins/windows/malfind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 8c0cb68ac..079274d69 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -155,12 +155,12 @@ class Malfind(interfaces.plugins.PluginInterface): for proc in procs: # by default, "Notes" column will be set to N/A - notes = renderers.NotApplicableValue() process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( self.context, kernel.layer_name, kernel.symbol_table_name, proc ): + notes = renderers.NotApplicableValue() # Check for unique headers and update "Notes" column if criteria is met if data[0:2] in refined_criteria: notes = refined_criteria[data[0:2]] From aee81276a2d921e44e58db28174f7e4782fe7423 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 11:27:32 -0500 Subject: [PATCH 182/979] Add processghosting plugin to detect process ghosting and related techniques. Add process filter that gathers only active, non-smeared userland processes --- .../plugins/windows/processghosting.py | 99 +++++++++++++++++++ .../framework/plugins/windows/pslist.py | 21 ++++ 2 files changed, 120 insertions(+) create mode 100644 volatility3/framework/plugins/windows/processghosting.py diff --git a/volatility3/framework/plugins/windows/processghosting.py b/volatility3/framework/plugins/windows/processghosting.py new file mode 100644 index 000000000..6c311f555 --- /dev/null +++ b/volatility3/framework/plugins/windows/processghosting.py @@ -0,0 +1,99 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging + +from volatility3.framework import interfaces, exceptions +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + + +class ProcessGhosting(interfaces.plugins.PluginInterface): + """Lists processes whose DeletePending bit is set or whose FILE_OBJECT is set to 0""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + ] + + def _generator(self, procs): + # determine if we're on a 32 or 64 bit kernel + kernel = self.context.modules[self.config["kernel"]] + + if not kernel.get_type("_EPROCESS").has_member("ImageFilePointer"): + vollog.warning("This plugin only supports Windows 10 builds when the ImageFilePointer member of _EPROCESS is present") + return + + for proc in procs: + delete_pending = renderers.UnreadableValue() + process_name = utility.array_to_string(proc.ImageFileName) + + # if it is 0 then its a side effect of process ghosting + if proc.ImageFilePointer.vol.offset != 0: + try: + file_object = proc.ImageFilePointer + delete_pending = file_object.DeletePending + except exceptions.InvalidAddressException: + file_object = 0 + + # ImageFilePointer equal to 0 means process ghosting or similar techniques were used + else: + file_object = 0 + + # delete_pending besides 0 or 1 = smear + if file_object == 0 or delete_pending == 1: + path = renderers.UnreadableValue() + if file_object: + try: + path = file_object.FileName.String + except exceptions.InvalidAddressException: + path = renderers.UnreadableValue() + + yield ( + 0, + ( + proc.UniqueProcessId, + process_name, + format_hints.Hex(file_object), + delete_pending, + path + ), + ) + + def run(self): + filter_func = pslist.PsList.create_active_process_filter() + kernel = self.context.modules[self.config["kernel"]] + + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("FILE_OBJECT", format_hints.Hex), + ("DeletePending", str), + ("Path", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index e7a0d5dd4..8ad0f9a05 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -136,6 +136,27 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): filter_func = lambda x: x.UniqueProcessId not in filter_list return filter_func + @classmethod + def create_active_process_filter( + cls + ) -> Callable[[interfaces.objects.ObjectInterface], bool]: + """A factory for producing a filter function that only returns + active, userland processes. This prevents plugins from operating on terminated + processes that are still in the process list due to smear or handle leaks as well + as kernel processes (System, Registry, etc.). Use of this filter for plugins searching + for system state anomalies significantly reduces false positive in smeared and terminated + processes. + Returns: + Filter function for passing to the `list_processes` method + """ + + return lambda x: not (x.is_valid() and \ + x.ActiveThreads > 0 and \ + x.UniqueProcessId != 4 and \ + x.InheritedFromUniqueProcessId != 4 and \ + x.ExitTime.QuadPart == 0 and \ + x.get_handle_count() != renderers.UnreadableValue()) + @classmethod def create_name_filter( cls, name_list: List[str] = None, exclude: bool = False From 6c42c51a8a6201255f120321885309cd92611c77 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 11:31:43 -0500 Subject: [PATCH 183/979] Fixes for black --- .../framework/plugins/windows/processghosting.py | 7 ++++--- volatility3/framework/plugins/windows/pslist.py | 16 +++++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/processghosting.py b/volatility3/framework/plugins/windows/processghosting.py index 6c311f555..b29ff04f0 100644 --- a/volatility3/framework/plugins/windows/processghosting.py +++ b/volatility3/framework/plugins/windows/processghosting.py @@ -33,11 +33,12 @@ class ProcessGhosting(interfaces.plugins.PluginInterface): ] def _generator(self, procs): - # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] if not kernel.get_type("_EPROCESS").has_member("ImageFilePointer"): - vollog.warning("This plugin only supports Windows 10 builds when the ImageFilePointer member of _EPROCESS is present") + vollog.warning( + "This plugin only supports Windows 10 builds when the ImageFilePointer member of _EPROCESS is present" + ) return for proc in procs: @@ -72,7 +73,7 @@ class ProcessGhosting(interfaces.plugins.PluginInterface): process_name, format_hints.Hex(file_object), delete_pending, - path + path, ), ) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 8ad0f9a05..55a3fc280 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -138,7 +138,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def create_active_process_filter( - cls + cls, ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing a filter function that only returns active, userland processes. This prevents plugins from operating on terminated @@ -150,12 +150,14 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Filter function for passing to the `list_processes` method """ - return lambda x: not (x.is_valid() and \ - x.ActiveThreads > 0 and \ - x.UniqueProcessId != 4 and \ - x.InheritedFromUniqueProcessId != 4 and \ - x.ExitTime.QuadPart == 0 and \ - x.get_handle_count() != renderers.UnreadableValue()) + return lambda x: not ( + x.is_valid() and + x.ActiveThreads > 0 and + x.UniqueProcessId != 4 and + x.InheritedFromUniqueProcessId != 4 and + x.ExitTime.QuadPart == 0 and + x.get_handle_count() != renderers.UnreadableValue() + ) @classmethod def create_name_filter( From 4e15019c46ae24899ea0527dd4855ade5fd7af00 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 11:34:06 -0500 Subject: [PATCH 184/979] Fixes for black --- volatility3/framework/plugins/windows/pslist.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 55a3fc280..8234b210f 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -151,12 +151,12 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ return lambda x: not ( - x.is_valid() and - x.ActiveThreads > 0 and - x.UniqueProcessId != 4 and - x.InheritedFromUniqueProcessId != 4 and - x.ExitTime.QuadPart == 0 and - x.get_handle_count() != renderers.UnreadableValue() + x.is_valid() + and x.ActiveThreads > 0 + and x.UniqueProcessId != 4 + and x.InheritedFromUniqueProcessId != 4 + and x.ExitTime.QuadPart == 0 + and x.get_handle_count() != renderers.UnreadableValue() ) @classmethod From c4e7e50180a45dfb80b453b61e9b94860d5a01d3 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 14:42:23 -0500 Subject: [PATCH 185/979] Add svclist and svcdiff plugins. Make svcscan more modular to support inheritance and cleaner code --- .../framework/plugins/windows/svcdiff.py | 74 ++++++++++++ .../framework/plugins/windows/svclist.py | 86 ++++++++++++++ .../framework/plugins/windows/svcscan.py | 111 +++++++++++------- .../framework/symbols/windows/versions.py | 9 ++ 4 files changed, 235 insertions(+), 45 deletions(-) create mode 100644 volatility3/framework/plugins/windows/svcdiff.py create mode 100644 volatility3/framework/plugins/windows/svclist.py diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py new file mode 100644 index 000000000..03f0afcd3 --- /dev/null +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -0,0 +1,74 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +# This module attempts to locate skeleton-key like function hooks. +# It does this by locating the CSystems array through a variety of methods, +# and then validating the entry for RC4 HMAC (0x17 / 23) +# +# For a thorough walkthrough on how the R&D was performed to develop this plugin, +# please see our blogpost here: +# +# https://volatility-labs.blogspot.com/2021/10/memory-forensics-r-illustrated.html + +import logging + +from volatility3.framework import symbols +from volatility3.framework.configuration import requirements +from volatility3.plugins.windows import svclist, svcscan +from volatility3.framework.symbols.windows import versions + +vollog = logging.getLogger(__name__) + +class SvcDiff(svclist.SvcList, svcscan.SvcScan): + """Compares services found through list walking versus scanning to find rootkits""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="svclist", component=svclist.SvcList, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="svcscan", component=svcscan.SvcScan, version=(2, 0, 0) + ), + ] + + def _generator(self): + """ + Finds services by walking the services.exe list on supported Windows 10 versions + """ + kernel = self.context.modules[self.config["kernel"]] + + if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) or \ + not versions.is_win10_15063_or_later(context=self.context, symbol_table=kernel.symbol_table_name): + vollog.info("This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples") + return + + from_scan = set() + from_list = set() + records = {} + + service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + + # collect unique service names from scanning + for service in self.service_scan(service_table_name, service_binary_dll_map, filter_func): + from_scan.add(service[6]) + records[service[6]] = service + + # collect services from listing walking + for service in self.service_list(service_table_name, service_binary_dll_map, filter_func): + from_list.add(service[6]) + + # report services found from scanning but not list walking + for hidden_service in from_scan-from_list: + yield (0, records[hidden_service]) + diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py new file mode 100644 index 000000000..b4541981d --- /dev/null +++ b/volatility3/framework/plugins/windows/svclist.py @@ -0,0 +1,86 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import List + +from volatility3.framework import interfaces, exceptions, symbols +from volatility3.framework.configuration import requirements +from volatility3.framework.symbols.windows import versions +from volatility3.plugins.windows import svcscan, pslist +from volatility3.framework.layers import scanners + +vollog = logging.getLogger(__name__) + + +class SvcList(svcscan.SvcScan): + """Lists services contained with the services.exe doubly linked list of services""" + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.PluginRequirement( + name="svcscan", plugin=svcscan.SvcScan, version=(2, 0, 0) + ), + ] + + def _get_exe_range(self, proc): + """ + Returns a tuple of starting,ending address for + the VAD containing services.exe + """ + + vad_root = proc.get_vad_root() + for vad in vad_root.traverse(): + filename = vad.get_file_name() + if isinstance(filename, str) and filename.lower().endswith("\\services.exe"): + return [(vad.get_start(), vad.get_size())] + + return None + + def service_list(self, service_table_name, service_binary_dll_map, filter_func): + kernel = self.context.modules[self.config["kernel"]] + + if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) or \ + not versions.is_win10_15063_or_later(context=self.context, symbol_table=kernel.symbol_table_name): + vollog.info("This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples") + return + + for proc in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ): + try: + layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + vollog.warning("Unable to access memory of services.exe running with PID: {}".format(proc.UniqueProcessId)) + continue + + layer = self.context.layers[layer_name] + + exe_range = self._get_exe_range(proc) + if not exe_range: + vollog.warning("Could not find the application executable VAD for services.exe. Unable to proceed.") + continue + + for offset in layer.scan( + context=self.context, + scanner=scanners.BytesScanner(needle = b"Sc27"), + sections=exe_range, + ): + for record in self.enumerate_vista_or_later_header(service_table_name, service_binary_dll_map, layer_name, offset): + yield record + + def _generator(self): + service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + + for record in self.service_list(service_table_name, service_binary_dll_map, filter_func): + yield (0, record) + diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 10de46e2a..08d11a946 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -19,7 +19,7 @@ from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions -from volatility3.framework.symbols.windows.extensions import services +from volatility3.framework.symbols.windows.extensions import services as services_types from volatility3.plugins.windows import poolscanner, pslist, vadyarascan from volatility3.plugins.windows.registry import hivelist @@ -140,7 +140,7 @@ class SvcScan(interfaces.plugins.PluginInterface): config_path, os.path.join("windows", "services"), symbol_filename, - class_types=services.class_types, + class_types=services_types.class_types, native_types=native_types, ) @@ -232,28 +232,44 @@ class SvcScan(interfaces.plugins.PluginInterface): for service_key in services } - def _generator(self): + def enumerate_vista_or_later_header( + self, + service_table_name, + service_binary_dll_map, + proc_layer_name, + offset + ): + if offset % 8: + return + + service_header = self.context.object( + service_table_name + constants.BANG + "_SERVICE_HEADER", + offset=offset, + layer_name=proc_layer_name, + ) + + if not service_header.is_valid(): + return + + # since we walk the s-list backwards, if we've seen + # an object, then we've also seen all objects that + # exist before it, thus we can break at that time. + for service_record in service_header.ServiceRecord.traverse(): + service_info = service_binary_dll_map.get( + service_record.get_name(), + ServiceBinaryInfo( + renderers.UnreadableValue(), renderers.UnreadableValue() + ), + ) + yield self.get_record_tuple(service_record, service_info) + + def service_scan(self, service_table_name, service_binary_dll_map, filter_func): kernel = self.context.modules[self.config["kernel"]] - service_table_name = self.create_service_table( - self.context, kernel.symbol_table_name, self.config_path - ) - - # Building the dictionary ahead of time is much better for performance - # vs looking up each service's DLL individually. - services_key = self._get_service_key(kernel) - service_binary_dll_map = ( - self._get_service_binary_map(services_key) - if services_key is not None - else {} - ) - relative_tag_offset = self.context.symbol_space.get_type( service_table_name + constants.BANG + "_SERVICE_RECORD" ).relative_child_offset("Tag") - filter_func = pslist.PsList.create_name_filter(["services.exe"]) - is_vista_or_later = versions.is_vista_or_later( context=self.context, symbol_table=kernel.symbol_table_name ) @@ -306,37 +322,42 @@ class SvcScan(interfaces.plugins.PluginInterface): renderers.UnreadableValue(), renderers.UnreadableValue() ), ) - yield ( - 0, - self.get_record_tuple(service_record, service_info), - ) + yield self.get_record_tuple(service_record, service_info) else: - service_header = self.context.object( - service_table_name + constants.BANG + "_SERVICE_HEADER", - offset=offset, - layer_name=proc_layer_name, - ) - - if not service_header.is_valid(): - continue - - # since we walk the s-list backwards, if we've seen - # an object, then we've also seen all objects that - # exist before it, thus we can break at that time. - for service_record in service_header.ServiceRecord.traverse(): + for service_record in self.enumerate_vista_or_later_header(service_table_name, service_binary_dll_map, proc_layer_name, offset): if service_record in seen: break seen.append(service_record) - service_info = service_binary_dll_map.get( - service_record.get_name(), - ServiceBinaryInfo( - renderers.UnreadableValue(), renderers.UnreadableValue() - ), - ) - yield ( - 0, - self.get_record_tuple(service_record, service_info), - ) + yield service_record + + + def get_prereq_info(self): + """ + Data structures and information needed to analyze service information + """ + kernel = self.context.modules[self.config["kernel"]] + + service_table_name = self.create_service_table( + self.context, kernel.symbol_table_name, self.config_path + ) + + services_key = self._get_service_key(kernel) + + service_binary_dll_map = ( + self._get_service_binary_map(services_key) + if services_key is not None + else {} + ) + + filter_func = pslist.PsList.create_name_filter(["services.exe"]) + + return service_table_name, service_binary_dll_map, filter_func + + def _generator(self): + service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + + for record in self.service_scan(service_table_name, service_binary_dll_map, filter_func): + yield (0, record) def run(self): return renderers.TreeGrid( diff --git a/volatility3/framework/symbols/windows/versions.py b/volatility3/framework/symbols/windows/versions.py index e1e74afc0..d8964f575 100644 --- a/volatility3/framework/symbols/windows/versions.py +++ b/volatility3/framework/symbols/windows/versions.py @@ -141,6 +141,15 @@ is_win10_15063 = OsDistinguisher( ], ) +is_win10_15063_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 15063), + fallback_checks=[ + ("ObHeaderCookie", None, True), + ("_HANDLE_TABLE", "HandleCount", False), + ("_EPROCESS", "KeepAliveCounter", False), + ], +) + is_win10_16299_or_later = OsDistinguisher( version_check=lambda x: x >= (10, 0, 16299), fallback_checks=[ From 91184c7d92f84cdd3371d3a2371cf1784a2eb1a2 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 14:54:20 -0500 Subject: [PATCH 186/979] Format fixes --- .../framework/plugins/windows/svcdiff.py | 23 ++++++++---- .../framework/plugins/windows/svclist.py | 36 +++++++++++++------ .../framework/plugins/windows/svcscan.py | 18 +++++----- 3 files changed, 51 insertions(+), 26 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py index 03f0afcd3..71aa03636 100644 --- a/volatility3/framework/plugins/windows/svcdiff.py +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -48,27 +48,36 @@ class SvcDiff(svclist.SvcList, svcscan.SvcScan): """ kernel = self.context.modules[self.config["kernel"]] - if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) or \ - not versions.is_win10_15063_or_later(context=self.context, symbol_table=kernel.symbol_table_name): - vollog.info("This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples") + if not symbols.symbol_table_is_64bit( + self.context, kernel.symbol_table_name + ) or not versions.is_win10_15063_or_later( + context=self.context, symbol_table=kernel.symbol_table_name + ): + vollog.info( + "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" + ) return from_scan = set() from_list = set() records = {} - + service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() # collect unique service names from scanning - for service in self.service_scan(service_table_name, service_binary_dll_map, filter_func): + for service in self.service_scan( + service_table_name, service_binary_dll_map, filter_func + ): from_scan.add(service[6]) records[service[6]] = service # collect services from listing walking - for service in self.service_list(service_table_name, service_binary_dll_map, filter_func): + for service in self.service_list( + service_table_name, service_binary_dll_map, filter_func + ): from_list.add(service[6]) # report services found from scanning but not list walking - for hidden_service in from_scan-from_list: + for hidden_service in from_scan - from_list: yield (0, records[hidden_service]) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index b4541981d..53ca68da7 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -6,7 +6,7 @@ import logging from typing import List -from volatility3.framework import interfaces, exceptions, symbols +from volatility3.framework import interfaces, exceptions, symbols from volatility3.framework.configuration import requirements from volatility3.framework.symbols.windows import versions from volatility3.plugins.windows import svcscan, pslist @@ -38,7 +38,9 @@ class SvcList(svcscan.SvcScan): vad_root = proc.get_vad_root() for vad in vad_root.traverse(): filename = vad.get_file_name() - if isinstance(filename, str) and filename.lower().endswith("\\services.exe"): + if isinstance(filename, str) and filename.lower().endswith( + "\\services.exe" + ): return [(vad.get_start(), vad.get_size())] return None @@ -46,9 +48,14 @@ class SvcList(svcscan.SvcScan): def service_list(self, service_table_name, service_binary_dll_map, filter_func): kernel = self.context.modules[self.config["kernel"]] - if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) or \ - not versions.is_win10_15063_or_later(context=self.context, symbol_table=kernel.symbol_table_name): - vollog.info("This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples") + if not symbols.symbol_table_is_64bit( + self.context, kernel.symbol_table_name + ) or not versions.is_win10_15063_or_later( + context=self.context, symbol_table=kernel.symbol_table_name + ): + vollog.info( + "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" + ) return for proc in pslist.PsList.list_processes( @@ -60,27 +67,34 @@ class SvcList(svcscan.SvcScan): try: layer_name = proc.add_process_layer() except exceptions.InvalidAddressException: - vollog.warning("Unable to access memory of services.exe running with PID: {}".format(proc.UniqueProcessId)) + vollog.warning( + "Unable to access memory of services.exe running with PID: {}".format( + proc.UniqueProcessId + ) + ) continue layer = self.context.layers[layer_name] exe_range = self._get_exe_range(proc) if not exe_range: - vollog.warning("Could not find the application executable VAD for services.exe. Unable to proceed.") + vollog.warning( + "Could not find the application executable VAD for services.exe. Unable to proceed." + ) continue for offset in layer.scan( context=self.context, - scanner=scanners.BytesScanner(needle = b"Sc27"), + scanner=scanners.BytesScanner(needle=b"Sc27"), sections=exe_range, + ): + for record in self.enumerate_vista_or_later_header( + service_table_name, service_binary_dll_map, layer_name, offset ): - for record in self.enumerate_vista_or_later_header(service_table_name, service_binary_dll_map, layer_name, offset): yield record def _generator(self): service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - + for record in self.service_list(service_table_name, service_binary_dll_map, filter_func): yield (0, record) - diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 08d11a946..c991ec943 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -233,11 +233,7 @@ class SvcScan(interfaces.plugins.PluginInterface): } def enumerate_vista_or_later_header( - self, - service_table_name, - service_binary_dll_map, - proc_layer_name, - offset + self, service_table_name, service_binary_dll_map, proc_layer_name, offset ): if offset % 8: return @@ -324,13 +320,17 @@ class SvcScan(interfaces.plugins.PluginInterface): ) yield self.get_record_tuple(service_record, service_info) else: - for service_record in self.enumerate_vista_or_later_header(service_table_name, service_binary_dll_map, proc_layer_name, offset): + for service_record in self.enumerate_vista_or_later_header( + service_table_name, + service_binary_dll_map, + proc_layer_name, + offset + ): if service_record in seen: break seen.append(service_record) yield service_record - def get_prereq_info(self): """ Data structures and information needed to analyze service information @@ -356,7 +356,9 @@ class SvcScan(interfaces.plugins.PluginInterface): def _generator(self): service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - for record in self.service_scan(service_table_name, service_binary_dll_map, filter_func): + for record in self.service_scan( + service_table_name, service_binary_dll_map, filter_func + ): yield (0, record) def run(self): From d42fb0a1451ca0a0cc00ee4a2a431be092c193b9 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 14:56:57 -0500 Subject: [PATCH 187/979] Format fixes --- volatility3/framework/plugins/windows/svclist.py | 4 +++- volatility3/framework/plugins/windows/svcscan.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index 53ca68da7..1938dd182 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -96,5 +96,7 @@ class SvcList(svcscan.SvcScan): def _generator(self): service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - for record in self.service_list(service_table_name, service_binary_dll_map, filter_func): + for record in self.service_list( + service_table_name, service_binary_dll_map, filter_func + ): yield (0, record) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index c991ec943..07274f03d 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -324,7 +324,7 @@ class SvcScan(interfaces.plugins.PluginInterface): service_table_name, service_binary_dll_map, proc_layer_name, - offset + offset, ): if service_record in seen: break From f6a053d7b3db2421c9d4d8501eff1fcb78001f1f Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 14:58:05 -0500 Subject: [PATCH 188/979] Format fixes --- volatility3/framework/plugins/windows/svcdiff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py index 71aa03636..809064946 100644 --- a/volatility3/framework/plugins/windows/svcdiff.py +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -20,6 +20,7 @@ from volatility3.framework.symbols.windows import versions vollog = logging.getLogger(__name__) + class SvcDiff(svclist.SvcList, svcscan.SvcScan): """Compares services found through list walking versus scanning to find rootkits""" @@ -80,4 +81,3 @@ class SvcDiff(svclist.SvcList, svcscan.SvcScan): # report services found from scanning but not list walking for hidden_service in from_scan - from_list: yield (0, records[hidden_service]) - From 3d1b9ef2bc1e968eed6f41195834752d2d328ec9 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 20:09:05 -0500 Subject: [PATCH 189/979] Add new plugin to detect hollowed processes using a variety of techniques and allowing for easy additions of future detection techniques --- .../plugins/windows/hollowprocesses.py | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 volatility3/framework/plugins/windows/hollowprocesses.py diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py new file mode 100644 index 000000000..a832e89e1 --- /dev/null +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -0,0 +1,175 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import NamedTuple + +from volatility3.framework import interfaces, exceptions, constants +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.plugins.windows import pslist, vadinfo + +vollog = logging.getLogger(__name__) + +VadInfo = NamedTuple( + "VadInfo", + [ + ("protection", str), + ("path", str), + ], +) + +DLLInfo = NamedTuple( + "DLLInfo", + [ + ("path", str), + ], +) + +class HollowProcesses(interfaces.plugins.PluginInterface): + """Lists hollowed processes""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + ] + + def _get_vads_map(self, proc): + vads = {} + + kernel = self.context.modules[self.config["kernel"]] + + for vad in proc.get_vad_root().traverse(): + protection_string = vad.get_protection( + vadinfo.VadInfo.protect_values( + self.context, kernel.layer_name, kernel.symbol_table_name + ), + vadinfo.winnt_protections, + ) + + fn = vad.get_file_name() + if not fn or not isinstance(fn, str): + fn = "" + + vads[vad.get_start()] = VadInfo(protection_string, fn) + + return vads + + def _get_dlls_map(self, proc): + dlls = {} + + for entry in proc.load_order_modules(): + try: + base = entry.DllBase + except exceptions.InvalidAddressException: + continue + + try: + FullDllName = entry.FullDllName.get_string() + except exceptions.InvalidAddressException: + FullDllName = renderers.UnreadableValue() + + dlls[base] = DLLInfo(FullDllName) + + return dlls + + def _get_image_base(self, proc): + kernel = self.context.modules[self.config["kernel"]] + + try: + proc_layer_name = proc.add_process_layer() + peb = self.context.object( + kernel.symbol_table_name + constants.BANG + "_PEB", + layer_name=proc_layer_name, + offset=proc.Peb, + ) + return peb.ImageBaseAddress + except exceptions.InvalidAddressException: + return None + + def _check_load_address(self, proc, _, __): + image_base = self._get_image_base(proc) + if image_base is not None and image_base != proc.SectionBaseAddress: + yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format(image_base, proc.SectionBaseAddress) + + def _check_exe_protection(self, proc, vads, __): + base = proc.SectionBaseAddress + + if base not in vads: + yield "There is no VAD starting at the base address of the process executable ({:#x})".format(base) + elif vads[base].protection != "PAGE_EXECUTE_WRITECOPY": + yield "Unexpected protection ({}) for VAD hosting the process executable ({:#x}) with path {}".format(vads[base].protection, base, vads[base].path) + + def _check_dlls_protection(self, _, vads, dlls): + for dll_base in dlls: + # could be malicious but triggers too many FPs from smear + if dll_base not in vads: + continue + + if vads[dll_base].protection != "PAGE_EXECUTE_WRITECOPY": + yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format(vads[dll_base].protection, dll_base, dlls[dll_base].path) + + def _generator(self, procs): + checks = [self._check_load_address, self._check_exe_protection, self._check_dlls_protection] + + for proc in procs: + proc_name = utility.array_to_string(proc.ImageFileName) + pid = proc.UniqueProcessId + + # smear and/or terminated process + dlls = self._get_dlls_map(proc) + if len(dlls) < 3: + continue + + vads = self._get_vads_map(proc) + if len(vads) < 5: + continue + + for check in checks: + for note in check(proc, vads, dlls): + yield 0, ( + pid, + proc_name, + note, + ) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] + + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Notes", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + ), + ) From 02bda980d429a49c18a1b908b5fcbfda8de5ad09 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 20:12:07 -0500 Subject: [PATCH 190/979] formatting fixes --- .../plugins/windows/hollowprocesses.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index a832e89e1..18fb24c8f 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging @@ -17,14 +17,14 @@ VadInfo = NamedTuple( [ ("protection", str), ("path", str), - ], + ], ) DLLInfo = NamedTuple( "DLLInfo", [ ("path", str), - ], + ], ) class HollowProcesses(interfaces.plugins.PluginInterface): @@ -111,15 +111,21 @@ class HollowProcesses(interfaces.plugins.PluginInterface): def _check_load_address(self, proc, _, __): image_base = self._get_image_base(proc) if image_base is not None and image_base != proc.SectionBaseAddress: - yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format(image_base, proc.SectionBaseAddress) + yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format( + image_base, proc.SectionBaseAddress + ) def _check_exe_protection(self, proc, vads, __): base = proc.SectionBaseAddress if base not in vads: - yield "There is no VAD starting at the base address of the process executable ({:#x})".format(base) + yield "There is no VAD starting at the base address of the process executable ({:#x})".format( + base + ) elif vads[base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for VAD hosting the process executable ({:#x}) with path {}".format(vads[base].protection, base, vads[base].path) + yield "Unexpected protection ({}) for VAD hosting the process executable ({:#x}) with path {}".format( + vads[base].protection, base, vads[base].path + ) def _check_dlls_protection(self, _, vads, dlls): for dll_base in dlls: @@ -128,10 +134,16 @@ class HollowProcesses(interfaces.plugins.PluginInterface): continue if vads[dll_base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format(vads[dll_base].protection, dll_base, dlls[dll_base].path) + yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format( + vads[dll_base].protection, dll_base, dlls[dll_base].path + ) def _generator(self, procs): - checks = [self._check_load_address, self._check_exe_protection, self._check_dlls_protection] + checks = [ + self._check_load_address, + self._check_exe_protection, + self._check_dlls_protection + ] for proc in procs: proc_name = utility.array_to_string(proc.ImageFileName) From 178f7c45f7ed1413277baba486929e5f2afae7fc Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 20:13:04 -0500 Subject: [PATCH 191/979] formatting fixes --- volatility3/framework/plugins/windows/hollowprocesses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index 18fb24c8f..9d7e800b7 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -27,6 +27,7 @@ DLLInfo = NamedTuple( ], ) + class HollowProcesses(interfaces.plugins.PluginInterface): """Lists hollowed processes""" @@ -142,7 +143,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): checks = [ self._check_load_address, self._check_exe_protection, - self._check_dlls_protection + self._check_dlls_protection, ] for proc in procs: From a68a4826329dbd44e95d4e173ea467be24dbc99b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 20 Jun 2024 04:00:20 +0200 Subject: [PATCH 192/979] fix #509 and simplify logic in _load_segments --- volatility3/framework/layers/crash.py | 105 +++++++++++++++----------- 1 file changed, 62 insertions(+), 43 deletions(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 042b18ddc..2d0f37008 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -96,12 +96,13 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): def get_summary_header(self) -> interfaces.objects.ObjectInterface: return self.context.object( self._crash_common_table_name + constants.BANG + "_SUMMARY_DUMP", - offset=0x1000 * self.headerpages, + offset=self._page_size * self.headerpages, layer_name=self._base_layer, ) def _load_segments(self) -> None: - """Loads up the segments from the meta_layer.""" + """Loads up the segments from the meta_layer. + A segment is a set of contiguous memory pages.""" segments = [] @@ -119,70 +120,88 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): for run in header.PhysicalMemoryBlockBuffer.Run: segments.append( ( - run.BasePage * 0x1000, - offset * 0x1000, - run.PageCount * 0x1000, - run.PageCount * 0x1000, + run.BasePage * self._page_size, + offset * self._page_size, + run.PageCount * self._page_size, + run.PageCount * self._page_size, ) ) offset += run.PageCount elif self.dump_type == 0x05: summary_header = self.get_summary_header() - first_bit = None # First bit in a run - first_offset = 0 # File offset of first bit - last_bit_seen = 0 # Most recent bit processed - offset = summary_header.HeaderSize # Size of file headers - buffer_char = summary_header.get_buffer_char() - buffer_long = summary_header.get_buffer_long() - - for outer_index in range(0, ((summary_header.BitmapSize + 31) // 32)): - if buffer_long[outer_index] == 0: - if first_bit is not None: - last_bit = ((outer_index - 1) * 32) + 31 - segment_length = (last_bit - first_bit + 1) * 0x1000 + seg_first_bit = None # First bit in a run + seg_first_offset = 0 # File offset of first bit + offset = ( + summary_header.HeaderSize + ) # Offset to the start of actual memory dump + ulong_bitmap_array = summary_header.get_buffer_long() + # outer_index points to a 32 bits array inside a list of arrays, + # each bit indicating a page mapping state + for outer_index in range(0, ulong_bitmap_array.vol.count): + ulong_bitmap = ulong_bitmap_array[outer_index] + # All pages in this 32 bits array are mapped (speedup iteration process) + if ulong_bitmap == 0xFFFFFFFF: + # New segment + if seg_first_bit is None: + seg_first_offset = offset + seg_first_bit = outer_index * 32 + offset += 32 * self._page_size + # No pages in this 32 bits array are mapped (speedup iteration process) + elif ulong_bitmap == 0: + # End of segment + if seg_first_bit is not None: + last_bit = (outer_index - 1) * 32 + 31 + segment_length = ( + last_bit - seg_first_bit + 1 + ) * self._page_size segments.append( ( - first_bit * 0x1000, - first_offset, + seg_first_bit * self._page_size, + seg_first_offset, segment_length, segment_length, ) ) - first_bit = None - elif buffer_long[outer_index] == 0xFFFFFFFF: - if first_bit is None: - first_offset = offset - first_bit = outer_index * 32 - offset = offset + (32 * 0x1000) + seg_first_bit = None + # Some pages in this 32 bits array are mapped and some aren't else: - for inner_index in range(0, 32): - bit_addr = outer_index * 32 + inner_index - if (buffer_char[bit_addr >> 3] >> (bit_addr & 0x7)) & 1: - if first_bit is None: - first_offset = offset - first_bit = bit_addr - offset = offset + 0x1000 + for inner_bit_position in range(0, 32): + current_bit = outer_index * 32 + inner_bit_position + page_mapped = ulong_bitmap & (1 << inner_bit_position) + if page_mapped: + # New segment + if seg_first_bit is None: + seg_first_offset = offset + seg_first_bit = current_bit + offset += self._page_size else: - if first_bit is not None: + # End of segment + if seg_first_bit is not None: segment_length = ( - (bit_addr - 1) - first_bit + 1 - ) * 0x1000 + current_bit - 1 - seg_first_bit + 1 + ) * self._page_size segments.append( ( - first_bit * 0x1000, - first_offset, + seg_first_bit * self._page_size, + seg_first_offset, segment_length, segment_length, ) ) - first_bit = None - last_bit_seen = (outer_index * 32) + 31 + seg_first_bit = None + else: + last_bit_seen = outer_index * 32 + 31 - if first_bit is not None: - segment_length = (last_bit_seen - first_bit + 1) * 0x1000 + if seg_first_bit is not None: + segment_length = (last_bit_seen - seg_first_bit + 1) * self._page_size segments.append( - (first_bit * 0x1000, first_offset, segment_length, segment_length) + ( + seg_first_bit * self._page_size, + seg_first_offset, + segment_length, + segment_length, + ) ) else: vollog.log( From c688744f66ae6af49123a530dd45abc29ed9abba Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 20 Jun 2024 04:16:32 +0200 Subject: [PATCH 193/979] revert useless for-else --- volatility3/framework/layers/crash.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 2d0f37008..3cfc0a25b 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -190,7 +190,6 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): ) ) seg_first_bit = None - else: last_bit_seen = outer_index * 32 + 31 if seg_first_bit is not None: From ed140e9ac451ab3f83f39f9788c344bfa658cf18 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 20 Jun 2024 13:06:43 -0500 Subject: [PATCH 194/979] placeholder --- .../plugins/windows/suspicious_threads.py | 201 ++++++++++++++++++ .../framework/plugins/windows/thrdscan.py | 59 +++-- .../framework/plugins/windows/threads.py | 80 +++++++ 3 files changed, 310 insertions(+), 30 deletions(-) create mode 100644 volatility3/framework/plugins/windows/suspicious_threads.py create mode 100644 volatility3/framework/plugins/windows/threads.py diff --git a/volatility3/framework/plugins/windows/suspicious_threads.py b/volatility3/framework/plugins/windows/suspicious_threads.py new file mode 100644 index 000000000..b24f3f2c2 --- /dev/null +++ b/volatility3/framework/plugins/windows/suspicious_threads.py @@ -0,0 +1,201 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List +from volatility3.framework import renderers, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist, threads, vadinfo, thrdscan + +vollog = logging.getLogger(__name__) + + +class SupsiciousThreads(interfaces.plugins.PluginInterface): + """Lists suspicious userland process threads""" + + _required_framework_version = (2, 4, 0) + _version = (2, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.PluginRequirement( + name="threads", plugin=threads.Threads, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + ] + + def _get_ranges(self, kernel, all_ranges, proc): + """ + Maintains a hash table so each process' VADs + are only enumerated once per plugin run + """ + key = proc.vol.offset + + if key not in all_ranges: + all_ranges[key] = [] + + for vad in proc.get_vad_root().traverse(): + fn = vad.get_file_name() + if not isinstance(fn, str) or not fn: + fn = None + + protection_string = vad.get_protection( + vadinfo.VadInfo.protect_values( + self.context, kernel.layer_name, kernel.symbol_table_name + ), + vadinfo.winnt_protections, + ) + + all_ranges[key].append( + (vad.get_start(), vad.get_end(), protection_string, fn) + ) + + return all_ranges[key] + + def _get_range(self, ranges, address): + for start, end, protection_string, fn in ranges: + if start <= address < end: + return start, protection_string, fn + + return None, None, None + + def _check_thread_address(self, exe_path, ranges, thread_address): + vad_base, prot, vad_path = self._get_range(ranges, thread_address) + + # threads outside of a VAD means either smear from this thread or this process' VAD tree + if vad_base is None: + return + + if vad_path is None: + # set this so checks after report the non file backed region in the path column + vad_path = "" + + yield ( + vad_path, + "This thread started execution in the VAD starting at base address ({:#x}), which is not backed by a file".format( + vad_base + ), + ) + + if prot != "PAGE_EXECUTE_WRITECOPY": + yield ( + vad_path, + "VAD at base address ({:#x}) hosting this thread has an unexpected starting protection {}".format( + vad_base, prot + ), + ) + + if ( + exe_path + and vad_path.lower().endswith(".exe") + and (vad_path.lower() != exe_path.lower()) + ): + yield ( + vad_path, + "VAD at base address ({:#x}) hosting this thread maps an application executable that is not the process exectuable".format( + vad_base + ), + ) + + def _enumerate_processes(self, kernel, all_ranges): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + for proc in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ): + ranges = self._get_ranges(kernel, all_ranges, proc) + + # smeared vads or process is terminating + if len(all_ranges[proc.vol.offset]) < 5: + continue + + pid = proc.UniqueProcessId + proc_name = utility.array_to_string(proc.ImageFileName) + + _, __, exe_path = self._get_range(ranges, proc.SectionBaseAddress) + if not isinstance(exe_path, str): + exe_path = None + + yield proc, pid, proc_name, exe_path, ranges + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + all_ranges = {} + + for proc, pid, proc_name, exe_path, ranges in self._enumerate_processes( + kernel, all_ranges + ): + # processes often schedule multiple threads at the same address + # there is no benefit to checking the same address more than once per process + checked = set() + + for thread in threads.Threads.list_threads(kernel, proc): + # do not process if a thread is exited or terminated (4 = Terminated) + if thread.ExitTime.QuadPart > 0 or thread.Tcb.State == 4: + continue + + # bail if accessing the threads members causes a page fault + info = thrdscan.ThrdScan.gather_thread_info(thread) + if not info: + continue + + tid, start_address = info[2], info[3] + + addresses = [ + (start_address, "Start"), + (thread.Win32StartAddress, "Win32Start"), + ] + + for address, context in addresses: + if address in checked: + continue + checked.add(address) + + for vad_path, note in self._check_thread_address( + exe_path, ranges, address + ): + yield 0, ( + proc_name, + pid, + tid, + context, + format_hints.Hex(address), + vad_path, + note, + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("TID", int), + ("Context", str), + ("Address", format_hints.Hex), + ("VAD Path", str), + ("Note", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 80906b3b9..bbf65cd6c 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -62,42 +62,41 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) _constraint, mem_object, _header = result yield mem_object + @classmethod + def gather_thread_info(cls, ethread): + try: + thread_offset = ethread.vol.offset + owner_proc_pid = ethread.Cid.UniqueProcess + thread_tid = ethread.Cid.UniqueThread + thread_start_addr = ethread.StartAddress + thread_create_time = ( + ethread.get_create_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_exit_time = ( + ethread.get_exit_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + except exceptions.InvalidAddressException: + vollog.debug("Thread invalid address {:#x}".format(thread.vol.offset)) + return None + + return ( + format_hints.Hex(thread_offset), + owner_proc_pid, + thread_tid, + format_hints.Hex(thread_start_addr), + thread_create_time, + thread_exit_time, + ) + def _generator(self): kernel = self.context.modules[self.config["kernel"]] for ethread in self.scan_threads( self.context, kernel.layer_name, kernel.symbol_table_name ): - try: - thread_offset = ethread.vol.offset - owner_proc_pid = ethread.Cid.UniqueProcess - thread_tid = ethread.Cid.UniqueThread - thread_start_addr = ethread.StartAddress - thread_create_time = ( - ethread.get_create_time() - ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object - thread_exit_time = ( - ethread.get_exit_time() - ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object - except (ValueError, exceptions.InvalidAddressException): - vollog.debug( - "Thread :{}, invalid address {} in layer {}".format( - thread_tid, thread_start_addr, kernel.layer_name - ) - ) - continue - - yield ( - 0, - ( - format_hints.Hex(thread_offset), - owner_proc_pid, - thread_tid, - format_hints.Hex(thread_start_addr), - thread_create_time, - thread_exit_time, - ), - ) + info = self.gather_thread_info(ethread) + if info: + yield (0, info) def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py new file mode 100644 index 000000000..80ff458cd --- /dev/null +++ b/volatility3/framework/plugins/windows/threads.py @@ -0,0 +1,80 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import Callable, List, Generator, Iterable, Type, Optional + +from volatility3.framework import renderers, interfaces, constants, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist, thrdscan + +vollog = logging.getLogger(__name__) + + +class Threads(thrdscan.ThrdScan): + """Lists process threads""" + + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.PluginRequirement( + name="thrdscan", plugin=thrdscan.ThrdScan, version=(1, 0, 0) + ), + ] + + @classmethod + def list_threads( + cls, kernel, proc: interfaces.objects.ObjectInterface + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Lists the Threads of a specific process. + + Args: + proc: _EPROCESS object from which to list the VADs + filter_func: Function to take a virtual address descriptor value and return True if it should be filtered out + + Returns: + A list of threads based on the process and filtered based on the filter function + """ + seen = set() + for thread in proc.ThreadListHead.to_list( + f"{kernel.symbol_table_name}{constants.BANG}_ETHREAD", "ThreadListEntry" + ): + if thread.vol.offset in seen: + break + seen.add(thread.vol.offset) + yield thread + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + kernel_layer = self.context.layers[kernel.layer_name] + + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + for proc in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ): + for thread in self.list_threads(kernel, proc): + info = self.gather_thread_info(thread) + if info: + yield (0, info) From b601250279fe206b2586d82820b38e1c2ea6fd83 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 08:29:50 -0500 Subject: [PATCH 195/979] #1175 - initial unloadedmodules plugin --- .../plugins/windows/unloadedmodules.py | 159 ++++++++++++++++++ .../symbols/windows/unloadedmodules-x64.json | 109 ++++++++++++ .../symbols/windows/unloadedmodules-x86.json | 109 ++++++++++++ 3 files changed, 377 insertions(+) create mode 100644 volatility3/framework/plugins/windows/unloadedmodules.py create mode 100644 volatility3/framework/symbols/windows/unloadedmodules-x64.json create mode 100644 volatility3/framework/symbols/windows/unloadedmodules-x86.json diff --git a/volatility3/framework/plugins/windows/unloadedmodules.py b/volatility3/framework/plugins/windows/unloadedmodules.py new file mode 100644 index 000000000..1fd4914ac --- /dev/null +++ b/volatility3/framework/plugins/windows/unloadedmodules.py @@ -0,0 +1,159 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import datetime +from typing import List, Iterable + +from volatility3.framework import constants +from volatility3.framework import interfaces, symbols +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, conversion +from volatility3.framework.symbols import intermed +from volatility3.plugins import timeliner + +vollog = logging.getLogger(__name__) + + +class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): + """Lists the unloaded kernel modules.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + ] + + @staticmethod + def create_unloadedmodules_table( + context: interfaces.context.ContextInterface, + symbol_table: str, + config_path: str, + ) -> str: + """Creates a symbol table for the unloaded modules. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + symbol_table: The name of an existing symbol table containing the kernel symbols + config_path: The configuration path within the context of the symbol table to create + + Returns: + The name of the constructed unloaded modules table + """ + native_types = context.symbol_space[symbol_table].natives + is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) + table_mapping = {"nt_symbols": symbol_table} + + if is_64bit: + symbol_filename = "unloadedmodules-x64" + else: + symbol_filename = "unloadedmodules-x86" + + return intermed.IntermediateSymbolTable.create( + context, + config_path, + "windows", + symbol_filename, + native_types=native_types, + table_mapping=table_mapping, + ) + + @classmethod + def list_unloadedmodules( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + unloadedmodule_table_name: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Lists all the unloaded modules in the primary layer. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + + Returns: + A list of Unloaded Modules as retrieved from MmUnloadedDrivers + """ + + kvo = context.layers[layer_name].config["kernel_virtual_offset"] + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) + unloadedmodules_offset = ntkrnlmp.get_symbol("MmUnloadedDrivers").address + unloadedmodules = ntkrnlmp.object( + object_type="pointer", + offset=unloadedmodules_offset, + subtype="array", + ) + is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) + + if is_64bit: + unloaded_count_type = "unsigned long long" + else: + unloaded_count_type = "unsigned long" + + last_unloadedmodule_offset = ntkrnlmp.get_symbol("MmLastUnloadedDriver").address + unloaded_count = ntkrnlmp.object( + object_type=unloaded_count_type, offset=last_unloadedmodule_offset + ) + + unloadedmodules_array = context.object( + object_type=unloadedmodule_table_name + + constants.BANG + + "_UNLOADED_DRIVERS", + layer_name=layer_name, + offset=unloadedmodules, + ) + unloadedmodules_array.UnloadedDrivers.count = unloaded_count + + for mod in unloadedmodules_array.UnloadedDrivers: + yield mod + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + unloadedmodule_table_name = self.create_unloadedmodules_table( + self.context, kernel.symbol_table_name, self.config_path + ) + + for mod in self.list_unloadedmodules( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + unloadedmodule_table_name, + ): + yield ( + 0, + ( + mod.Name.String, + format_hints.Hex(mod.StartAddress), + format_hints.Hex(mod.EndAddress), + conversion.wintime_to_datetime(mod.CurrentTime), + ), + ) + + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + description = f"Unloaded Module: {row_data[0]}" + yield (description, timeliner.TimeLinerType.CHANGED, row_data[3]) + + def run(self): + return renderers.TreeGrid( + [ + ("Name", str), + ("StartAddress", format_hints.Hex), + ("EndAddress", format_hints.Hex), + ("Time", datetime.datetime), + ], + self._generator(), + ) diff --git a/volatility3/framework/symbols/windows/unloadedmodules-x64.json b/volatility3/framework/symbols/windows/unloadedmodules-x64.json new file mode 100644 index 000000000..75ab7c690 --- /dev/null +++ b/volatility3/framework/symbols/windows/unloadedmodules-x64.json @@ -0,0 +1,109 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_UNLOADED_DRIVER": { + "fields": { + "Name": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "StartAddress": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 16 + }, + "EndAddress": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 24 + }, + "CurrentTime": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 40 + }, + "_UNLOADED_DRIVERS": { + "fields": { + "UnloadedDrivers": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_UNLOADED_DRIVER" + } + } + } + }, + "kind": "struct", + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle by hand", + "datetime": "2024-06-19T17:57:16.394003" + }, + "format": "4.0.0" + } +} \ No newline at end of file diff --git a/volatility3/framework/symbols/windows/unloadedmodules-x86.json b/volatility3/framework/symbols/windows/unloadedmodules-x86.json new file mode 100644 index 000000000..ff9e78965 --- /dev/null +++ b/volatility3/framework/symbols/windows/unloadedmodules-x86.json @@ -0,0 +1,109 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_UNLOADED_DRIVER": { + "fields": { + "Name": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "StartAddress": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 8 + }, + "EndAddress": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 12 + }, + "CurrentTime": { + "type": { + "kind": "base", + "name": "unsigned long long" + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_UNLOADED_DRIVERS": { + "fields": { + "UnloadedDrivers": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_UNLOADED_DRIVER" + } + } + } + }, + "kind": "struct", + "size": 4 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle by hand", + "datetime": "2024-06-19T17:57:16.394003" + }, + "format": "4.0.0" + } +} \ No newline at end of file From e172676d3caaecd3592211cd561aa612d934bcbc Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 15:07:18 -0500 Subject: [PATCH 196/979] #118 - initial timers plugin --- volatility3/framework/objects/utility.py | 21 ++ .../framework/plugins/windows/timers.py | 264 ++++++++++++++++++ .../framework/symbols/windows/__init__.py | 1 + .../symbols/windows/extensions/__init__.py | 80 ++++++ 4 files changed, 366 insertions(+) create mode 100644 volatility3/framework/plugins/windows/timers.py diff --git a/volatility3/framework/objects/utility.py b/volatility3/framework/objects/utility.py index 0292608c1..8aa527cdb 100644 --- a/volatility3/framework/objects/utility.py +++ b/volatility3/framework/objects/utility.py @@ -7,6 +7,27 @@ from typing import Optional, Union from volatility3.framework import interfaces, objects, constants +def rol(value: int, count: int, max_bits: int = 64) -> int: + """A rotate-left instruction in Python""" + max_bits_mask = (1 << max_bits) - 1 + return (value << count % max_bits) & max_bits_mask | ( + (value & max_bits_mask) >> (max_bits - (count % max_bits)) + ) + + +def bswap_32(value: int) -> int: + value = ((value << 8) & 0xFF00FF00) | ((value >> 8) & 0x00FF00FF) + + return ((value << 16) | (value >> 16)) & 0xFFFFFFFF + + +def bswap_64(value: int) -> int: + low = bswap_32((value >> 32)) + high = bswap_32((value & 0xFFFFFFFF)) + + return ((high << 32) | low) & 0xFFFFFFFFFFFFFFFF + + def array_to_string( array: "objects.Array", count: Optional[int] = None, errors: str = "replace" ) -> interfaces.objects.ObjectInterface: diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py new file mode 100644 index 000000000..19d52d9bf --- /dev/null +++ b/volatility3/framework/plugins/windows/timers.py @@ -0,0 +1,264 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import Iterator, List, Tuple, Iterable + +from volatility3.framework import exceptions, layers, renderers, interfaces, constants, symbols +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols.windows import versions +from volatility3.plugins.windows import ssdt + +vollog = logging.getLogger(__name__) + + +class Timers(interfaces.plugins.PluginInterface): + """Print kernel timers and associated module DPCs""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) + ), + ] + + @classmethod + def get_kernel_module( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ): + """Returns the kernel module based on the layer and symbol_table""" + virtual_layer = context.layers[layer_name] + if not isinstance(virtual_layer, layers.intel.Intel): + raise TypeError("Virtual Layer is not an intel layer") + + kvo = virtual_layer.config["kernel_virtual_offset"] + + ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) + return ntkrnlmp + + @classmethod + def get_kpcrs( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> interfaces.objects.ObjectInterface: + """Returns the KPCR structure for each processor + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + symbol_table: The name of an existing symbol table containing the kernel symbols + config_path: The configuration path within the context of the symbol table to create + + Returns: + The _KPCR structure for each processor + """ + + ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table) + cpu_count_offset = ntkrnlmp.get_symbol("KeNumberProcessors").address + cpu_count = ntkrnlmp.object( + object_type="unsigned int", layer_name=layer_name, offset=cpu_count_offset + ) + processor_block = ntkrnlmp.object( + object_type="pointer", + layer_name=layer_name, + offset=ntkrnlmp.get_symbol("KiProcessorBlock").address, + ) + processor_pointers = utility.array_of_pointers( + context=context, + array=processor_block, + count=cpu_count, + subtype=symbol_table + constants.BANG + "_KPRCB", + ) + for pointer in processor_pointers: + kprcb = pointer.dereference() + reloff = ntkrnlmp.get_type("_KPCR").relative_child_offset("Prcb") + kpcr = context.object( + symbol_table + constants.BANG + "_KPCR", + offset=kprcb.vol.offset - reloff, + layer_name=layer_name, + ) + yield kpcr + + @classmethod + def list_timers( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[Tuple[str, int, str]]: + """Lists all kernel timers. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + + Yields: + A _KTIMER entry + """ + ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table) + + if versions.is_windows_7( + context=context, symbol_table=symbol_table + ) or versions.is_windows_8_or_later( + context=context, symbol_table=symbol_table + ): + # Starting with Windows 7, there is no more KiTimerTableListHead. The list is + # at _KPCR.PrcbData.TimerTable.TimerEntries + # See http://pastebin.com/FiRsGW3f + for kpcr in cls.get_kpcrs(context, layer_name, symbol_table): + if hasattr(kpcr.Prcb.TimerTable, "TableState"): + for timer_entries in kpcr.Prcb.TimerTable.TimerEntries: + for timer_entry in timer_entries: + for timer in timer_entry.Entry.to_list( + symbol_table + constants.BANG + "_KTIMER", + "TimerListEntry", + ): + yield timer + + else: + for timer_entries in kpcr.Prcb.TimerTable.TimerEntries: + for timer in timer_entries.Entry.to_list( + symbol_table + constants.BANG + "_KTIMER", + "TimerListEntry", + ): + yield timer + + elif versions.is_xp_or_2003( + context=context, symbol_table=symbol_table + ) or versions.is_vista_or_later( + context=context, symbol_table=symbol_table + ): + is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) + if is_64bit or versions.is_vista_or_later(context=context, symbol_table=symbol_table): + # On XP x64, Windows 2003 SP1-SP2, and Vista SP0-SP2, KiTimerTableListHead + # is an array of 512 _KTIMER_TABLE_ENTRY structs. + array_size = 512 + else: + # On XP SP0-SP3 x86 and Windows 2003 SP0, KiTimerTableListHead + # is an array of 256 _LIST_ENTRY for _KTIMERs. + array_size = 256 + + timer_table_list_head = ntkrnlmp.object( + object_type="array", + offset=ntkrnlmp.get_symbol("KiTimerTableListHead").address, + subtype=ntkrnlmp.get_type("_LIST_ENTRY"), + count=array_size, + ) + for table in timer_table_list_head: + for timer in table.to_list( + symbol_table + constants.BANG + "_KTIMER", + "TimerListEntry", + ): + yield timer + + else: + raise NotImplementedError("This version of Windows is not supported!") + + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config["kernel"]] + layer_name = kernel.layer_name + symbol_table = kernel.symbol_table_name + + collection = ssdt.SSDT.build_module_collection( + self.context, kernel.layer_name, kernel.symbol_table_name + ) + + for timer in self.list_timers(self.context, layer_name, symbol_table): + if not timer.valid_type(): + continue + try: + dpc = timer.get_dpc() + if dpc == 0: + continue + if dpc.DeferredRoutine == 0: + continue + deferred_routine = dpc.DeferredRoutine + except Exception as e: + continue + + module_symbols = list( + collection.get_module_symbols_by_absolute_location(deferred_routine) + ) + + if module_symbols: + for module_name, symbol_generator in module_symbols: + symbols_found = False + + # we might have multiple symbols pointing to the same location + for symbol in symbol_generator: + symbols_found = True + yield ( + 0, + ( + format_hints.Hex(timer.vol.offset), + timer.get_due_time(), + timer.Period, + timer.get_signaled(), + format_hints.Hex(deferred_routine), + module_name, + symbol.split(constants.BANG)[1], + ), + ) + + # no symbols, but we at least can report the module name + if not symbols_found: + yield ( + 0, + ( + format_hints.Hex(timer.vol.offset), + timer.get_due_time(), + timer.Period, + timer.get_signaled(), + format_hints.Hex(deferred_routine), + module_name, + renderers.NotAvailableValue(), + ), + ) + else: + # no module was found at the absolute location + yield ( + 0, + ( + format_hints.Hex(timer.vol.offset), + timer.get_due_time(), + timer.Period, + timer.get_signaled(), + format_hints.Hex(deferred_routine), + renderers.NotAvailableValue(), + renderers.NotAvailableValue(), + ), + ) + + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("DueTime", str), + ("Period(ms)", int), + ("Signaled", str), + ("Routine", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index abf9f6da3..4aeb22dcf 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -39,6 +39,7 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("_VACB", extensions.VACB) self.set_type_class("_POOL_TRACKER_BIG_PAGES", pool.POOL_TRACKER_BIG_PAGES) self.set_type_class("_IMAGE_DOS_HEADER", pe.IMAGE_DOS_HEADER) + self.set_type_class("_KTIMER", extensions.KTIMER) # Might not necessarily defined in every version of windows self.optional_set_type_class("_IMAGE_NT_HEADERS", pe.IMAGE_NT_HEADERS) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index a8fa7b2ff..040c57bae 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -20,6 +20,7 @@ from volatility3.framework import ( ) from volatility3.framework.interfaces.objects import ObjectInterface from volatility3.framework.layers import intel +from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic from volatility3.framework.symbols.windows.extensions import kdbg, pe, pool @@ -994,6 +995,85 @@ class TOKEN(objects.StructType): vollog.log(constants.LOGLEVEL_VVVV, "Broken Token Privileges.") +class KTIMER(objects.StructType): + """A class for Kernel Timers""" + + VALID_TYPES = { + 8: "TimerNotificationObject", + 9: "TimerSynchronizationObject", + } + + def get_signaled(self): + if self.Header.SignalState: + return "Yes" + return "-" + + def get_raw_dpc(self): + """Returns the encoded DPC since it may not look like a pointer after encoding""" + symbol_table_name = self.get_symbol_table_name() + ulonglong_type = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "unsigned long long" + ) + + return self._context.object( + object_type=ulonglong_type, + layer_name=self.vol.layer_name, + offset=self.Dpc.vol.offset, + ) + def valid_type(self): + return self.Header.Type in self.VALID_TYPES + + def get_due_time(self): + return "{0:#010x}:{1:#010x}".format(self.DueTime.HighPart, self.DueTime.LowPart) + + def get_dpc(self): + """Return Dpc, and if Windows 7 or later, decode it""" + symbol_table_name = self.get_symbol_table_name() + kvo = self._context.layers[self.vol.native_layer_name].config[ + "kernel_virtual_offset" + ] + ntkrnlmp = self._context.module( + symbol_table_name, + layer_name=self.vol.native_layer_name, + offset=kvo, + native_layer_name=self.vol.native_layer_name, + ) + + try: + wait_never = ntkrnlmp.object( + object_type="unsigned long long", + offset=ntkrnlmp.get_symbol("KiWaitNever").address, + ) + + wait_always = ntkrnlmp.object( + object_type="unsigned long long", + offset=ntkrnlmp.get_symbol("KiWaitAlways").address, + ) + except exceptions.SymbolError: + wait_never = None + wait_always = None + + if wait_never is None or wait_always is None: + return self.Dpc + else: + low_byte = (wait_never) & 0xFF + entry = utility.rol(self.get_raw_dpc() ^ wait_never, low_byte) + swap_xor = self.vol.offset | 0xFFFF000000000000 + entry = utility.bswap_64(entry ^ swap_xor) + dpc = entry ^ wait_always + + symbol_table_name = self.get_symbol_table_name() + kdpc_type = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "_KDPC" + ) + + return self._context.object( + object_type=kdpc_type, + layer_name=self.vol.layer_name, + offset=dpc, + ) + + class KTHREAD(objects.StructType): """A class for thread control block objects.""" From c79dc52a854d2a5be41283a720081f63471c0df8 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 15:09:21 -0500 Subject: [PATCH 197/979] #118 - black formatted --- .../framework/plugins/windows/timers.py | 23 +++++++++++-------- .../symbols/windows/extensions/__init__.py | 7 +++--- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index 19d52d9bf..e54b2a13f 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -6,7 +6,14 @@ import logging from typing import Iterator, List, Tuple, Iterable -from volatility3.framework import exceptions, layers, renderers, interfaces, constants, symbols +from volatility3.framework import ( + exceptions, + layers, + renderers, + interfaces, + constants, + symbols, +) from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints @@ -117,9 +124,7 @@ class Timers(interfaces.plugins.PluginInterface): if versions.is_windows_7( context=context, symbol_table=symbol_table - ) or versions.is_windows_8_or_later( - context=context, symbol_table=symbol_table - ): + ) or versions.is_windows_8_or_later(context=context, symbol_table=symbol_table): # Starting with Windows 7, there is no more KiTimerTableListHead. The list is # at _KPCR.PrcbData.TimerTable.TimerEntries # See http://pastebin.com/FiRsGW3f @@ -143,11 +148,11 @@ class Timers(interfaces.plugins.PluginInterface): elif versions.is_xp_or_2003( context=context, symbol_table=symbol_table - ) or versions.is_vista_or_later( - context=context, symbol_table=symbol_table - ): + ) or versions.is_vista_or_later(context=context, symbol_table=symbol_table): is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) - if is_64bit or versions.is_vista_or_later(context=context, symbol_table=symbol_table): + if is_64bit or versions.is_vista_or_later( + context=context, symbol_table=symbol_table + ): # On XP x64, Windows 2003 SP1-SP2, and Vista SP0-SP2, KiTimerTableListHead # is an array of 512 _KTIMER_TABLE_ENTRY structs. array_size = 512 @@ -172,7 +177,6 @@ class Timers(interfaces.plugins.PluginInterface): else: raise NotImplementedError("This version of Windows is not supported!") - def _generator(self) -> Iterator[Tuple]: kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name @@ -248,7 +252,6 @@ class Timers(interfaces.plugins.PluginInterface): ), ) - def run(self): return renderers.TreeGrid( [ diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 040c57bae..b830a9d9f 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -453,9 +453,9 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): ].is_valid(self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name: Union[str, interfaces.renderers.BaseAbsentValue] = ( - renderers.UnreadableValue() - ) + name: Union[ + str, interfaces.renderers.BaseAbsentValue + ] = renderers.UnreadableValue() # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. @@ -1020,6 +1020,7 @@ class KTIMER(objects.StructType): layer_name=self.vol.layer_name, offset=self.Dpc.vol.offset, ) + def valid_type(self): return self.Header.Type in self.VALID_TYPES From 7065446e8ab9efb02254e4415235072d30422596 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 15:17:40 -0500 Subject: [PATCH 198/979] #118 - fix black issues --- volatility3/framework/plugins/windows/timers.py | 1 - volatility3/framework/symbols/windows/extensions/__init__.py | 1 - 2 files changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index e54b2a13f..ec8bc17e9 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -7,7 +7,6 @@ import logging from typing import Iterator, List, Tuple, Iterable from volatility3.framework import ( - exceptions, layers, renderers, interfaces, diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b830a9d9f..895a8c094 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1045,7 +1045,6 @@ class KTIMER(objects.StructType): object_type="unsigned long long", offset=ntkrnlmp.get_symbol("KiWaitNever").address, ) - wait_always = ntkrnlmp.object( object_type="unsigned long long", offset=ntkrnlmp.get_symbol("KiWaitAlways").address, From 2d7789f8509664f3930705bf83524d6b5ec8de33 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 15:49:03 -0500 Subject: [PATCH 199/979] #118 - fix black issues --- .../framework/symbols/windows/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 895a8c094..4d4ffc055 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -453,9 +453,9 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): ].is_valid(self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name: Union[ - str, interfaces.renderers.BaseAbsentValue - ] = renderers.UnreadableValue() + name: Union[str, interfaces.renderers.BaseAbsentValue] = ( + renderers.UnreadableValue() + ) # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. From 146baab14c4a17bdcbc35715b30a4d190ed82214 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 21 Jun 2024 15:54:16 -0500 Subject: [PATCH 200/979] #118 - refactor get_dpc --- .../framework/symbols/windows/extensions/__init__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 4d4ffc055..b9eabcda6 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1040,7 +1040,7 @@ class KTIMER(objects.StructType): native_layer_name=self.vol.native_layer_name, ) - try: + if ntkrnlmp.has_symbol("KiWaitNever") and ntkrnlmp.has_symbol("KiWaitAlways"): wait_never = ntkrnlmp.object( object_type="unsigned long long", offset=ntkrnlmp.get_symbol("KiWaitNever").address, @@ -1049,13 +1049,7 @@ class KTIMER(objects.StructType): object_type="unsigned long long", offset=ntkrnlmp.get_symbol("KiWaitAlways").address, ) - except exceptions.SymbolError: - wait_never = None - wait_always = None - if wait_never is None or wait_always is None: - return self.Dpc - else: low_byte = (wait_never) & 0xFF entry = utility.rol(self.get_raw_dpc() ^ wait_never, low_byte) swap_xor = self.vol.offset | 0xFFFF000000000000 @@ -1072,6 +1066,8 @@ class KTIMER(objects.StructType): layer_name=self.vol.layer_name, offset=dpc, ) + else: + return self.Dpc class KTHREAD(objects.StructType): From 898c0844c464f87ab7bbd1cae6756a69b2a8048e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 21 Jun 2024 16:15:17 -0500 Subject: [PATCH 201/979] Windows: fixes scanner bug for versions < win10 This commit fixes a bug where the `layer_name` gets discarded when constructing objects. Previously, it was assumed that we would not want to construct an object for a module with a layer_name different from that of the module. However, because we switch to scanning the memory layer on samples where the version is < 10, but still construct kernel executive objects based on the result of the memory layer scan, we actually do sometimes need to specify a different layer. --- volatility3/framework/contexts/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 73868a58f..4c169728c 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -256,12 +256,13 @@ class Module(interfaces.context.ModuleInterface): if not absolute: offset += self._offset - # Ensure we don't use a layer_name other than the module's, why would anyone do that? - if "layer_name" in kwargs: - del kwargs["layer_name"] + # We have to allow using an alternative layer name due to pool scanners switching + # to the memory layer for scanning samples prior to Windows 10. + layer_name = kwargs.pop("layer_name", self._layer_name) + return self._context.object( object_type=object_type, - layer_name=self._layer_name, + layer_name=layer_name, offset=offset, native_layer_name=native_layer_name or self._native_layer_name, **kwargs, From 7ece5fb1bf2f27d2731d09ed8329fb20552c713c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 23 Jun 2024 00:16:23 +0100 Subject: [PATCH 202/979] Windows: Improve Virtmap error messages slightly --- volatility3/framework/plugins/windows/virtmap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/virtmap.py b/volatility3/framework/plugins/windows/virtmap.py index 5190bec8d..3f3f270e2 100644 --- a/volatility3/framework/plugins/windows/virtmap.py +++ b/volatility3/framework/plugins/windows/virtmap.py @@ -78,7 +78,7 @@ class VirtMap(interfaces.plugins.PluginInterface): ) else: raise exceptions.SymbolError( - None, module.name, "Required structures not found" + "SystemVaRegions", module.name, "Required structures not found" ) elif module.has_symbol("MiSystemVaType"): system_range_start = module.object( @@ -99,7 +99,7 @@ class VirtMap(interfaces.plugins.PluginInterface): ) else: raise exceptions.SymbolError( - None, module.name, "Required structures not found" + "MiVisibleState", module.name, "Required structures not found" ) return result From 8c1a5c46e32c8ffaacd6507ec036cbd07c9072b7 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 27 Jun 2024 17:13:37 -0500 Subject: [PATCH 203/979] Add timeliner support to userassist --- .../plugins/windows/registry/userassist.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 70c75b50b..54ec2fc74 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -17,11 +17,12 @@ from volatility3.framework.layers.registry import RegistryHive from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.plugins.windows.registry import hivelist +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class UserAssist(interfaces.plugins.PluginInterface): +class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Print userassist registry keys and information.""" _required_framework_version = (2, 0, 0) @@ -335,6 +336,19 @@ class UserAssist(interfaces.plugins.PluginInterface): ) yield result + + def generate_timeline(self): + self._reg_table_name = intermed.IntermediateSymbolTable.create( + self.context, self._config_path, "windows", "registry" + ) + + for row in self._generator(): + _depth, row_data = row + # check the name and the timestamp to not be empty + if isinstance(row_data[5], str) and not isinstance(row_data[10], renderers.NotApplicableValue): + description = f"UserAssist: {row_data[5]} {row_data[2]} ({row_data[7]})" + yield (description, timeliner.TimeLinerType.MODIFIED, row_data[10]) + def run(self): self._reg_table_name = intermed.IntermediateSymbolTable.create( self.context, self._config_path, "windows", "registry" From 8525edd3331c6ccd967c8048d61adf5d2c3e1015 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 27 Jun 2024 17:15:49 -0500 Subject: [PATCH 204/979] Formatting fixes --- volatility3/framework/plugins/windows/registry/userassist.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 54ec2fc74..cf345c901 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -336,7 +336,6 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac ) yield result - def generate_timeline(self): self._reg_table_name = intermed.IntermediateSymbolTable.create( self.context, self._config_path, "windows", "registry" @@ -345,7 +344,9 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac for row in self._generator(): _depth, row_data = row # check the name and the timestamp to not be empty - if isinstance(row_data[5], str) and not isinstance(row_data[10], renderers.NotApplicableValue): + if isinstance(row_data[5], str) and not isinstance( + row_data[10], renderers.NotApplicableValue + ): description = f"UserAssist: {row_data[5]} {row_data[2]} ({row_data[7]})" yield (description, timeliner.TimeLinerType.MODIFIED, row_data[10]) From 857cd8df491310d4c8ed56833a9fae6714f9c582 Mon Sep 17 00:00:00 2001 From: atcuno Date: Tue, 2 Jul 2024 19:14:15 -0500 Subject: [PATCH 205/979] Updates from ikelos' feedback --- .../plugins/windows/hollowprocesses.py | 67 ++++++++++++++++--- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index 9d7e800b7..3158667ac 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import NamedTuple +from typing import NamedTuple, Dict from volatility3.framework import interfaces, exceptions, constants from volatility3.framework import renderers @@ -12,21 +12,25 @@ from volatility3.plugins.windows import pslist, vadinfo vollog = logging.getLogger(__name__) -VadInfo = NamedTuple( - "VadInfo", +VadData = NamedTuple( + "VadData", [ ("protection", str), ("path", str), ], ) -DLLInfo = NamedTuple( - "DLLInfo", +DLLData = NamedTuple( + "DLLData", [ ("path", str), ], ) +### Useful references on process hollowing +# https://cysinfo.com/detecting-deceptive-hollowing-techniques/ +# https://github.com/m0n0ph1/Process-Hollowing + class HollowProcesses(interfaces.plugins.PluginInterface): """Lists hollowed processes""" @@ -56,7 +60,16 @@ class HollowProcesses(interfaces.plugins.PluginInterface): ), ] - def _get_vads_map(self, proc): + def _get_vads_data( + self, proc: interfaces.objects.ObjectInterface + ) -> Dict[int, VadData]: + """ + Returns a dictionary of: + base address -> (protection string, file name) + For each mapped VAD in the process. This is used + for quick lookups of data and matching the DLL + at the same base address as the VAD + """ vads = {} kernel = self.context.modules[self.config["kernel"]] @@ -73,11 +86,23 @@ class HollowProcesses(interfaces.plugins.PluginInterface): if not fn or not isinstance(fn, str): fn = "" - vads[vad.get_start()] = VadInfo(protection_string, fn) + vads[vad.get_start()] = VadData(protection_string, fn) return vads - def _get_dlls_map(self, proc): + def _get_dlls_map( + self, proc: interfaces.objects.ObjectInterface + ) -> Dict[int, DLLData]: + """ + Returns a dictionary of: + base address -> path + for each DLL loaded in the process + + This is used to cross compare with + the corresponding VAD and to have a + backup path source in case of smear + in the VAD + """ dlls = {} for entry in proc.load_order_modules(): @@ -91,11 +116,14 @@ class HollowProcesses(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: FullDllName = renderers.UnreadableValue() - dlls[base] = DLLInfo(FullDllName) + dlls[base] = DLLData(FullDllName) return dlls - def _get_image_base(self, proc): + def _get_image_base(self, proc: interfaces.objects.ObjectInterface) -> int: + """ + Uses the PEB to get the image base of the process + """ kernel = self.context.modules[self.config["kernel"]] try: @@ -110,6 +138,12 @@ class HollowProcesses(interfaces.plugins.PluginInterface): return None def _check_load_address(self, proc, _, __): + """ + Detects when the image base in the PEB, which is writable by process malware, + does not match the section base address - whose value lives in kernel memory. + Many malware samples will manipulate their image base to fool AVs/EDRs and + as a necessary part of certain hollowing techniques + """ image_base = self._get_image_base(proc) if image_base is not None and image_base != proc.SectionBaseAddress: yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format( @@ -117,6 +151,16 @@ class HollowProcesses(interfaces.plugins.PluginInterface): ) def _check_exe_protection(self, proc, vads, __): + """ + Legitimately mapped application executables and DLLs + will have a VAD present and its initial protection will be + PAGE_EXECUTE_WRITECOPY. + Many process hollowing and code injection techniques will + unmap the real executable and/or map in executables with + incorrect permissions. + This check verifies the VAD for the application exe. + `_check_dlls_protection` checks for DLLs mapped in the process. + """ base = proc.SectionBaseAddress if base not in vads: @@ -134,6 +178,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): if dll_base not in vads: continue + # PAGE_EXECUTE_WRITECOPY is the only valid permission for mapped DLLs and .exe files if vads[dll_base].protection != "PAGE_EXECUTE_WRITECOPY": yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format( vads[dll_base].protection, dll_base, dlls[dll_base].path @@ -155,7 +200,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): if len(dlls) < 3: continue - vads = self._get_vads_map(proc) + vads = self._get_vads_data(proc) if len(vads) < 5: continue From 020005f43cf32e9a38622926c184be1b18d9b18e Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 3 Jul 2024 09:55:07 -0500 Subject: [PATCH 206/979] Address feedback from ikelos --- .../plugins/windows/suspicious_threads.py | 45 ++++++++++++------- .../framework/plugins/windows/threads.py | 7 +-- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspicious_threads.py b/volatility3/framework/plugins/windows/suspicious_threads.py index b24f3f2c2..2679b191a 100644 --- a/volatility3/framework/plugins/windows/suspicious_threads.py +++ b/volatility3/framework/plugins/windows/suspicious_threads.py @@ -3,7 +3,7 @@ # import logging -from typing import List +from typing import List, Dict, Tuple, Generator from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility @@ -42,7 +42,12 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): ), ] - def _get_ranges(self, kernel, all_ranges, proc): + def _get_ranges( + self, + kernel: interfaces.context.ModuleInterface, + all_ranges: Dict[int, List[Tuple[int, int, str, str]]], + proc, + ) -> Tuple[int, int, str, str]: """ Maintains a hash table so each process' VADs are only enumerated once per plugin run @@ -70,14 +75,24 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): return all_ranges[key] - def _get_range(self, ranges, address): + def _get_range( + self, ranges: Dict[int, List[Tuple[int, int, str, str]]], address: int + ) -> Tuple[int, str, str]: + """ + Walks a process' VADs looking for the one + containing `address` + + Returns its base address, protection string, and mapped file, if any + """ for start, end, protection_string, fn in ranges: if start <= address < end: return start, protection_string, fn return None, None, None - def _check_thread_address(self, exe_path, ranges, thread_address): + def _check_thread_address( + self, exe_path: str, ranges, thread_address: int + ) -> Generator[Tuple[str, str], None, None]: vad_base, prot, vad_path = self._get_range(ranges, thread_address) # threads outside of a VAD means either smear from this thread or this process' VAD tree @@ -90,19 +105,17 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): yield ( vad_path, - "This thread started execution in the VAD starting at base address ({:#x}), which is not backed by a file".format( - vad_base - ), + f"This thread started execution in the VAD starting at base address ({vad_base:#x}), which is not backed by a file", ) + # All threads should point to PAGE_EXECUTE_WRITECOPY mapped regions if prot != "PAGE_EXECUTE_WRITECOPY": yield ( vad_path, - "VAD at base address ({:#x}) hosting this thread has an unexpected starting protection {}".format( - vad_base, prot - ), + f"VAD at base address ({vad_base:#x}) hosting this thread has an unexpected starting protection {prot}", ) + # check for process hollowing type techniques that mapped in a second, malicious exe file if ( exe_path and vad_path.lower().endswith(".exe") @@ -110,12 +123,12 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): ): yield ( vad_path, - "VAD at base address ({:#x}) hosting this thread maps an application executable that is not the process exectuable".format( - vad_base - ), + "VAD at base address ({vad_base:#x}) hosting this thread maps an application executable that is not the process exectuable", ) - def _enumerate_processes(self, kernel, all_ranges): + def _enumerate_processes( + self, kernel: interfaces.context.ModuleInterface, all_ranges + ): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) for proc in pslist.PsList.list_processes( @@ -147,7 +160,7 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): for proc, pid, proc_name, exe_path, ranges in self._enumerate_processes( kernel, all_ranges ): - # processes often schedule multiple threads at the same address + # processes often create multiple threads at the same address # there is no benefit to checking the same address more than once per process checked = set() @@ -161,7 +174,7 @@ class SupsiciousThreads(interfaces.plugins.PluginInterface): if not info: continue - tid, start_address = info[2], info[3] + _, _, tid, start_address, _, _ = info addresses = [ (start_address, "Start"), diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index 80ff458cd..83d231abb 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -3,12 +3,10 @@ # import logging -from typing import Callable, List, Generator, Iterable, Type, Optional +from typing import List, Generator -from volatility3.framework import renderers, interfaces, constants, exceptions +from volatility3.framework import interfaces, constants from volatility3.framework.configuration import requirements -from volatility3.framework.objects import utility -from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import pslist, thrdscan vollog = logging.getLogger(__name__) @@ -64,7 +62,6 @@ class Threads(thrdscan.ThrdScan): def _generator(self): kernel = self.context.modules[self.config["kernel"]] - kernel_layer = self.context.layers[kernel.layer_name] filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) From acc41c2b8635df059dc77738121edd7b043d52a0 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 20 Feb 2024 16:59:51 -0600 Subject: [PATCH 207/979] Windows: Callbacks - update symbol files Updates the windows callbacks symbol files to include structures that were missing from the original volatility plugin. --- .../symbols/windows/callbacks-x64.json | 184 ++++++++++++++++-- .../symbols/windows/callbacks-x86.json | 179 ++++++++++++++++- 2 files changed, 342 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/symbols/windows/callbacks-x64.json b/volatility3/framework/symbols/windows/callbacks-x64.json index 87682cb92..3f891b94f 100644 --- a/volatility3/framework/symbols/windows/callbacks-x64.json +++ b/volatility3/framework/symbols/windows/callbacks-x64.json @@ -1,6 +1,18 @@ { "symbols": {}, - "enums": {}, + "enums": { + "EventCategory": { + "base": "long", + "constants": { + "EventCategoryReserved": 0, + "EventCategoryHardwareProfileChange": 1, + "EventCategoryDeviceInterfaceChange": 2, + "EventCategoryTargetDeviceChange": 3, + "EventCategoryKernelSoftRestart": 4 + }, + "size": 4 + } + }, "base_types": { "unsigned long": { "kind": "int", @@ -48,19 +60,161 @@ "user_types": { "_GENERIC_CALLBACK": { "fields": { - "Callback": { - "type": { - "kind": "pointer", - "subtype": { - "kind": "base", - "name": "void" - } + "Callback": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 8 }, - "offset": 8 - } + "Associated": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + } }, "kind": "struct", - "size": 16 + "size": 24 + }, + "_NOTIFICATION_PACKET": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "DriverObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DRIVER_OBJECT" + } + }, + "offset": 16 + }, + "NotificationRoutine": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 48 + }, + "_SHUTDOWN_PACKET": { + "fields": { + "Entry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "DeviceObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DEVICE_OBJECT" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_DBGPRINT_CALLBACK": { + "fields": { + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_NOTIFY_ENTRY_HEADER": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "EventCategory": { + "type": { + "kind": "enum", + "name": "EventCategory" + }, + "offset": 16 + }, + "CallbackRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "DriverObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DRIVER_OBJECT" + } + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 64 + }, + "_REGISTRY_CALLBACK": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + } + }, + "kind": "struct", + "size": 48 }, "_KBUGCHECK_CALLBACK_RECORD": { "fields": { @@ -184,10 +338,10 @@ }, "metadata": { "producer": { - "version": "0.0.1", - "name": "mhl by hand", - "datetime": "2019-08-27T18:17:16.417006" + "version": "0.0.2", + "name": "dgmcdona by hand", + "datetime": "2024-02-02T19:32:00.000000" }, "format": "4.0.0" } -} \ No newline at end of file +} diff --git a/volatility3/framework/symbols/windows/callbacks-x86.json b/volatility3/framework/symbols/windows/callbacks-x86.json index 702b68a65..63b8c69b4 100644 --- a/volatility3/framework/symbols/windows/callbacks-x86.json +++ b/volatility3/framework/symbols/windows/callbacks-x86.json @@ -1,6 +1,18 @@ { "symbols": {}, - "enums": {}, + "enums": { + "EventCategory": { + "base": "long", + "constants": { + "EventCategoryReserved": 0, + "EventCategoryHardwareProfileChange": 1, + "EventCategoryDeviceInterfaceChange": 2, + "EventCategoryTargetDeviceChange": 3, + "EventCategoryKernelSoftRestart": 4 + }, + "size": 4 + } + }, "base_types": { "unsigned long": { "kind": "int", @@ -46,6 +58,151 @@ } }, "user_types": { + "_NOTIFICATION_PACKET": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "DriverObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DRIVER_OBJECT" + } + }, + "offset": 8 + }, + "NotificationRoutine": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_SHUTDOWN_PACKET": { + "fields": { + "Entry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "DeviceObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DEVICE_OBJECT" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 12 + }, + "_REGISTRY_CALLBACK": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 28 + } + }, + "kind": "struct", + "size": 32 + }, + "_REGISTRY_CALLBACK_LEGACY": { + "fields": { + "CreateTime": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 28 + } + }, + "kind": "struct", + "size": 32 + }, + "_DBGPRINT_CALLBACK": { + "fields": { + "Function": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 20 + }, + "_NOTIFY_ENTRY_HEADER": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "EventCategory": { + "type": { + "kind": "enum", + "name": "EventCategory" + }, + "offset": 8 + }, + "CallbackRoutine": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + }, + "DriverObject": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_DRIVER_OBJECT" + } + }, + "offset": 28 + } + }, + "kind": "struct", + "size": 32 + }, "_GENERIC_CALLBACK": { "fields": { "Callback": { @@ -57,10 +214,20 @@ } }, "offset": 4 + }, + "Associated": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 8 } }, "kind": "struct", - "size": 8 + "size": 12 }, "_KBUGCHECK_CALLBACK_RECORD": { "fields": { @@ -184,10 +351,10 @@ }, "metadata": { "producer": { - "version": "0.0.1", - "name": "mhl by hand", - "datetime": "2019-08-27T18:17:16.417006" + "version": "0.0.2", + "name": "dgmcdona by hand", + "datetime": "2024-02-02T19:32:00.000000" }, "format": "4.0.0" } -} \ No newline at end of file +} From 89b024d8f58894008bf26238e24b512c4840a66c Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 20 Feb 2024 17:00:38 -0600 Subject: [PATCH 208/979] Framework: Fix bad format string Fixes a bad format string inside of a raised exception --- volatility3/framework/symbols/intermed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 8c40e466a..751f88e39 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -282,7 +282,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): urls = list(cls.file_symbol_url(sub_path, filename)) if not urls: raise FileNotFoundError( - "No symbol files found at provided filename: {}", filename + f"No symbol files found at provided filename: {filename}", ) table_name = context.symbol_space.free_table_name(filename) table = cls( From 55881535bb39dc610119155a098f2aaa1d259adf Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 20 Feb 2024 17:01:02 -0600 Subject: [PATCH 209/979] Windows: Callbacks - parse missing callback types Updates the Windows callbacks plugin to support several types of callbacks that were present in the original volatility framework, but were missing in volatility3. Adds an extension for `_SHUTDOWN_PACKET` structures for determining validity of structure. --- .../framework/plugins/windows/callbacks.py | 347 ++++++++++++++++-- .../symbols/windows/extensions/callbacks.py | 51 +++ 2 files changed, 368 insertions(+), 30 deletions(-) create mode 100644 volatility3/framework/symbols/windows/extensions/callbacks.py diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index fcc333b9f..d5eeda1ea 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -2,16 +2,24 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import logging import contextlib -from typing import List, Iterable, Tuple, Optional, Union +import logging +from typing import Dict, Iterable, List, Optional, Tuple, Union, cast -from volatility3.framework import constants, exceptions, renderers, interfaces, symbols +from volatility3.framework import ( + constants, + exceptions, + interfaces, + objects, + renderers, + symbols, +) from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions -from volatility3.plugins.windows import ssdt +from volatility3.framework.symbols.windows.extensions import callbacks +from volatility3.plugins.windows import driverirp, handles, poolscanner, ssdt vollog = logging.getLogger(__name__) @@ -20,7 +28,7 @@ class Callbacks(interfaces.plugins.PluginInterface): """Lists kernel callbacks and notification routines.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -33,27 +41,154 @@ class Callbacks(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) ), + requirements.PluginRequirement( + name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="driverirp", plugin=driverirp.DriverIrp, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="handles", plugin=handles.Handles, version=(1, 0, 0) + ), ] - @staticmethod - def create_callback_table( + @classmethod + def create_callback_scan_constraints( + cls, context: interfaces.context.ContextInterface, symbol_table: str, - config_path: str, - ) -> str: - """Creates a symbol table for a set of callbacks. + is_vista_or_above: bool, + ) -> List[poolscanner.PoolConstraint]: + """Creates a list of Pool Tag Constraints for callback objects. Args: context: The context to retrieve required elements (layers, symbol tables) from - symbol_table: The name of an existing symbol table containing the kernel symbols - config_path: The configuration path within the context of the symbol table to create + symbol_table: The name of an existing symbol table containing the symbols / types + is_vista_or_above: A boolean indicating whether the OS version is Vista or newer. Returns: - The name of the constructed callback table + The list containing the built constraints. """ - native_types = context.symbol_space[symbol_table].natives - is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) - table_mapping = {"nt_symbols": symbol_table} + constraints = cls._create_default_scan_constraints(context, symbol_table) + if is_vista_or_above: + constraints += cls._create_scan_constraints_vista_and_above(symbol_table) + return constraints + + @staticmethod + def _create_default_scan_constraints( + context: interfaces.context.ContextInterface, symbol_table: str + ) -> List[poolscanner.PoolConstraint]: + + shutdown_packet_size = context.symbol_space.get_type( + symbol_table + constants.BANG + "_SHUTDOWN_PACKET" + ).size + generic_callback_size = context.symbol_space.get_type( + symbol_table + constants.BANG + "_GENERIC_CALLBACK" + ).size + notification_packet_size = context.symbol_space.get_type( + symbol_table + constants.BANG + "_NOTIFICATION_PACKET" + ).size + + return [ + poolscanner.PoolConstraint( + b"IoFs", + type_name=symbol_table + constants.BANG + "_NOTIFICATION_PACKET", + size=(notification_packet_size, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + ), + poolscanner.PoolConstraint( + b"IoSh", + type_name=symbol_table + constants.BANG + "_SHUTDOWN_PACKET", + size=(shutdown_packet_size, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + index=(0, 0), + ), + poolscanner.PoolConstraint( + b"Cbrb", + type_name=symbol_table + constants.BANG + "_GENERIC_CALLBACK", + size=(generic_callback_size, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + ), + ] + + @staticmethod + def _create_scan_constraints_vista_and_above( + symbol_table: str, + ) -> List[poolscanner.PoolConstraint]: + """Creates a list of Pool Tag Constraints for callback objects. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + symbol_table: The name of an existing symbol table containing the symbols / types + + Returns: + The list containing the built constraints. + """ + + return [ + poolscanner.PoolConstraint( + b"DbCb", + type_name=symbol_table + constants.BANG + "_DBGPRINT_CALLBACK", + size=(0x20, 0x40), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + ), + poolscanner.PoolConstraint( + b"Pnp9", + type_name=symbol_table + constants.BANG + "_NOTIFY_ENTRY_HEADER", + size=(0x30, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + index=(1, 1), + ), + poolscanner.PoolConstraint( + b"PnpD", + type_name=symbol_table + constants.BANG + "_NOTIFY_ENTRY_HEADER", + size=(0x40, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + index=(1, 1), + ), + poolscanner.PoolConstraint( + b"PnpC", + type_name=symbol_table + constants.BANG + "_NOTIFY_ENTRY_HEADER", + size=(0x38, None), + page_type=poolscanner.PoolType.NONPAGED + | poolscanner.PoolType.PAGED + | poolscanner.PoolType.FREE, + index=(1, 1), + ), + ] + + @classmethod + def create_callback_symbol_table( + cls, + context: interfaces.context.ContextInterface, + nt_symbol_table: str, + config_path: str, + ) -> str: + """Creates a symbol table for kernel callback objects. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + nt_symbol_table: The name of the table containing the kernel symbols + config_path: The config path where to find symbol files + + Returns: + The name of the constructed symbol table + """ + native_types = context.symbol_space[nt_symbol_table].natives + is_64bit = symbols.symbol_table_is_64bit(context, nt_symbol_table) + table_mapping = {"nt_symbols": nt_symbol_table} if is_64bit: symbol_filename = "callbacks-x64" @@ -65,10 +200,143 @@ class Callbacks(interfaces.plugins.PluginInterface): config_path, "windows", symbol_filename, + class_types=callbacks.class_types_x86 if not is_64bit else None, native_types=native_types, table_mapping=table_mapping, ) + @classmethod + def scan( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + nt_symbol_table: str, + callback_symbol_table: str, + ) -> Iterable[ + Tuple[ + Union[str, interfaces.renderers.BaseAbsentValue], + int, + Union[str, interfaces.renderers.BaseAbsentValue], + ] + ]: + """Scans for callback objects using the poolscanner module and constraints. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + nt_symbol_table: The name of the table containing the kernel symbols + callback_symbol_table: The name of the table containing the callback object symbols (_SHUTDOWN_PACKET etc.) + + Returns: + A list of callback objects found by scanning the `layer_name` layer for callback pool signatures + """ + is_vista_or_later = versions.is_vista_or_later( + context=context, symbol_table=nt_symbol_table + ) + + type_map = handles.Handles.get_type_map(context, layer_name, nt_symbol_table) + + constraints = cls.create_callback_scan_constraints( + context, callback_symbol_table, is_vista_or_later + ) + + for ( + _constraint, + mem_object, + _header, + ) in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, nt_symbol_table, constraints + ): + try: + if hasattr(mem_object, "is_valid") and not mem_object.is_valid(): + continue + + yield cls._process_scanned_callback(mem_object, type_map) + except exceptions.InvalidAddressException: + continue + + @classmethod + def _process_scanned_callback( + cls, memory_object: objects.StructType, type_map: Dict[int, str] + ) -> Tuple[ + Union[str, interfaces.renderers.BaseAbsentValue], + int, + Union[str, interfaces.renderers.BaseAbsentValue], + ]: + symbol_table = memory_object.get_symbol_table_name() + type_name = memory_object.vol.type_name + + if isinstance(memory_object, callbacks._SHUTDOWN_PACKET) or ( + type_name == symbol_table + constants.BANG + "_SHUTDOWN_PACKET" + ): + callback_type = "IoRegisterShutdownNotification" + + try: + driver = memory_object.DeviceObject.DriverObject + index = driverirp.MAJOR_FUNCTIONS.index("IRP_MJ_SHUTDOWN") + callback_address = driver.MajorFunction[index] + details = driver.DriverName.String or renderers.UnparsableValue() + except exceptions.InvalidAddressException: + callback_address = memory_object.vol.offset + details = renderers.NotApplicableValue() + + elif type_name == symbol_table + constants.BANG + "_NOTIFICATION_PACKET": + callback_type = "IoRegisterFsRegistrationChange" + callback_address = memory_object.NotificationRoutine + details = renderers.NotApplicableValue() + + elif type_name == symbol_table + constants.BANG + "_NOTIFY_ENTRY_HEADER": + driver = ( + memory_object.DriverObject.dereference() + if memory_object.DriverObject.is_readable() + else None + ) + + if driver: + # Instantiate an object header for the driver name + header = driver.get_object_header() + try: + if header.get_object_type(type_map) == "Driver": + # Grab the object name + details = header.NameInfo.Name.String + else: + details = renderers.NotApplicableValue() + except exceptions.InvalidAddressException: + details = renderers.UnreadableValue() + else: + details = renderers.UnreadableValue() + + callback_type = ( + memory_object.EventCategory.description + if memory_object.EventCategory.is_valid_choice + else renderers.UnparsableValue() + ) + + callback_address = memory_object.CallbackRoutine + + elif type_name == symbol_table + constants.BANG + "_GENERIC_CALLBACK": + callback_type = "GenericKernelCallback" + callback_address = memory_object.Callback + details = renderers.NotApplicableValue() + + elif type_name == symbol_table + constants.BANG + "_DBGPRINT_CALLBACK": + callback_type = "DbgSetDebugPrintCallback" + callback_address = memory_object.Function + details = renderers.NotApplicableValue() + + else: + raise ValueError(f"Unexpected object type {type_name}") + + return ( + callback_type, + callback_address, + ( + details + if not isinstance(details, interfaces.renderers.BaseAbsentValue) + else details + ), + ) + @classmethod def list_notify_routines( cls, @@ -115,11 +383,14 @@ class Callbacks(interfaces.plugins.PluginInterface): else: count = 8 - fast_refs = ntkrnlmp.object( - object_type="array", - offset=symbol_offset, - subtype=ntkrnlmp.get_type("_EX_FAST_REF"), - count=count, + fast_refs = cast( + List[objects.Pointer], + ntkrnlmp.object( + object_type="array", + offset=symbol_offset, + subtype=ntkrnlmp.get_type("_EX_FAST_REF"), + count=count, + ), ) for fast_ref in fast_refs: @@ -159,11 +430,14 @@ class Callbacks(interfaces.plugins.PluginInterface): if callback_count == 0: return None - fast_refs = ntkrnlmp.object( - object_type="array", - offset=symbol_offset, - subtype=ntkrnlmp.get_type("_EX_FAST_REF"), - count=callback_count, + fast_refs = cast( + List[objects.Pointer], + ntkrnlmp.object( + object_type="array", + offset=symbol_offset, + subtype=ntkrnlmp.get_type("_EX_FAST_REF"), + count=callback_count, + ), ) for fast_ref in fast_refs: @@ -265,7 +539,13 @@ class Callbacks(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, callback_table_name: str, - ) -> Iterable[Tuple[str, int, str]]: + ) -> Iterable[ + Tuple[ + str, + int, + interfaces.renderers.BaseAbsentValue, + ] + ]: """Lists all kernel bugcheck reason callbacks. Args: @@ -323,7 +603,13 @@ class Callbacks(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, callback_table_name: str, - ) -> Iterable[Tuple[str, int, str]]: + ) -> Iterable[ + Tuple[ + str, + int, + Union[interfaces.objects.ObjectInterface, renderers.UnreadableValue], + ] + ]: """Lists all kernel bugcheck callbacks. Args: @@ -372,7 +658,7 @@ class Callbacks(interfaces.plugins.PluginInterface): def _generator(self): kernel = self.context.modules[self.config["kernel"]] - callback_table_name = self.create_callback_table( + callback_symbol_table = self.create_callback_symbol_table( self.context, kernel.symbol_table_name, self.config_path ) @@ -385,6 +671,7 @@ class Callbacks(interfaces.plugins.PluginInterface): self.list_bugcheck_callbacks, self.list_bugcheck_reason_callbacks, self.list_registry_callbacks, + self.scan, ) for callback_method in callback_methods: @@ -392,7 +679,7 @@ class Callbacks(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, - callback_table_name, + callback_symbol_table, ): if callback_detail is None: detail = renderers.NotApplicableValue() diff --git a/volatility3/framework/symbols/windows/extensions/callbacks.py b/volatility3/framework/symbols/windows/extensions/callbacks.py new file mode 100644 index 000000000..f894644db --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/callbacks.py @@ -0,0 +1,51 @@ +import logging + +from volatility3.framework import exceptions, objects +from volatility3.framework.symbols.windows.extensions import pool + +vollog = logging.getLogger(__name__) + + +class _SHUTDOWN_PACKET(objects.StructType, pool.ExecutiveObject): + """Class for _SHUTDOWN_PACKET objects found in IoSh pools. + + This class serves as a base class for all pooled shutdown callback packets. + + It exposes a function which sanity-checks structure members. + """ + + def is_valid(self) -> bool: + """ + Perform some checks. + """ + try: + if not ( + self.Entry.Flink.is_readable() + and self.Entry.Blink.is_readable() + and self.DeviceObject.is_readable() + ): + return False + + device = self.DeviceObject + if not device or not (device.DriverObject.DriverStart % 0x1000 == 0): + vollog.debug( + f"callback obj 0x{self.vol.offset:x} invalid due to invalid device object" + ) + return False + + except exceptions.InvalidAddressException: + vollog.debug( + f"callback obj 0x{self.vol.offset:x} invalid due to invalid address access" + ) + return False + + try: + header = device.get_object_header() + valid = header.NameInfo.Name == "Device" + return valid + except ValueError: + vollog.debug(f"Could not get NameInfo for object at 0x{self.vol.offset:x}") + return False + + +class_types_x86 = {"_SHUTDOWN_PACKET": _SHUTDOWN_PACKET} From 4d6f4bba0d5127f4a78074266091cf03b808b0f9 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 11 Mar 2024 21:48:58 -0500 Subject: [PATCH 210/979] Windows: Add version to driverirp.DriverIrp plugin This plugin class was missing a `_version` attribute, so I added one and set it to (1, 0, 0). --- volatility3/framework/plugins/windows/driverirp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index 433c61ca2..c3eb7c5c1 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -44,6 +44,7 @@ class DriverIrp(interfaces.plugins.PluginInterface): """List IRPs for drivers in a particular windows memory image.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls): From b36fd84ab57114c1250df57f3017937ba302d5ea Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 4 Jul 2024 13:07:17 +1000 Subject: [PATCH 211/979] Handle virtual_process_from_physical() invalid address exceptions to prevent psscan from breaking --- .../framework/plugins/windows/psscan.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 5634298a8..5ce470cd8 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -266,20 +266,28 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if proc.vol.layer_name == kernel.layer_name: vproc = proc else: - vproc = self.virtual_process_from_physical( - self.context, kernel.layer_name, kernel.symbol_table_name, proc + try: + vproc = self.virtual_process_from_physical( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + proc, + ) + except exceptions.PagedInvalidAddressException: + vproc = None + + file_output = "Error outputting file" + if vproc: + file_handle = pslist.PsList.process_dump( + self.context, + kernel.symbol_table_name, + pe_table_name, + vproc, + self.open, ) - file_handle = pslist.PsList.process_dump( - self.context, - kernel.symbol_table_name, - pe_table_name, - vproc, - self.open, - ) - file_output = "Error outputting file" - if file_handle: - file_output = file_handle.preferred_filename + if file_handle: + file_output = file_handle.preferred_filename if not self.config["physical"]: offset = proc.vol.offset From fcc9eab471ac050670d255bc5f1fba53862b6cf8 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 4 Jul 2024 13:09:12 +1000 Subject: [PATCH 212/979] Improve smear protection on windows processes validation --- .../symbols/windows/extensions/__init__.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index a8fa7b2ff..ae855d6c6 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -22,7 +22,7 @@ from volatility3.framework.interfaces.objects import ObjectInterface from volatility3.framework.layers import intel from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic -from volatility3.framework.symbols.windows.extensions import kdbg, pe, pool +from volatility3.framework.symbols.windows.extensions import pool vollog = logging.getLogger(__name__) @@ -611,11 +611,22 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): ctime = self.get_create_time() if not isinstance(ctime, datetime.datetime): + # A process must have a creation time return False if not (1998 < ctime.year < 2030): return False + etime = self.get_exit_time() + if isinstance(etime, datetime.datetime): + if not (1998 < etime.year < 2030): + return False + + # Exit time, if available, must be after the creation time + # At this point, we are sure both are datetimes, so let's compare them + if ctime > etime: + return False + # NT pids are divisible by 4 if self.UniqueProcessId % 4 != 0: return False @@ -633,7 +644,11 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): if dtb & ~0xFFF == 0: return False - ## TODO: we can also add the thread Flink and Blink tests if necessary + # Quick smear test on thread Flink and Blink + kernel = 0x80000000 # Yes, it's a quick test + list_head = self.ThreadListHead + if list_head.Flink < kernel or list_head.Blink < kernel: + return False except exceptions.InvalidAddressException: return False From 52e6812d390485403740af4ac526e07d39e86e10 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 4 Jul 2024 14:03:59 +1000 Subject: [PATCH 213/979] Fix hexdump text render. Set default to 16 bytes width --- volatility3/cli/text_renderer.py | 33 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index b0ba6baa7..ab9e44141 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -25,7 +25,7 @@ except ImportError: vollog.debug("Disassembly library capstone not found") -def hex_bytes_as_text(value: bytes) -> str: +def hex_bytes_as_text(value: bytes, width: int = 16) -> str: """Renders HexBytes as text. Args: @@ -36,19 +36,24 @@ def hex_bytes_as_text(value: bytes) -> str: """ if not isinstance(value, bytes): raise TypeError(f"hex_bytes_as_text takes bytes not: {type(value)}") - ascii = [] - hex = [] - count = 0 - output = "" - for byte in value: - hex.append(f"{byte:02x}") - ascii.append(chr(byte) if 0x20 < byte <= 0x7E else ".") - if (count % 8) == 7: - output += "\n" - output += " ".join(hex[count - 7 : count + 1]) - output += "\t" - output += "".join(ascii[count - 7 : count + 1]) - count += 1 + + printables = "" + output = "\n" + for count, byte in enumerate(value): + output += f"{byte:02x} " + char = chr(byte) + printables += char if 0x20 <= byte <= 0x7E else "." + if count % width == width - 1: + output += printables + if count < len(value) - 1: + output += "\n" + printables = "" + + # Handle leftovers when the lenght is not mutiple of width + if printables: + output += " " * (width - len(printables)) + output += printables + return output From 43fa84d2d1220d01826dd86c30138e3649d8806c Mon Sep 17 00:00:00 2001 From: Hannah Sarkey Date: Thu, 4 Jul 2024 21:33:55 -0400 Subject: [PATCH 214/979] Update dlllist.py to have more detailed --base description --- volatility3/framework/plugins/windows/dlllist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 4f50bdda5..d48a53663 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -61,7 +61,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), requirements.IntRequirement( name="base", - description="Specify a base address", + description="Specify a base virtual address in process memory", optional=True, ), requirements.BooleanRequirement( From 96a382ed525bb041cbe4fb0917e9af07160ea62a Mon Sep 17 00:00:00 2001 From: atcuno Date: Fri, 5 Jul 2024 09:33:34 -0500 Subject: [PATCH 215/979] Further type information --- .../plugins/windows/hollowprocesses.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index 3158667ac..69fa94f06 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import NamedTuple, Dict +from typing import NamedTuple, Dict, Generator from volatility3.framework import interfaces, exceptions, constants from volatility3.framework import renderers @@ -137,7 +137,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: return None - def _check_load_address(self, proc, _, __): + def _check_load_address(self, proc, _, __) -> Generator[str, None, None]: """ Detects when the image base in the PEB, which is writable by process malware, does not match the section base address - whose value lives in kernel memory. @@ -150,7 +150,9 @@ class HollowProcesses(interfaces.plugins.PluginInterface): image_base, proc.SectionBaseAddress ) - def _check_exe_protection(self, proc, vads, __): + def _check_exe_protection( + self, proc, vads: Dict[int, VadData], __ + ) -> Generator[str, None, None]: """ Legitimately mapped application executables and DLLs will have a VAD present and its initial protection will be @@ -172,7 +174,9 @@ class HollowProcesses(interfaces.plugins.PluginInterface): vads[base].protection, base, vads[base].path ) - def _check_dlls_protection(self, _, vads, dlls): + def _check_dlls_protection( + self, _, vads: Dict[int, VadData], dlls: Dict[int, DLLData] + ) -> Generator[str, None, None]: for dll_base in dlls: # could be malicious but triggers too many FPs from smear if dll_base not in vads: @@ -192,9 +196,6 @@ class HollowProcesses(interfaces.plugins.PluginInterface): ] for proc in procs: - proc_name = utility.array_to_string(proc.ImageFileName) - pid = proc.UniqueProcessId - # smear and/or terminated process dlls = self._get_dlls_map(proc) if len(dlls) < 3: @@ -204,6 +205,9 @@ class HollowProcesses(interfaces.plugins.PluginInterface): if len(vads) < 5: continue + proc_name = utility.array_to_string(proc.ImageFileName) + pid = proc.UniqueProcessId + for check in checks: for note in check(proc, vads, dlls): yield 0, ( From 4938195113613bd262f21d24886d674ae3c27d4d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 6 Jul 2024 17:04:51 +0100 Subject: [PATCH 216/979] Implement the fix that was reverted as part of #1162. --- volatility3/framework/layers/intel.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 75e561b33..b969b952b 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -180,7 +180,9 @@ class Intel(linear.LinearlyMappedLayer): position = self._initial_position entry = self._initial_entry - if self.minimum_address > offset > self.maximum_address: + if not ( + self.minimum_address <= (offset & self.address_mask) <= self.maximum_address + ): raise exceptions.PagedInvalidAddressException( self.name, offset, From bb4cfb6e00a9457081ebf4242c0f18996c4c1d08 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 7 Jul 2024 15:13:06 +0100 Subject: [PATCH 217/979] Initial draft of complete-vad vadyarascan --- .../framework/plugins/windows/vadyarascan.py | 52 ++++++++++++------- volatility3/framework/plugins/yarascan.py | 15 ++++-- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index d795818e9..81530ec67 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -2,10 +2,11 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import io import logging from typing import Iterable, List, Tuple -from volatility3.framework import interfaces, renderers +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan @@ -18,7 +19,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -32,11 +33,8 @@ class VadYaraScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) - ), requirements.PluginRequirement( - name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) + name="yarascan", plugin=yarascan.YaraScan, version=(1, 3, 0) ), requirements.ListRequirement( name="pid", @@ -59,6 +57,8 @@ class VadYaraScan(interfaces.plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + sanity_check = 0x1000 * 0x1000 * 0x1000 + for task in pslist.PsList.list_processes( context=self.context, layer_name=kernel.layer_name, @@ -67,18 +67,34 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ): layer_name = task.add_process_layer() layer = self.context.layers[layer_name] - for offset, rule_name, name, value in layer.scan( - context=self.context, - scanner=yarascan.YaraScanner(rules=rules), - sections=self.get_vad_maps(task), - ): - yield 0, ( - format_hints.Hex(offset), - task.UniqueProcessId, - rule_name, - name, - value, - ) + for start, end in self.get_vad_maps(task): + size = end - start + if size > sanity_check: + vollog.warn( + f"VAD at 0x{start:x} over sanity-check size, not scanning" + ) + continue + + for match in rules.match(data=layer.read(start, end - start, True)): + if yarascan.YaraScan.yara_returns_instances(): + for match_string in match.strings: + for instance in match_string.instances: + yield 0, ( + format_hints.Hex(instance.offset + start), + task.UniqueProcessId, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield 0, ( + format_hints.Hex(offset + start), + task.UniqueProcessId, + match.rule, + name, + value, + ) @staticmethod def get_vad_maps( diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 11c708607..887b6027c 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -2,10 +2,11 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import io import logging from typing import Any, Dict, Iterable, List, Tuple -from volatility3.framework import interfaces, renderers +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import resources @@ -43,7 +44,7 @@ class YaraScanner(interfaces.layers.ScannerInterface): self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: for match in self._rules.match(data=data): - if self.st_object: + if YaraScan.yara_returns_instances(): for match_string in match.strings: for instance in match_string.instances: yield ( @@ -61,7 +62,7 @@ class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" _required_framework_version = (2, 0, 0) - _version = (1, 2, 0) + _version = (1, 3, 0) # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string # or something that makes more sense @@ -119,6 +120,14 @@ class YaraScan(plugins.PluginInterface): ), ] + @classmethod + def yara_returns_instances(self) -> bool: + st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( + 4, + 3, + ) + return st_object + @classmethod def process_yara_options(cls, config: Dict[str, Any]): rules = None From ac15840fd817026d64959b51a63163e8d9bc5261 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 7 Jul 2024 15:23:48 +0100 Subject: [PATCH 218/979] Windows: Fix up yara plugin code scanning warnings --- volatility3/framework/plugins/windows/vadyarascan.py | 3 +-- volatility3/framework/plugins/yarascan.py | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 81530ec67..4b00fee57 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -2,11 +2,10 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import io import logging from typing import Iterable, List, Tuple -from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 887b6027c..496ec1844 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -2,11 +2,10 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import io import logging from typing import Any, Dict, Iterable, List, Tuple -from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import resources @@ -121,7 +120,7 @@ class YaraScan(plugins.PluginInterface): ] @classmethod - def yara_returns_instances(self) -> bool: + def yara_returns_instances(cls) -> bool: st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( 4, 3, From b35169d0bc3e802ced2d81534ac4452ac64f796b Mon Sep 17 00:00:00 2001 From: Brandon Barnacle Date: Thu, 20 Jun 2024 16:00:04 -0400 Subject: [PATCH 219/979] Windows: Bug fixes and additions to modules and modscan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This pull request adds several changes to the modules and modscan plugins that fix bugs, unify options and operations, and allow for filtering by name and base address in both plugins. Previously, modscan and modules had a substantial amount of partially duplicated code, which led to bugs, such as: Only one allowed filtering by --name Only one cycled through the session layers to find the one hosting the kernel module With the new inheritance and combined implementation, the correct and complete implementation applies to both. This PR also addresses the feedback in https://github.com/volatilityfoundation/volatility3/pull/1099. In this PR, --name and –base apply to the processing of modules and determine both which plugins are displayed in output as well as which are processed for extraction. This fixes the issue of using --filters, which only filters the final command line output, but leads to every module being extracted. --- .../framework/plugins/windows/modscan.py | 178 +++--------------- .../framework/plugins/windows/modules.py | 104 ++++++---- 2 files changed, 90 insertions(+), 192 deletions(-) diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index 99fadac07..3a786d2ca 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -9,16 +9,20 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import poolscanner, dlllist, pslist +from volatility3.plugins.windows import poolscanner, dlllist, pslist, modules vollog = logging.getLogger(__name__) -class ModScan(interfaces.plugins.PluginInterface): +class ModScan(modules.Modules): """Scans for modules present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._enumeration_method = self.scan_modules @classmethod def get_requirements(cls): @@ -31,6 +35,9 @@ class ModScan(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="poolscanner", component=poolscanner.PoolScanner, version=(1, 0, 0) ), + requirements.VersionRequirement( + name="modules", component=modules.Modules, version=(1, 1, 0) + ), requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), @@ -43,6 +50,17 @@ class ModScan(interfaces.plugins.PluginInterface): default=False, optional=True, ), + requirements.IntRequirement( + name="base", + description="Extract a single module with BASE address", + optional=True, + ), + requirements.StringRequirement( + name="name", + description="module name/sub string", + optional=True, + default=None, + ), ] @classmethod @@ -72,157 +90,3 @@ class ModScan(interfaces.plugins.PluginInterface): ): _constraint, mem_object, _header = result yield mem_object - - @classmethod - def get_session_layers( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - pids: List[int] = None, - ) -> Generator[str, None, None]: - """Build a cache of possible virtual layers, in priority starting with - the primary/kernel layer. Then keep one layer per session by cycling - through the process list. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - symbol_table: The name of the table containing the kernel symbols - pids: A list of process identifiers to include exclusively or None for no filter - - Returns: - A list of session layer names - """ - seen_ids: List[interfaces.objects.ObjectInterface] = [] - filter_func = pslist.PsList.create_pid_filter(pids or []) - - for proc in pslist.PsList.list_processes( - context=context, - layer_name=layer_name, - symbol_table=symbol_table, - filter_func=filter_func, - ): - proc_id = "Unknown" - try: - proc_id = proc.UniqueProcessId - proc_layer_name = proc.add_process_layer() - - # create the session space object in the process' own layer. - # not all processes have a valid session pointer. - session_space = context.object( - symbol_table + constants.BANG + "_MM_SESSION_SPACE", - layer_name=layer_name, - offset=proc.Session, - ) - - if session_space.SessionId in seen_ids: - continue - - except exceptions.InvalidAddressException: - vollog.log( - constants.LOGLEVEL_VVV, - "Process {} does not have a valid Session or a layer could not be constructed for it".format( - proc_id - ), - ) - continue - - # save the layer if we haven't seen the session yet - seen_ids.append(session_space.SessionId) - yield proc_layer_name - - @classmethod - def find_session_layer( - cls, - context: interfaces.context.ContextInterface, - session_layers: Iterable[str], - base_address: int, - ): - """Given a base address and a list of layer names, find a layer that - can access the specified address. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - symbol_table: The name of the table containing the kernel symbols - session_layers: A list of session layer names - base_address: The base address to identify the layers that can access it - - Returns: - Layer name or None if no layers that contain the base address can be found - """ - - for layer_name in session_layers: - if context.layers[layer_name].is_valid(base_address): - return layer_name - - return None - - def _generator(self): - kernel = self.context.modules[self.config["kernel"]] - - session_layers = list( - self.get_session_layers( - self.context, kernel.layer_name, kernel.symbol_table_name - ) - ) - pe_table_name = intermed.IntermediateSymbolTable.create( - self.context, self.config_path, "windows", "pe", class_types=pe.class_types - ) - - for mod in self.scan_modules( - self.context, kernel.layer_name, kernel.symbol_table_name - ): - try: - BaseDllName = mod.BaseDllName.get_string() - except exceptions.InvalidAddressException: - BaseDllName = "" - - try: - FullDllName = mod.FullDllName.get_string() - except exceptions.InvalidAddressException: - FullDllName = "" - - file_output = "Disabled" - if self.config["dump"]: - session_layer_name = self.find_session_layer( - self.context, session_layers, mod.DllBase - ) - file_output = f"Cannot find a viable session layer for {mod.DllBase:#x}" - if session_layer_name: - file_handle = dlllist.DllList.dump_pe( - self.context, - pe_table_name, - mod, - self.open, - layer_name=session_layer_name, - ) - file_output = "Error outputting file" - if file_handle: - file_output = file_handle.preferred_filename - - yield ( - 0, - ( - format_hints.Hex(mod.vol.offset), - format_hints.Hex(mod.DllBase), - format_hints.Hex(mod.SizeOfImage), - BaseDllName, - FullDllName, - file_output, - ), - ) - - def run(self): - return renderers.TreeGrid( - [ - ("Offset", format_hints.Hex), - ("Base", format_hints.Hex), - ("Size", format_hints.Hex), - ("Name", str), - ("Path", str), - ("File output", str), - ], - self._generator(), - ) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 7dabf6954..8aea3dd45 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -22,6 +22,10 @@ class Modules(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (1, 1, 0) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._enumeration_method = self.list_modules + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ @@ -42,6 +46,11 @@ class Modules(interfaces.plugins.PluginInterface): default=False, optional=True, ), + requirements.IntRequirement( + name="base", + description="Extract a single module with BASE address", + optional=True, + ), requirements.StringRequirement( name="name", description="module name/sub string", @@ -50,49 +59,74 @@ class Modules(interfaces.plugins.PluginInterface): ), ] + def dump_module(self, session_layers, pe_table_name, mod): + session_layer_name = self.find_session_layer( + self.context, session_layers, mod.DllBase + ) + file_output = f"Cannot find a viable session layer for {mod.DllBase:#x}" + if session_layer_name: + file_handle = dlllist.DllList.dump_pe( + self.context, + pe_table_name, + mod, + self.open, + layer_name=session_layer_name, + ) + file_output = "Error outputting file" + if file_handle: + file_output = file_handle.preferred_filename + + return file_output + + def process_module(self, session_layers, pe_table_name, mod): + if self.config["base"] and self.config["base"] != mod.DllBase: + return None + + try: + BaseDllName = mod.BaseDllName.get_string() + except exceptions.InvalidAddressException: + BaseDllName = "" + + if self.config["name"] and self.config["name"] not in BaseDllName: + return None + + try: + FullDllName = mod.FullDllName.get_string() + except exceptions.InvalidAddressException: + FullDllName = "" + + file_output = "Disabled" + if self.config["dump"]: + file_output = self.dump_module(session_layers, pe_table_name, mod) + + return ( + format_hints.Hex(mod.vol.offset), + format_hints.Hex(mod.DllBase), + format_hints.Hex(mod.SizeOfImage), + BaseDllName, + FullDllName, + file_output, + ) + def _generator(self): kernel = self.context.modules[self.config["kernel"]] + pe_table_name = intermed.IntermediateSymbolTable.create( self.context, self.config_path, "windows", "pe", class_types=pe.class_types ) - for mod in self.list_modules( + session_layers = list( + self.get_session_layers( + self.context, kernel.layer_name, kernel.symbol_table_name + ) + ) + + for mod in self._enumeration_method( self.context, kernel.layer_name, kernel.symbol_table_name ): - try: - BaseDllName = mod.BaseDllName.get_string() - except exceptions.InvalidAddressException: - BaseDllName = "" - - try: - FullDllName = mod.FullDllName.get_string() - except exceptions.InvalidAddressException: - FullDllName = "" - - if self.config["name"] and self.config["name"] not in BaseDllName: - continue - - file_output = "Disabled" - if self.config["dump"]: - file_handle = dlllist.DllList.dump_pe( - self.context, pe_table_name, mod, self.open - ) - file_output = "Error outputting file" - if file_handle: - file_handle.close() - file_output = file_handle.preferred_filename - - yield ( - 0, - ( - format_hints.Hex(mod.vol.offset), - format_hints.Hex(mod.DllBase), - format_hints.Hex(mod.SizeOfImage), - BaseDllName, - FullDllName, - file_output, - ), - ) + record = self.process_module(session_layers, pe_table_name, mod) + if record: + yield (0, record) @classmethod def get_session_layers( From 8d60e9160f231460c20bde89488ceecc03814aec Mon Sep 17 00:00:00 2001 From: Brandon Barnacle Date: Mon, 8 Jul 2024 09:34:30 -0400 Subject: [PATCH 220/979] PR comment changes. --- .../framework/plugins/windows/modscan.py | 9 +- .../framework/plugins/windows/modules.py | 89 ++++++++++--------- 2 files changed, 48 insertions(+), 50 deletions(-) diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index 3a786d2ca..98546bc9c 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -2,13 +2,10 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import Iterable, List, Generator +from typing import Iterable -from volatility3.framework import renderers, interfaces, exceptions, constants +from volatility3.framework import interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints -from volatility3.framework.symbols import intermed -from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import poolscanner, dlllist, pslist, modules vollog = logging.getLogger(__name__) @@ -36,7 +33,7 @@ class ModScan(modules.Modules): name="poolscanner", component=poolscanner.PoolScanner, version=(1, 0, 0) ), requirements.VersionRequirement( - name="modules", component=modules.Modules, version=(1, 1, 0) + name="modules", component=modules.Modules, version=(2, 0, 0) ), requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 8aea3dd45..79eea1cd7 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -4,9 +4,7 @@ import logging from typing import List, Iterable, Generator -from volatility3.framework import constants -from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers +from volatility3.framework import exceptions, interfaces, constants, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed @@ -20,7 +18,7 @@ class Modules(interfaces.plugins.PluginInterface): """Lists the loaded kernel modules.""" _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -78,55 +76,58 @@ class Modules(interfaces.plugins.PluginInterface): return file_output - def process_module(self, session_layers, pe_table_name, mod): - if self.config["base"] and self.config["base"] != mod.DllBase: - return None - - try: - BaseDllName = mod.BaseDllName.get_string() - except exceptions.InvalidAddressException: - BaseDllName = "" - - if self.config["name"] and self.config["name"] not in BaseDllName: - return None - - try: - FullDllName = mod.FullDllName.get_string() - except exceptions.InvalidAddressException: - FullDllName = "" - - file_output = "Disabled" - if self.config["dump"]: - file_output = self.dump_module(session_layers, pe_table_name, mod) - - return ( - format_hints.Hex(mod.vol.offset), - format_hints.Hex(mod.DllBase), - format_hints.Hex(mod.SizeOfImage), - BaseDllName, - FullDllName, - file_output, - ) - def _generator(self): kernel = self.context.modules[self.config["kernel"]] - pe_table_name = intermed.IntermediateSymbolTable.create( - self.context, self.config_path, "windows", "pe", class_types=pe.class_types - ) + pe_table_name = None + session_layers = None - session_layers = list( - self.get_session_layers( - self.context, kernel.layer_name, kernel.symbol_table_name + if self.config["dump"]: + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "windows", + "pe", + class_types=pe.class_types, + ) + + session_layers = list( + self.get_session_layers( + self.context, kernel.layer_name, kernel.symbol_table_name + ) ) - ) for mod in self._enumeration_method( self.context, kernel.layer_name, kernel.symbol_table_name ): - record = self.process_module(session_layers, pe_table_name, mod) - if record: - yield (0, record) + if self.config["base"] and self.config["base"] != mod.DllBase: + continue + + try: + BaseDllName = mod.BaseDllName.get_string() + except exceptions.InvalidAddressException: + BaseDllName = interfaces.renderers.BaseAbsentValue() + + if self.config["name"] and self.config["name"] not in BaseDllName: + continue + + try: + FullDllName = mod.FullDllName.get_string() + except exceptions.InvalidAddressException: + FullDllName = interfaces.renderers.BaseAbsentValue() + + file_output = "Disabled" + if self.config["dump"]: + file_output = self.dump_module(session_layers, pe_table_name, mod) + + yield 0, ( + format_hints.Hex(mod.vol.offset), + format_hints.Hex(mod.DllBase), + format_hints.Hex(mod.SizeOfImage), + BaseDllName, + FullDllName, + file_output, + ) @classmethod def get_session_layers( From c3cf319f7e44161ef6fd6537787de70daf71d126 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 12 Jul 2024 09:13:08 +0100 Subject: [PATCH 221/979] Windows: remove size from filescan output as it is not the file size but the object size --- volatility3/framework/plugins/windows/filescan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/filescan.py b/volatility3/framework/plugins/windows/filescan.py index 0f68f39d4..e21b5f518 100644 --- a/volatility3/framework/plugins/windows/filescan.py +++ b/volatility3/framework/plugins/windows/filescan.py @@ -13,7 +13,7 @@ from volatility3.plugins.windows import poolscanner class FileScan(interfaces.plugins.PluginInterface): """Scans for file objects present in a particular windows memory image.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 0, 1) @classmethod def get_requirements(cls): @@ -67,10 +67,10 @@ class FileScan(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: continue - yield (0, (format_hints.Hex(fileobj.vol.offset), file_name, fileobj.Size)) + yield (0, (format_hints.Hex(fileobj.vol.offset), file_name)) def run(self): return renderers.TreeGrid( - [("Offset", format_hints.Hex), ("Name", str), ("Size", int)], + [("Offset", format_hints.Hex), ("Name", str)], self._generator(), ) From e5a530f08acc198939e3fa90616fb12062da34e7 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 12 Jul 2024 10:20:44 +0100 Subject: [PATCH 222/979] Windows: revert required framework version for filescan plugin --- volatility3/framework/plugins/windows/filescan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/filescan.py b/volatility3/framework/plugins/windows/filescan.py index e21b5f518..34c0c60d3 100644 --- a/volatility3/framework/plugins/windows/filescan.py +++ b/volatility3/framework/plugins/windows/filescan.py @@ -13,7 +13,7 @@ from volatility3.plugins.windows import poolscanner class FileScan(interfaces.plugins.PluginInterface): """Scans for file objects present in a particular windows memory image.""" - _required_framework_version = (2, 0, 1) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): From d4585be7cf4b6fbce85a6fa7a5b448b604036f86 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 14 Jul 2024 15:21:08 +0100 Subject: [PATCH 223/979] Linux: Update vmayarascan to scan complete VMA blocks --- .../framework/plugins/linux/vmayarascan.py | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index eda0d7dca..3edbe7784 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -33,9 +33,6 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) ), - requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) - ), requirements.ModuleRequirement( name="kernel", description="Linux kernel", @@ -69,19 +66,29 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): # get the proc_layer object from the context proc_layer = self.context.layers[proc_layer_name] - # scan the process layer with the yarascanner - for offset, rule_name, name, value in proc_layer.scan( - context=self.context, - scanner=yarascan.YaraScanner(rules=rules), - sections=self.get_vma_maps(task), - ): - yield 0, ( - format_hints.Hex(offset), - task.tgid, - rule_name, - name, - value, - ) + for start, end in self.get_vma_maps(task): + for match in rules.match( + data=proc_layer.read(start, end - start, True) + ): + if yarascan.YaraScan.yara_returns_instances(): + for match_string in match.strings: + for instance in match_string.instances: + yield 0, ( + format_hints.Hex(instance.offset + start), + task.UniqueProcessId, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield 0, ( + format_hints.Hex(offset + start), + task.tgid, + match.rule, + name, + value, + ) @staticmethod def get_vma_maps( From 61e3e894eea28a8e0368c43f0fecb1b5031e0d2b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 14 Jul 2024 19:04:06 +0100 Subject: [PATCH 224/979] Core: Bump the version and minium python to support pip --- README.md | 2 +- pyproject.toml | 2 +- volatility3/framework/__init__.py | 2 +- volatility3/framework/constants/_version.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d303d2dff..886790df8 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.7.3 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/pyproject.toml b/pyproject.toml index 98c562a90..207f762cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, ] -requires-python = ">=3.7.0" +requires-python = ">=3.7.3" license = { text = "VSL" } dynamic = ["dependencies", "optional-dependencies", "version"] diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index feb97810b..74db773cf 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 7, 0) +required_python_version = (3, 7, 3) if ( sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 37c115431..21c339a6e 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 7 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From ac9236cb436d40a156f94b7aeab63afcc545663d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 15 Jul 2024 19:59:41 +1000 Subject: [PATCH 225/979] linux.netfilter.Netfilter: Add LinuxUtilities version requirement check on AbstractNetfilter --- .../framework/plugins/linux/netfilter.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 60a71f798..e0fbcbd70 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -11,6 +11,7 @@ from volatility3.framework import ( constants, interfaces, renderers, + exceptions, ) from volatility3.framework.renderers import format_hints from volatility3.framework.configuration import requirements @@ -80,6 +81,16 @@ class AbstractNetfilter(ABC): self.list_head_size = self.vmlinux.get_type("list_head").size modules = lsmod.Lsmod.list_modules(context, kernel_module_name) + + linuxutils_required_version = Netfilter._required_linuxutils_version + linuxutils_current_version = linux.LinuxUtilities._version + if not requirements.VersionRequirement.matches_required( + linuxutils_required_version, linuxutils_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" + ) + self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( context, kernel_module_name, modules ) @@ -658,6 +669,8 @@ class Netfilter(interfaces.plugins.PluginInterface): _version = (1, 0, 0) + _required_linuxutils_version = (2, 1, 0) + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ @@ -670,7 +683,9 @@ class Netfilter(interfaces.plugins.PluginInterface): name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), requirements.VersionRequirement( - name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) + name="linuxutils", + component=linux.LinuxUtilities, + version=cls._required_linuxutils_version, ), ] From a6d8ba8bfaeeceebd564e35e84ae0e288f69871e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 15 Jul 2024 20:22:40 +1000 Subject: [PATCH 226/979] windows eprocess validation: Make the creation and exit year checks dynamic --- volatility3/framework/symbols/windows/extensions/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ae855d6c6..93c19599e 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -614,12 +614,13 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): # A process must have a creation time return False - if not (1998 < ctime.year < 2030): + current_year = datetime.datetime.now().year + if not (1998 < ctime.year < current_year + 10): return False etime = self.get_exit_time() if isinstance(etime, datetime.datetime): - if not (1998 < etime.year < 2030): + if not (1998 < etime.year < current_year + 10): return False # Exit time, if available, must be after the creation time From cb6929163b31726d811f7e825bcfad8690170b9c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 15 Jul 2024 18:14:50 +0100 Subject: [PATCH 227/979] Windows: Fix vadyarascan sanity check and bad documentation --- volatility3/framework/plugins/windows/vadyarascan.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 4b00fee57..219faf028 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -56,7 +56,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - sanity_check = 0x1000 * 0x1000 * 0x1000 + sanity_check = 0x2000 * 0x1000 * 0x1000 for task in pslist.PsList.list_processes( context=self.context, @@ -66,15 +66,14 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ): layer_name = task.add_process_layer() layer = self.context.layers[layer_name] - for start, end in self.get_vad_maps(task): - size = end - start + for start, size in self.get_vad_maps(task): if size > sanity_check: vollog.warn( f"VAD at 0x{start:x} over sanity-check size, not scanning" ) continue - for match in rules.match(data=layer.read(start, end - start, True)): + for match in rules.match(data=layer.read(start, size, True)): if yarascan.YaraScan.yara_returns_instances(): for match_string in match.strings: for instance in match_string.instances: @@ -106,7 +105,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): task: The EPROCESS object of which to traverse the vad tree Returns: - An iterable of tuples containing start and end addresses for each descriptor + An iterable of tuples containing start and size for each descriptor """ vad_root = task.get_vad_root() for vad in vad_root.traverse(): From 55fe4ba47aece0882a0b5c690710cba1fa438989 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 15 Jul 2024 15:20:10 -0500 Subject: [PATCH 228/979] #118 - MR feedback --- .../framework/plugins/windows/kpcrs.py | 106 ++++++++++++++++++ .../framework/plugins/windows/timers.py | 86 +++----------- .../symbols/windows/extensions/__init__.py | 6 +- 3 files changed, 125 insertions(+), 73 deletions(-) create mode 100644 volatility3/framework/plugins/windows/kpcrs.py diff --git a/volatility3/framework/plugins/windows/kpcrs.py b/volatility3/framework/plugins/windows/kpcrs.py new file mode 100644 index 000000000..558ea844c --- /dev/null +++ b/volatility3/framework/plugins/windows/kpcrs.py @@ -0,0 +1,106 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from typing import Iterator, List, Tuple + +from volatility3.framework import ( + renderers, + interfaces, + constants, +) +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints + +vollog = logging.getLogger(__name__) + + +class KPCRs(interfaces.plugins.PluginInterface): + """Print KPCR structure for each processor""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + ] + + @classmethod + def list_kpcrs( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + layer_name: str, + symbol_table: str, + ) -> interfaces.objects.ObjectInterface: + """Returns the KPCR structure for each processor + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + kernel_module_name: The name of the kernel module on which to operate + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + + Returns: + The _KPCR structure for each processor + """ + + kernel = context.modules[kernel_module_name] + cpu_count_offset = kernel.get_symbol("KeNumberProcessors").address + cpu_count = kernel.object( + object_type="unsigned int", layer_name=layer_name, offset=cpu_count_offset + ) + processor_block = kernel.object( + object_type="pointer", + layer_name=layer_name, + offset=kernel.get_symbol("KiProcessorBlock").address, + ) + processor_pointers = utility.array_of_pointers( + context=context, + array=processor_block, + count=cpu_count, + subtype=symbol_table + constants.BANG + "_KPRCB", + ) + for pointer in processor_pointers: + kprcb = pointer.dereference() + reloff = kernel.get_type("_KPCR").relative_child_offset("Prcb") + kpcr = context.object( + symbol_table + constants.BANG + "_KPCR", + offset=kprcb.vol.offset - reloff, + layer_name=layer_name, + ) + yield kpcr + + def _generator(self) -> Iterator[Tuple]: + kernel = self.context.modules[self.config["kernel"]] + layer_name = kernel.layer_name + symbol_table = kernel.symbol_table_name + + for kpcr in self.list_kpcrs( + self.context, self.config["kernel"], layer_name, symbol_table + ): + yield ( + 0, + ( + format_hints.Hex(kpcr.vol.offset), + format_hints.Hex(kpcr.CurrentPrcb), + ), + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("PRCB Offset", format_hints.Hex), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index ec8bc17e9..d49c28784 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -7,17 +7,15 @@ import logging from typing import Iterator, List, Tuple, Iterable from volatility3.framework import ( - layers, renderers, interfaces, constants, symbols, ) from volatility3.framework.configuration import requirements -from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols.windows import versions -from volatility3.plugins.windows import ssdt +from volatility3.plugins.windows import ssdt, kpcrs vollog = logging.getLogger(__name__) @@ -39,73 +37,16 @@ class Timers(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) ), + requirements.PluginRequirement( + name="kpcrs", plugin=kpcrs.KPCRs, version=(1, 0, 0) + ), ] - @classmethod - def get_kernel_module( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - ): - """Returns the kernel module based on the layer and symbol_table""" - virtual_layer = context.layers[layer_name] - if not isinstance(virtual_layer, layers.intel.Intel): - raise TypeError("Virtual Layer is not an intel layer") - - kvo = virtual_layer.config["kernel_virtual_offset"] - - ntkrnlmp = context.module(symbol_table, layer_name=layer_name, offset=kvo) - return ntkrnlmp - - @classmethod - def get_kpcrs( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, - ) -> interfaces.objects.ObjectInterface: - """Returns the KPCR structure for each processor - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - symbol_table: The name of an existing symbol table containing the kernel symbols - config_path: The configuration path within the context of the symbol table to create - - Returns: - The _KPCR structure for each processor - """ - - ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table) - cpu_count_offset = ntkrnlmp.get_symbol("KeNumberProcessors").address - cpu_count = ntkrnlmp.object( - object_type="unsigned int", layer_name=layer_name, offset=cpu_count_offset - ) - processor_block = ntkrnlmp.object( - object_type="pointer", - layer_name=layer_name, - offset=ntkrnlmp.get_symbol("KiProcessorBlock").address, - ) - processor_pointers = utility.array_of_pointers( - context=context, - array=processor_block, - count=cpu_count, - subtype=symbol_table + constants.BANG + "_KPRCB", - ) - for pointer in processor_pointers: - kprcb = pointer.dereference() - reloff = ntkrnlmp.get_type("_KPCR").relative_child_offset("Prcb") - kpcr = context.object( - symbol_table + constants.BANG + "_KPCR", - offset=kprcb.vol.offset - reloff, - layer_name=layer_name, - ) - yield kpcr - @classmethod def list_timers( cls, context: interfaces.context.ContextInterface, + kernel_module_name: str, layer_name: str, symbol_table: str, ) -> Iterable[Tuple[str, int, str]]: @@ -113,21 +54,24 @@ class Timers(interfaces.plugins.PluginInterface): Args: context: The context to retrieve required elements (layers, symbol tables) from + kernel_module_name: The name of the kernel module on which to operate layer_name: The name of the layer on which to operate symbol_table: The name of the table containing the kernel symbols Yields: A _KTIMER entry """ - ntkrnlmp = cls.get_kernel_module(context, layer_name, symbol_table) + kernel = context.modules[kernel_module_name] if versions.is_windows_7( context=context, symbol_table=symbol_table ) or versions.is_windows_8_or_later(context=context, symbol_table=symbol_table): # Starting with Windows 7, there is no more KiTimerTableListHead. The list is # at _KPCR.PrcbData.TimerTable.TimerEntries # See http://pastebin.com/FiRsGW3f - for kpcr in cls.get_kpcrs(context, layer_name, symbol_table): + for kpcr in kpcrs.KPCRs.list_kpcrs( + context, kernel_module_name, layer_name, symbol_table + ): if hasattr(kpcr.Prcb.TimerTable, "TableState"): for timer_entries in kpcr.Prcb.TimerTable.TimerEntries: for timer_entry in timer_entries: @@ -160,10 +104,10 @@ class Timers(interfaces.plugins.PluginInterface): # is an array of 256 _LIST_ENTRY for _KTIMERs. array_size = 256 - timer_table_list_head = ntkrnlmp.object( + timer_table_list_head = kernel.object( object_type="array", - offset=ntkrnlmp.get_symbol("KiTimerTableListHead").address, - subtype=ntkrnlmp.get_type("_LIST_ENTRY"), + offset=kernel.get_symbol("KiTimerTableListHead").address, + subtype=kernel.get_type("_LIST_ENTRY"), count=array_size, ) for table in timer_table_list_head: @@ -185,7 +129,9 @@ class Timers(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name ) - for timer in self.list_timers(self.context, layer_name, symbol_table): + for timer in self.list_timers( + self.context, self.config["kernel"], layer_name, symbol_table + ): if not timer.valid_type(): continue try: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index b9eabcda6..86ea2febb 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1011,12 +1011,12 @@ class KTIMER(objects.StructType): def get_raw_dpc(self): """Returns the encoded DPC since it may not look like a pointer after encoding""" symbol_table_name = self.get_symbol_table_name() - ulonglong_type = self._context.symbol_space.get_type( - symbol_table_name + constants.BANG + "unsigned long long" + pointer_type = self._context.symbol_space.get_type( + symbol_table_name + constants.BANG + "pointer" ) return self._context.object( - object_type=ulonglong_type, + object_type=pointer_type, layer_name=self.vol.layer_name, offset=self.Dpc.vol.offset, ) From 7e3615335c44f79b2b39e9f669a33df6f3edf2fc Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 15 Jul 2024 15:40:22 -0500 Subject: [PATCH 229/979] #1175 - config_path change --- volatility3/framework/plugins/windows/unloadedmodules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/unloadedmodules.py b/volatility3/framework/plugins/windows/unloadedmodules.py index 1fd4914ac..0d88e96ff 100644 --- a/volatility3/framework/plugins/windows/unloadedmodules.py +++ b/volatility3/framework/plugins/windows/unloadedmodules.py @@ -10,6 +10,7 @@ from volatility3.framework import constants from volatility3.framework import interfaces, symbols from volatility3.framework import renderers from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import configuration from volatility3.framework.renderers import format_hints, conversion from volatility3.framework.symbols import intermed from volatility3.plugins import timeliner @@ -60,7 +61,7 @@ class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInt return intermed.IntermediateSymbolTable.create( context, - config_path, + configuration.path_join(config_path, "unloadedmodules"), "windows", symbol_filename, native_types=native_types, From 99cf48597abbc1126ef06731ed7836c04614514d Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 15 Jul 2024 15:46:27 -0500 Subject: [PATCH 230/979] #118 - use canonicalize for offset --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 86ea2febb..d5c3d3f96 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1052,7 +1052,7 @@ class KTIMER(objects.StructType): low_byte = (wait_never) & 0xFF entry = utility.rol(self.get_raw_dpc() ^ wait_never, low_byte) - swap_xor = self.vol.offset | 0xFFFF000000000000 + swap_xor = self._context.layers[self.vol.native_layer_name].canonicalize(self.vol.offset) entry = utility.bswap_64(entry ^ swap_xor) dpc = entry ^ wait_always From cf6c7fb1f40c4bfa1c60cc050cfd346014a854ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 16 Jul 2024 09:38:49 +1000 Subject: [PATCH 231/979] linux.netfilter.Netfilter: Add lsmod version requirement check on AbstractNetfilter --- volatility3/framework/plugins/linux/netfilter.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index e0fbcbd70..d392a8371 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -80,7 +80,14 @@ class AbstractNetfilter(ABC): self.ptr_size = self.vmlinux.get_type("pointer").size self.list_head_size = self.vmlinux.get_type("list_head").size - modules = lsmod.Lsmod.list_modules(context, kernel_module_name) + lsmod_required_version = Netfilter._required_lsmod_version + lsmod_current_version = lsmod.Lsmod._version + if not requirements.VersionRequirement.matches_required( + lsmod_required_version, lsmod_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.lsmod.Lsmod version not suitable: required {lsmod_required_version} found {lsmod_current_version}" + ) linuxutils_required_version = Netfilter._required_linuxutils_version linuxutils_current_version = linux.LinuxUtilities._version @@ -91,6 +98,7 @@ class AbstractNetfilter(ABC): f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" ) + modules = lsmod.Lsmod.list_modules(context, kernel_module_name) self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( context, kernel_module_name, modules ) @@ -670,6 +678,7 @@ class Netfilter(interfaces.plugins.PluginInterface): _version = (1, 0, 0) _required_linuxutils_version = (2, 1, 0) + _required_lsmod_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -680,7 +689,7 @@ class Netfilter(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + name="lsmod", plugin=lsmod.Lsmod, version=cls._required_lsmod_version ), requirements.VersionRequirement( name="linuxutils", From 43e22d72bdaa2cbb6366b1911a6ff38ced83394f Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Tue, 16 Jul 2024 09:23:06 -0500 Subject: [PATCH 232/979] #118 - black formatting --- volatility3/framework/symbols/windows/extensions/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d5c3d3f96..b333755f7 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1052,7 +1052,9 @@ class KTIMER(objects.StructType): low_byte = (wait_never) & 0xFF entry = utility.rol(self.get_raw_dpc() ^ wait_never, low_byte) - swap_xor = self._context.layers[self.vol.native_layer_name].canonicalize(self.vol.offset) + swap_xor = self._context.layers[self.vol.native_layer_name].canonicalize( + self.vol.offset + ) entry = utility.bswap_64(entry ^ swap_xor) dpc = entry ^ wait_always From be5423f786827ee4056b2c638b2d1214a419c47c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jul 2024 17:08:15 +0100 Subject: [PATCH 233/979] Windows: Fix up broken imports in info plugin --- volatility3/framework/plugins/windows/info.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/info.py b/volatility3/framework/plugins/windows/info.py index 100a677c2..137d29c22 100644 --- a/volatility3/framework/plugins/windows/info.py +++ b/volatility3/framework/plugins/windows/info.py @@ -10,7 +10,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.renderers import TreeGrid from volatility3.framework.symbols import intermed -from volatility3.framework.symbols.windows import extensions +from volatility3.framework.symbols.windows.extensions import kdbg, pe class Info(plugins.PluginInterface): @@ -94,16 +94,16 @@ class Info(plugins.PluginInterface): "windows", "kdbg", native_types=native_types, - class_types=extensions.kdbg.class_types, + class_types=kdbg.class_types, ) - kdbg = context.object( + kdbg_obj = context.object( kdbg_table_name + constants.BANG + "_KDDEBUGGER_DATA64", offset=ntkrnlmp.offset + kdbg_offset, layer_name=layer_name, ) - return kdbg + return kdbg_obj @classmethod def get_kuser_structure( @@ -173,7 +173,7 @@ class Info(plugins.PluginInterface): interfaces.configuration.path_join(config_path, "pe"), "windows", "pe", - class_types=extensions.pe.class_types, + class_types=pe.class_types, ) dos_header = context.object( From a5cf635fd8b81c56f14ee490bcf63e4e5aafd32d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jul 2024 17:31:44 +0100 Subject: [PATCH 234/979] Windows: Fix the vadyarascan sanity check --- volatility3/framework/plugins/windows/vadyarascan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 219faf028..dc318dd93 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -56,7 +56,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - sanity_check = 0x2000 * 0x1000 * 0x1000 + sanity_check = 1024 * 1024 * 1024 # 1 GB for task in pslist.PsList.list_processes( context=self.context, From a684e284ccca7675d4e3cd07c39679ecba703627 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 5 Jul 2024 15:45:46 -0500 Subject: [PATCH 235/979] Windows: Adds shimcache symbol files + extensions --- .../framework/symbols/windows/__init__.py | 1 + .../symbols/windows/extensions/__init__.py | 49 +- .../symbols/windows/extensions/shimcache.py | 278 ++++++++++ .../windows/shimcache/shimcache-2003-x64.json | 327 ++++++++++++ .../windows/shimcache/shimcache-2003-x86.json | 334 ++++++++++++ .../shimcache/shimcache-vista-x64.json | 334 ++++++++++++ .../shimcache/shimcache-vista-x86.json | 334 ++++++++++++ .../shimcache/shimcache-win10-x64.json | 371 ++++++++++++++ .../shimcache/shimcache-win10-x86.json | 371 ++++++++++++++ .../windows/shimcache/shimcache-win7-x64.json | 348 +++++++++++++ .../windows/shimcache/shimcache-win7-x86.json | 348 +++++++++++++ .../windows/shimcache/shimcache-win8-x64.json | 392 ++++++++++++++ .../windows/shimcache/shimcache-win8-x86.json | 386 ++++++++++++++ .../shimcache/shimcache-xp-sp2-x86.json | 485 ++++++++++++++++++ .../shimcache/shimcache-xp-sp3-x86.json | 485 ++++++++++++++++++ .../framework/symbols/windows/versions.py | 27 + 16 files changed, 4863 insertions(+), 7 deletions(-) create mode 100644 volatility3/framework/symbols/windows/extensions/shimcache.py create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-2003-x64.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-2003-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-vista-x64.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-vista-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win10-x64.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win10-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win7-x64.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win7-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win8-x64.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-win8-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp2-x86.json create mode 100644 volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp3-x86.json diff --git a/volatility3/framework/symbols/windows/__init__.py b/volatility3/framework/symbols/windows/__init__.py index abf9f6da3..e43a2486b 100755 --- a/volatility3/framework/symbols/windows/__init__.py +++ b/volatility3/framework/symbols/windows/__init__.py @@ -17,6 +17,7 @@ class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("_KTHREAD", extensions.KTHREAD) self.set_type_class("_LIST_ENTRY", extensions.LIST_ENTRY) self.set_type_class("_EPROCESS", extensions.EPROCESS) + self.set_type_class("_ERESOURCE", extensions.ERESOURCE) self.set_type_class("_UNICODE_STRING", extensions.UNICODE_STRING) self.set_type_class("_EX_FAST_REF", extensions.EX_FAST_REF) self.set_type_class("_TOKEN", extensions.TOKEN) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 93c19599e..39b50e180 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -306,16 +306,19 @@ class MMVAD_SHORT(objects.StructType): raise AttributeError("Unable to find the private memory member") + @property + def Protection(self): + if self.has_member("u"): + return self.u.VadFlags.Protection + elif self.has_member("Core"): + return self.Core.u.VadFlags.Protection + else: + return None + def get_protection(self, protect_values, winnt_protections): """Get the VAD's protection constants as a string.""" - protect = None - - if self.has_member("u"): - protect = self.u.VadFlags.Protection - - elif self.has_member("Core"): - protect = self.Core.u.VadFlags.Protection + protect = self.Protection try: value = protect_values[protect] @@ -593,6 +596,38 @@ class UNICODE_STRING(objects.StructType): String = property(get_string) +class ERESOURCE(objects.StructType): + def is_valid(self) -> bool: + vollog.debug(f"Checking ERESOURCE Validity: {hex(self.vol.offset)}") + + if not self._context.layers[self.vol.layer_name].is_valid(self.vol.offset): + return False + + sym_table = self.get_symbol_table_name() + + waiters_valid = self.SharedWaiters == 0 or self._context.layers[ + self.vol.layer_name + ].is_valid( + self.SharedWaiters.vol.offset, + self._context.symbol_space.get_type( + sym_table + constants.BANG + "_KSEMAPHORE" + ).size, + ) + + try: + return ( + waiters_valid + and self.SystemResourcesList.Flink is not None + and self.SystemResourcesList.Blink is not None + and self.SystemResourcesList.Flink != self.SystemResourcesList.Blink + and self.SystemResourcesList.Flink.Blink == self.vol.offset + and self.SystemResourcesList.Blink.Flink == self.vol.offset + and self.NumberOfSharedWaiters == 0 + ) + except exceptions.InvalidAddressException: + return False + + class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): """A class for executive kernel processes objects.""" diff --git a/volatility3/framework/symbols/windows/extensions/shimcache.py b/volatility3/framework/symbols/windows/extensions/shimcache.py new file mode 100644 index 000000000..b84a7df6f --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/shimcache.py @@ -0,0 +1,278 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import struct +from datetime import datetime +from typing import Dict, Optional, Tuple, Union + +from volatility3.framework import constants, exceptions, interfaces, objects, renderers +from volatility3.framework.symbols.windows.extensions import conversion + +vollog = logging.getLogger(__name__) + + +class SHIM_CACHE_ENTRY(objects.StructType): + """Class for abstracting variations in the shimcache LRU list entry structure""" + + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + size: int, + members: Dict[str, Tuple[int, interfaces.objects.Template]], + ) -> None: + super().__init__(context, type_name, object_info, size, members) + self._exec_flag = None + self._file_path = None + self._file_size = None + self._last_modified = None + self._last_updated = None + + @property + def exec_flag(self) -> Union[bool, interfaces.renderers.BaseAbsentValue]: + """Checks if InsertFlags fields has been bitwise OR'd with a value of 2. + This behavior was observed when processes are created by CSRSS.""" + if self._exec_flag is not None: + return self._exec_flag + + if hasattr(self, "ListEntryDetail") and hasattr( + self.ListEntryDetail, "InsertFlags" + ): + self._exec_flag = self.ListEntryDetail.InsertFlags & 0x2 == 2 + + elif hasattr(self, "InsertFlags"): + self._exec_flag = self.InsertFlags & 0x2 == 2 + + elif hasattr(self, "ListEntryDetail") and hasattr( + self.ListEntryDetail, "BlobBuffer" + ): + blob_offset = self.ListEntryDetail.BlobBuffer + blob_size = self.ListEntryDetail.BlobSize + + if not self._context.layers[self.vol.native_layer_name].is_valid( + blob_offset, blob_size + ): + self._exec_flag = renderers.UnparsableValue() + + raw_flag = self._context.layers[self.vol.native_layer_name].read( + blob_offset, blob_size + ) + if not raw_flag: + self._exec_flag = renderers.UnparsableValue() + + try: + self._exec_flag = bool(struct.unpack(" Union[int, interfaces.renderers.BaseAbsentValue]: + if self._file_size is not None: + return self._file_size + try: + self._file_size = self.FileSize + if self._file_size < 0: + self._file_size = 0 + + except AttributeError: + self._file_size = renderers.NotApplicableValue() + except exceptions.InvalidAddressException: + self._file_size = renderers.UnreadableValue() + + return self._file_size + + @property + def last_modified(self) -> Union[datetime, interfaces.renderers.BaseAbsentValue]: + if self._last_modified is not None: + return self._last_modified + try: + self._last_modified = conversion.wintime_to_datetime( + self.ListEntryDetail.LastModified.QuadPart + ) + except AttributeError: + self._last_modified = conversion.wintime_to_datetime( + self.LastModified.QuadPart + ) + except exceptions.InvalidAddressException: + self._last_modified = renderers.UnreadableValue() + + return self._last_modified + + @property + def last_update(self) -> Union[datetime, interfaces.renderers.BaseAbsentValue]: + if self._last_updated is not None: + return self._last_updated + + try: + self._last_updated = conversion.wintime_to_datetime( + self.LastUpdate.QuadPart + ) + except AttributeError: + self._last_updated = renderers.NotApplicableValue() + + return self._last_updated + + @property + def file_path(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: + if self._file_path is not None: + return self._file_path + + if not hasattr(self.Path, "Buffer"): + return self.Path.cast( + "string", max_length=self.Path.vol.count, encoding="utf-16le" + ) + + try: + file_path_raw = ( + self._context.layers[self.vol.native_layer_name].read( + self.Path.Buffer, self.Path.Length + ) + or b"" + ) + self._file_path = file_path_raw.decode("utf-16", errors="replace") + except exceptions.InvalidAddressException: + self._file_path = renderers.UnreadableValue() + + return self._file_path + + def is_valid(self) -> bool: + """Shim cache validation is limited to ensuring that a subset of the + pointers in the LIST_ENTRY field are valid (similar to validation of + ERESOURCE)""" + + # shim entries on Windows XP do not have list entry attributes; in this case, + # perform a different set of validations + try: + if not hasattr(self, "ListEntry"): + return bool(self.last_modified and self.last_update and self.file_size) + + # on some platforms ListEntry.Blink is null, so this cannot be validated + if ( + self.ListEntry.Flink != 0 + and ( + self.ListEntry.Blink.dereference() + != self.ListEntry.Flink.dereference() + ) + and ( + self.ListEntry.Flink.Blink + == self.ListEntry.Flink.Blink.dereference().vol.offset + ) + ): + + return True + else: + return False + except exceptions.InvalidAddressException: + return False + + +class SHIM_CACHE_HANDLE(objects.StructType): + def __init__( + self, + context: interfaces.context.ContextInterface, + type_name: str, + object_info: interfaces.objects.ObjectInformation, + size: int, + members: Dict[str, Tuple[int, interfaces.objects.Template]], + ) -> None: + super().__init__(context, type_name, object_info, size, members) + + @property + def head(self) -> Optional[SHIM_CACHE_ENTRY]: + try: + if not self.eresource.is_valid(): + return None + except exceptions.InvalidAddressException: + return None + + rtl_avl_table = self._context.object( + self.get_symbol_table_name() + constants.BANG + "_RTL_AVL_TABLE", + self.vol.layer_name, + self.rtl_avl_table, + self.vol.native_layer_name, + ) + + if not self._context.layers[self.vol.layer_name].is_valid( + self.rtl_avl_table.vol.offset + ): + return None + + offset_head = rtl_avl_table.vol.offset + rtl_avl_table.vol.size + + head = self._context.object( + self.get_symbol_table_name() + constants.BANG + "SHIM_CACHE_ENTRY", + self.vol.layer_name, + offset_head, + ) + + if not head.is_valid(): + return None + + return head + + def is_valid(self, avl_section_start: int, avl_section_end: int) -> bool: + if self.vol.offset == 0: + return False + + vollog.debug(f"Checking SHIM_CACHE_HANDLE validity @ {hex(self.vol.offset)}") + + if not ( + self._context.layers[self.vol.layer_name].is_valid(self.vol.offset) + and self.eresource.is_valid() + and self.rtl_avl_table.is_valid(avl_section_start, avl_section_end) + and self.head + ): + return False + + return self.head.is_valid() + + +class RTL_AVL_TABLE(objects.StructType): + def is_valid(self, page_start: int, page_end: int) -> bool: + try: + if self.BalancedRoot.Parent != self.BalancedRoot.vol.offset: + vollog.debug( + f"RTL_AVL_TABLE @ {self.vol.offset} Invalid: Failed BalancedRoot parent equality check" + ) + return False + + elif self.AllocateRoutine < page_start or self.AllocateRoutine > page_end: + vollog.debug( + f"RTL_AVL_TABLE @ {self.vol.offset} Invalid: Failed AllocateRoutine range check" + ) + return False + + elif self.CompareRoutine < page_start or self.CompareRoutine > page_end: + vollog.debug( + f"RTL_AVL_TABLE @ {self.vol.offset} Invalid: Failed CompareRoutine range check" + ) + return False + + elif ( + (self.AllocateRoutine.vol.offset == self.CompareRoutine.vol.offset) + or (self.AllocateRoutine.vol.offset == self.FreeRoutine.vol.offset) + or (self.CompareRoutine.vol.offset == self.FreeRoutine.vol.offset) + ): + vollog.debug( + f"RTL_AVL_TABLE @ {self.vol.offset} Invalid: Failed (Compare|Allocate|Free)Routine uniqueness check" + ) + return False + + return True + except exceptions.InvalidAddressException: + return False + + +class_types = { + "SHIM_CACHE_HANDLE": SHIM_CACHE_HANDLE, + "SHIM_CACHE_ENTRY": SHIM_CACHE_ENTRY, + "_RTL_AVL_TABLE": RTL_AVL_TABLE, +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x64.json b/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x64.json new file mode 100644 index 000000000..d1540f12f --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x64.json @@ -0,0 +1,327 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 16 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 24 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 25 + } + }, + "kind": "struct", + "size": 32 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 40 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 44 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 48 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 56 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 64 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 72 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 80 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 88 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 104 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 16 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 16, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 32, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "FileSize": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 48 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x86.json new file mode 100644 index 000000000..e4739c4a0 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-2003-x86.json @@ -0,0 +1,334 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 8, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 16, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "FileSize": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "Padding": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 36 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x64.json b/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x64.json new file mode 100644 index 000000000..0c5183a4a --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x64.json @@ -0,0 +1,334 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 16 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 24 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 25 + } + }, + "kind": "struct", + "size": 32 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 40 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 44 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 48 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 56 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 64 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 72 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 80 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 88 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 104 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 16 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 16, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 32, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 44, + "type": { + "kind": "base", + "name": "unsigned int" + } + } + }, + "kind": "struct", + "size": 48 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x86.json new file mode 100644 index 000000000..91290b1ba --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-vista-x86.json @@ -0,0 +1,334 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 8, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 16, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 28, + "type": { + "kind": "base", + "name": "unsigned int" + } + } + }, + "kind": "struct", + "size": 36 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x64.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x64.json new file mode 100644 index 000000000..fe9593af3 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x64.json @@ -0,0 +1,371 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 16 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 24 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 25 + } + }, + "kind": "struct", + "size": 32 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 40 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 44 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 48 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 56 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 64 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 72 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 80 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 88 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 104 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 16 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "u1": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "Path": { + "offset": 24, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "ListEntryDetail": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "SHIM_CACHE_ENTRY_DETAIL" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "SHIM_CACHE_ENTRY_DETAIL": { + "fields": { + "u1": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "LastModified": { + "offset": 8, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "BlobSize": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "u2": { + "offset": 20, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "BlobBuffer": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + } + }, + "kind": "struct", + "size": 32 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x86.json new file mode 100644 index 000000000..26d493c37 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win10-x86.json @@ -0,0 +1,371 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 4 + } + }, + "kind": "struct", + "size": 8 + }, + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "u1": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "Path": { + "offset": 12, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "ListEntryDetail": { + "offset": 20, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "SHIM_CACHE_ENTRY_DETAIL" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "SHIM_CACHE_ENTRY_DETAIL": { + "fields": { + "u1": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "InsertFlags": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "LastModified": { + "offset": 8, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "BlobSize": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "BlobBuffer": { + "offset": 20, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 24 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x64.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x64.json new file mode 100644 index 000000000..eac5407ba --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x64.json @@ -0,0 +1,348 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 16 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 24 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 25 + } + }, + "kind": "struct", + "size": 32 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 40 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 44 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 48 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 56 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 64 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 72 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 80 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 88 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 104 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 16, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 32, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 44, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "BlobSize": { + "offset": 48, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "BlobBuffer": { + "offset": 56, + "type": { + "kind": "base", + "name": "unsigned long long" + } + } + }, + "kind": "struct", + "size": 64 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x86.json new file mode 100644 index 000000000..423f7e255 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win7-x86.json @@ -0,0 +1,348 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 4 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Path": { + "offset": 8, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "LastModified": { + "offset": 16, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 28, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "BlobSize": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "BlobBuffer": { + "offset": 36, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 40 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x64.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x64.json new file mode 100644 index 000000000..a40c8d680 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x64.json @@ -0,0 +1,392 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 16 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 24 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 25 + } + }, + "kind": "struct", + "size": 32 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 32 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 40 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 44 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 48 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 56 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 64 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 72 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 80 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 88 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 104 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 8 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "u1": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "Path": { + "offset": 24, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "u2": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "u3": { + "offset": 48, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ListEntryDetail": { + "offset": 56, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "SHIM_CACHE_ENTRY_DETAIL" + } + } + } + }, + "kind": "struct", + "size": 64 + }, + "SHIM_CACHE_ENTRY_DETAIL": { + "fields": { + "LastModified": { + "offset": 0, + "type": { + "kind": "struct", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "BlobSize": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "Padding": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "BlobBuffer": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long long" + } + } + }, + "kind": "struct", + "size": 40 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x86.json new file mode 100644 index 000000000..c3cee5feb --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-win8-x86.json @@ -0,0 +1,386 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 4 + } + }, + "kind": "struct", + "size": 8 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "ListEntry": { + "offset": 0, + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "u1": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "u2": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "Path": { + "offset": 16, + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + } + }, + "u3": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "ListEntryDetail": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "SHIM_CACHE_ENTRY_DETAIL" + } + } + } + }, + "kind": "struct", + "size": 36 + }, + "SHIM_CACHE_ENTRY_DETAIL": { + "fields": { + "LastModified": { + "offset": 0, + "type": { + "kind": "struct", + "name": "_LARGE_INTEGER" + } + }, + "InsertFlags": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ShimFlags": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "BlobSize": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "BlobBuffer": { + "offset": 20, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 24 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} + diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp2-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp2-x86.json new file mode 100644 index 000000000..6114e6c85 --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp2-x86.json @@ -0,0 +1,485 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 4 + } + }, + "kind": "struct", + "size": 8 + }, + "SHIM_CACHE_HEADER": { + "fields": { + "Magic": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 0 + }, + "u1": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 4 + }, + "NumEntries": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 8 + }, + "u2": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 400 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "Path": { + "type": { + "count": 520, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "LastModified": { + "type": { + "kind": "union", + "name": "LARGE_INTEGER" + }, + "offset": 4 + }, + "FileSize": { + "type": { + "kind": "base", + "name": "long long" + }, + "offset": 8 + }, + "LastUpdate": { + "type": { + "kind": "union", + "name": "LARGE_INTEGER" + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 552 + }, + "_SEGMENT": { + "fields": { + "ControlArea": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_CONTROL_AREA" + } + } + }, + "TotalNumberOfPtes": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "NonExtendedPtes": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "WritableUserReferences": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "SizeOfSegment": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "SegmentPteTemplate": { + "offset": 24, + "type": { + "kind": "struct", + "name": "nt_symbols!_MMPTE" + } + }, + "NumberOfCommittedPages": { + "offset": 28, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "ExtendInfo": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_MMEXTEND_INFO" + } + } + }, + "SystemImageBase": { + "offset": 36, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + }, + "BasedAddress": { + "offset": 40, + "type": { + "kind": "base", + "name": "long" + } + }, + "u1": { + "offset": 44, + "type": { + "kind": "base", + "name": "long" + } + }, + "u2": { + "offset": 48, + "type": { + "kind": "base", + "name": "long" + } + }, + "PrototypePte": { + "offset": 52, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_MMPTE" + } + } + }, + "ThePtes": { + "offset": 60, + "type": { + "kind": "array", + "count": 1, + "subtype": { + "kind": "base", + "name": "nt_symbols!_MMPTE" + } + } + } + }, + "kind": "struct", + "size": 64 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp3-x86.json b/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp3-x86.json new file mode 100644 index 000000000..a9b86d93c --- /dev/null +++ b/volatility3/framework/symbols/windows/shimcache/shimcache-xp-sp3-x86.json @@ -0,0 +1,485 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned long long": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "__unnamed_2": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 + }, + "_RTL_BALANCED_LINKS": { + "fields": { + "Parent": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 0 + }, + "LeftChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 4 + }, + "RightChild": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 8 + }, + "Balance": { + "type": { + "kind": "base", + "name": "unsigned char" + }, + "offset": 12 + }, + "Reserved": { + "type": { + "kind": "array", + "count": 3, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 16 + }, + "_RTL_AVL_TABLE": { + "fields": { + "BalancedRoot": { + "type": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + }, + "offset": 0 + }, + "OrderedPointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 16 + }, + "WhichOrderedElement": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 20 + }, + "NumberGenericTableElements": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 24 + }, + "DepthOfTree": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 28 + }, + "RestartKey": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_BALANCED_LINKS" + } + }, + "offset": 32 + }, + "DeleteCount": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 36 + }, + "CompareRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 40 + }, + "AllocateRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 44 + }, + "FreeRoutine": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 48 + }, + "TableContext": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 52 + } + }, + "kind": "struct", + "size": 56 + }, + "SHIM_CACHE_HEADER": { + "fields": { + "Magic": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 0 + }, + "u1": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 4 + }, + "NumEntries": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 8 + }, + "u2": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 400 + }, + "SHIM_CACHE_ENTRY": { + "fields": { + "Path": { + "type": { + "count": 520, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "LastModified": { + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + }, + "offset": 528 + }, + "FileSize": { + "type": { + "kind": "base", + "name": "long long" + }, + "offset": 536 + }, + "LastUpdate": { + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + }, + "offset": 544 + } + }, + "kind": "struct", + "size": 552 + }, + "SHIM_CACHE_HANDLE": { + "fields": { + "eresource": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!ERESOURCE" + } + }, + "offset": 0 + }, + "rtl_avl_table": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_RTL_AVL_TABLE" + } + }, + "offset": 4 + } + }, + "kind": "struct", + "size": 8 + }, + "_SEGMENT": { + "fields": { + "ControlArea": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_CONTROL_AREA" + } + } + }, + "TotalNumberOfPtes": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "NonExtendedPtes": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "WritableUserReferences": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "SizeOfSegment": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "SegmentPteTemplate": { + "offset": 24, + "type": { + "kind": "struct", + "name": "nt_symbols!_MMPTE" + } + }, + "NumberOfCommittedPages": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "ExtendInfo": { + "offset": 36, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_MMEXTEND_INFO" + } + } + }, + "SystemImageBase": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + }, + "BasedAddress": { + "offset": 44, + "type": { + "kind": "base", + "name": "long" + } + }, + "u1": { + "offset": 48, + "type": { + "kind": "base", + "name": "long" + } + }, + "u2": { + "offset": 52, + "type": { + "kind": "base", + "name": "long" + } + }, + "PrototypePte": { + "offset": 56, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_MMPTE" + } + } + }, + "ThePtes": { + "offset": 64, + "type": { + "kind": "array", + "count": 1, + "subtype": { + "kind": "base", + "name": "nt_symbols!_MMPTE" + } + } + } + }, + "kind": "struct", + "size": 72 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona by hand", + "datetime": "2024-07-05T18:28:00.000000+00:00" + }, + "format": "4.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/versions.py b/volatility3/framework/symbols/windows/versions.py index e1e74afc0..78e90a1d4 100644 --- a/volatility3/framework/symbols/windows/versions.py +++ b/volatility3/framework/symbols/windows/versions.py @@ -114,6 +114,24 @@ is_windows_xp = OsDistinguisher( ], ) +is_windows_xp_sp2 = OsDistinguisher( + version_check=lambda x: (5, 1) <= x < (5, 2), + fallback_checks=[ + ("KdCopyDataBlock", None, False), + ("_MMFREE_POOL_ENTRY", None, False), + ("_HANDLE_TABLE", "HandleCount", True), + ], +) + +is_windows_xp_sp3 = OsDistinguisher( + version_check=lambda x: (5, 1) <= x < (5, 2), + fallback_checks=[ + ("KdCopyDataBlock", None, False), + ("_MMFREE_POOL_ENTRY", None, True), + ("_HANDLE_TABLE", "HandleCount", True), + ], +) + is_xp_or_2003 = OsDistinguisher( version_check=lambda x: (5, 1) <= x < (6, 0), fallback_checks=[ @@ -122,6 +140,15 @@ is_xp_or_2003 = OsDistinguisher( ], ) +is_2003 = OsDistinguisher( + version_check=lambda x: (5, 2) <= x < (5, 3), + fallback_checks=[ + ("KdCopyDataBlock", None, False), + ("_HANDLE_TABLE", "HandleCount", True), + ("_MM_AVL_TABLE", None, True), + ], +) + is_win10_up_to_15063 = OsDistinguisher( version_check=lambda x: (10, 0) <= x < (10, 0, 15063), fallback_checks=[ From 4048d0b122eb9030a011012fd4f1ff59d1e0ca46 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 9 Jul 2024 11:43:29 -0500 Subject: [PATCH 236/979] Windows: Adds shimcachemem plugin --- .../framework/plugins/windows/shimcachemem.py | 610 ++++++++++++++++++ 1 file changed, 610 insertions(+) create mode 100644 volatility3/framework/plugins/windows/shimcachemem.py diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py new file mode 100644 index 000000000..6afaf4356 --- /dev/null +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -0,0 +1,610 @@ +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +import os +from datetime import datetime +from itertools import count +from typing import Iterator, List, Optional, Tuple + +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols +from volatility3.framework.configuration import requirements +from volatility3.framework.objects.utility import array_to_string +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows import versions +from volatility3.framework.symbols.windows.extensions import pe, shimcache +from volatility3.plugins import timeliner +from volatility3.plugins.windows import modules, pslist, vadinfo + +# from volatility3.plugins.windows import pslist, vadinfo, modules + +vollog = logging.getLogger(__name__) + + +class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): + """Reads Shimcache entries from the ahcache.sys AVL tree""" + + _required_framework_version = (2, 0, 0) + + # These checks must be completed from newest -> oldest OS version. + _win_version_file_map: List[Tuple[versions.OsDistinguisher, bool, str]] = [ + (versions.is_win10, True, "shimcache-win10-x64"), + (versions.is_win10, False, "shimcache-win10-x86"), + (versions.is_windows_8_or_later, True, "shimcache-win8-x64"), + (versions.is_windows_8_or_later, False, "shimcache-win8-x86"), + (versions.is_windows_7, True, "shimcache-win7-x64"), + (versions.is_windows_7, False, "shimcache-win7-x86"), + (versions.is_vista_or_later, True, "shimcache-vista-x64"), + (versions.is_vista_or_later, False, "shimcache-vista-x86"), + (versions.is_2003, False, "shimcache-2003-x86"), + (versions.is_2003, True, "shimcache-2003-x64"), + (versions.is_windows_xp_sp3, False, "shimcache-xp-sp3-x86"), + (versions.is_windows_xp_sp2, False, "shimcache-xp-sp2-x86"), + (versions.is_xp_or_2003, True, "shimcache-xp-2003-x64"), + (versions.is_xp_or_2003, False, "shimcache-xp-2003-x86"), + ] + + NT_KRNL_MODS = ["ntoskrnl.exe", "ntkrnlpa.exe", "ntkrnlmp.exe", "ntkrpamp.exe"] + + def generate_timeline( + self, + ) -> Iterator[Tuple[str, timeliner.TimeLinerType, datetime]]: + for _, (_, last_modified, last_update, _, _, file_path) in self._generator(): + if isinstance(last_update, datetime): + yield f"Shimcache: File {file_path} executed", timeliner.TimeLinerType.ACCESSED, last_update + if isinstance(last_modified, datetime): + yield f"Shimcache: File {file_path} modified", timeliner.TimeLinerType.MODIFIED, last_modified + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="modules", component=modules.Modules, version=(2, 0, 0) + ), + ] + + @staticmethod + def create_shimcache_table( + context: interfaces.context.ContextInterface, + symbol_table: str, + config_path: str, + ) -> str: + """Creates a shimcache symbol table + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + symbol_table: The name of an existing symbol table containing the kernel symbols + config_path: The configuration path within the context of the symbol table to create + + Returns: + The name of the constructed shimcache table + """ + native_types = context.symbol_space[symbol_table].natives + is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) + table_mapping = {"nt_symbols": symbol_table} + + try: + symbol_filename = next( + filename + for version_check, for_64bit, filename in ShimcacheMem._win_version_file_map + if is_64bit == for_64bit + and version_check(context=context, symbol_table=symbol_table) + ) + except StopIteration: + raise NotImplementedError("This version of Windows is not supported!") + + vollog.debug(f"Using shimcache table {symbol_filename}") + + return intermed.IntermediateSymbolTable.create( + context, + config_path, + os.path.join("windows", "shimcache"), + symbol_filename, + class_types=shimcache.class_types, + native_types=native_types, + table_mapping=table_mapping, + ) + + @classmethod + def find_shimcache_win_xp( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + kernel_symbol_table: str, + shimcache_symbol_table: str, + ) -> Iterator[shimcache.SHIM_CACHE_ENTRY]: + """Attempts to find the shimcache in a Windows XP memory image + + :param context: The context to retrieve required elements (layers, symbol tables) from + :param layer_name: The name of the memory layer on which to operate. + :param kernel_symbol_table: The name of an existing symbol table containing the kernel symbols + :param shimcache_symbol_table: The name of a symbol table containing the hand-crafted shimcache symbols + """ + + SHIM_NUM_ENTRIES_OFFSET = 0x8 + SHIM_MAX_ENTRIES = 0x60 # 96 max entries in XP shim cache + SHIM_LRU_OFFSET = 0x10 + SHIM_HEADER_SIZE = 0x190 + SHIM_CACHE_ENTRY_SIZE = 0x228 + + seen = set() + + for process in pslist.PsList.list_processes( + context, layer_name, kernel_symbol_table + ): + pid = process.UniqueProcessId + vollog.debug("checking process %d" % pid) + for vad in vadinfo.VadInfo.list_vads( + process, lambda x: x.get_tag() == b"Vad " and x.Protection == 4 + ): + try: + proc_layer_name = process.add_process_layer() + proc_layer = context.layers[proc_layer_name] + except exceptions.InvalidAddressException: + continue + + try: + if proc_layer.read(vad.get_start(), 4) != b"\xEF\xBE\xAD\xDE": + if pid == 624: + vollog.debug("VAD magic bytes don't match DEADBEEF") + continue + except exceptions.InvalidAddressException: + continue + + num_entries = context.object( + shimcache_symbol_table + constants.BANG + "unsigned int", + proc_layer_name, + vad.get_start() + SHIM_NUM_ENTRIES_OFFSET, + ) + + if num_entries > SHIM_MAX_ENTRIES: + continue + + cache_idx_ptr = vad.get_start() + SHIM_LRU_OFFSET + + for _ in range(num_entries): + cache_idx_val = proc_layer.context.object( + shimcache_symbol_table + constants.BANG + "unsigned long", + proc_layer_name, + cache_idx_ptr, + ) + + cache_idx_ptr += 4 + + if cache_idx_val > SHIM_MAX_ENTRIES - 1: + continue + + shim_entry_offset = ( + vad.get_start() + + SHIM_HEADER_SIZE + + (SHIM_CACHE_ENTRY_SIZE * cache_idx_val) + ) + + if not proc_layer.is_valid(shim_entry_offset): + continue + + physical_addr = proc_layer.translate(shim_entry_offset) + + if physical_addr in seen: + continue + seen.add(physical_addr) + + shim_entry = proc_layer.context.object( + shimcache_symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", + proc_layer_name, + shim_entry_offset, + ) + if not proc_layer.is_valid(shim_entry.vol.offset): + continue + if not shim_entry.is_valid(): + continue + + yield shim_entry + + @classmethod + def find_shimcache_win_2k3_to_7( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + kernel_layer_name: str, + nt_symbol_table: str, + shimcache_symbol_table: str, + ) -> Iterator[shimcache.SHIM_CACHE_ENTRY]: + """Implements the algorithm to search for the shim cache on Windows 2000 + (x64) through Windows 7 / 2008 R2. The algorithm consists of the following: + + 1) Find the NT kernel module's .data and PAGE sections + 2) Iterate over every 4/8 bytes (depending on OS bitness) in the .data + section and test for the following: + a) offset represents a valid RTL_AVL_TABLE object + b) RTL_AVL_TABLE is preceeded by an ERESOURCE object + c) RTL_AVL_TABLE is followed by the beginning of the SHIM LRU list + + :param context: The context to retrieve required elements (layers, symbol tables) from + :param layer_name: The name of the memory layer on which to operate. + :param kernel_symbol_table: The name of an existing symbol table containing the kernel symbols + :param shimcache_symbol_table: The name of a symbol table containing the hand-crafted shimcache symbols + """ + + data_sec = cls.get_module_section_range( + context, + config_path, + kernel_layer_name, + nt_symbol_table, + cls.NT_KRNL_MODS, + ".data", + ) + mod_page = cls.get_module_section_range( + context, + config_path, + kernel_layer_name, + nt_symbol_table, + cls.NT_KRNL_MODS, + "PAGE", + ) + + # We require both in order to accurately handle AVL table + if not (data_sec and mod_page): + return None + + data_sec_offset, data_sec_size = data_sec + mod_page_offset, mod_page_size = mod_page + + addr_size = 8 if symbols.symbol_table_is_64bit(context, nt_symbol_table) else 4 + + shim_head = None + for offset in range( + data_sec_offset, data_sec_offset + data_sec_size, addr_size + ): + shim_head = cls.try_get_shim_head_at_offset( + context, + shimcache_symbol_table, + nt_symbol_table, + kernel_layer_name, + mod_page_offset, + mod_page_offset + mod_page_size, + offset, + ) + + if shim_head: + break + + if not shim_head: + return + + for shim_entry in shim_head.ListEntry.to_list( + shimcache_symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", "ListEntry" + ): + yield shim_entry + + @classmethod + def try_get_shim_head_at_offset( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + kernel_symbol_table: str, + layer_name: str, + mod_page_start: int, + mod_page_end: int, + offset: int, + ) -> Optional[shimcache.SHIM_CACHE_ENTRY]: + """Attempts to construct a SHIM_CACHE_HEAD within a layer of the given context, + using the provided offset within that layer, as well as the start and end offsets + of the kernel module's `PAGE` section start and end offsets. + + If a number of validity checks are passed, this method will return the `SHIM_CACHE_HEAD` + object. Otherwise, `None` is returned. + """ + # print("checking RTL_AVL_TABLE at offset %s" % hex(offset)) + rtl_avl_table = context.object( + symbol_table + constants.BANG + "_RTL_AVL_TABLE", layer_name, offset + ) + if not rtl_avl_table.is_valid(mod_page_start, mod_page_end): + return None + + vollog.debug(f"Candidate RTL_AVL_TABLE found at offset {hex(offset)}") + + ersrc_size = context.symbol_space.get_type( + kernel_symbol_table + constants.BANG + "_ERESOURCE" + ).size + ersrc_alignment = ( + 0x20 + if symbols.symbol_table_is_64bit(context, kernel_symbol_table) + else 0x10 + # 0x20 if context.symbol_space.get_type("pointer").size == 8 else 0x10 + ) + vollog.debug( + f"ERESOURCE size: {hex(ersrc_size)}, ERESOURCE alignment: {hex(ersrc_alignment)}" + ) + + eresource_rel_off = ersrc_size + ((offset - ersrc_size) % ersrc_alignment) + eresource_offset = offset - eresource_rel_off + + vollog.debug("Constructing ERESOURCE at %s" % hex(eresource_offset)) + eresource = context.object( + kernel_symbol_table + constants.BANG + "_ERESOURCE", + layer_name, + eresource_offset, + ) + if not eresource.is_valid(): + vollog.debug("ERESOURCE Invalid") + return None + + shim_head_offset = offset + rtl_avl_table.vol.size + + if not context.layers[layer_name].is_valid(shim_head_offset): + return None + + shim_head = context.object( + symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", + layer_name, + shim_head_offset, + ) + + if not shim_head.is_valid(): + vollog.debug("shim head invalid") + return None + else: + vollog.debug("returning shim head") + return shim_head + + @classmethod + def find_shimcache_win_8_or_later( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + kernel_layer_name: str, + nt_symbol_table: str, + shimcache_symbol_table: str, + ) -> Iterator[shimcache.SHIM_CACHE_ENTRY]: + """Attempts to locate and yield shimcache entries from a Windows 8 or later memory image. + + :param context: The context to retrieve required elements (layers, symbol tables) from + :param layer_name: The name of the memory layer on which to operate. + :param kernel_symbol_table: The name of an existing symbol table containing the kernel symbols + :param shimcache_symbol_table: The name of a symbol table containing the hand-crafted shimcache symbols + """ + + is_8_1_or_later = versions.is_windows_8_1_or_later( + context, nt_symbol_table + ) or versions.is_win10(context, nt_symbol_table) + + module_names = ["ahcache.sys"] if is_8_1_or_later else cls.NT_KRNL_MODS + vollog.debug(f"Searching for modules {module_names}") + + data_sec = cls.get_module_section_range( + context, + config_path, + kernel_layer_name, + nt_symbol_table, + module_names, + ".data", + ) + mod_page = cls.get_module_section_range( + context, + config_path, + kernel_layer_name, + nt_symbol_table, + module_names, + "PAGE", + ) + + if not (data_sec and mod_page): + return None + + mod_page_offset, mod_page_size = mod_page + data_sec_offset, data_sec_size = data_sec + + # iterate over ahcache kernel module's .data section in search of *two* SHIM handles + shim_heads = [] + + vollog.debug(f"PAGE offset: {hex(mod_page_offset)}") + vollog.debug(f".data offset: {hex(data_sec_offset)}") + + handle_type = context.symbol_space.get_type( + shimcache_symbol_table + constants.BANG + "SHIM_CACHE_HANDLE" + ) + for offset in range( + data_sec_offset, + data_sec_offset + data_sec_size, + 8 if symbols.symbol_table_is_64bit(context, nt_symbol_table) else 4, + ): + vollog.debug(f"Building shim handle pointer at {hex(offset)}") + shim_handle = context.object( + object_type=shimcache_symbol_table + constants.BANG + "pointer", + layer_name=kernel_layer_name, + subtype=handle_type, + offset=offset, + ) + + if shim_handle.is_valid(mod_page_offset, mod_page_offset + mod_page_size): + if shim_handle.head is not None: + vollog.debug( + f"Found valid shim handle @ {hex(shim_handle.vol.offset)}" + ) + shim_heads.append(shim_handle.head) + if len(shim_heads) == 2: + break + + if len(shim_heads) != 2: + vollog.debug("Failed to identify two valid SHIM_CACHE_HANDLE structures") + return + + # On Windows 8 x64, the frist cache contains the shim cache + # On Windows 8 x86, 8.1 x86/x64, and 10, the second cache contains the shim cache. + if ( + not symbols.symbol_table_is_64bit(context, nt_symbol_table) + and not is_8_1_or_later + ): + valid_head = shim_heads[1] + elif not is_8_1_or_later: + valid_head = shim_heads[0] + else: + valid_head = shim_heads[1] + + for shim_entry in valid_head.ListEntry.to_list( + shimcache_symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", "ListEntry" + ): + if shim_entry.is_valid(): + yield shim_entry + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + shimcache_table_name = self.create_shimcache_table( + self.context, kernel.symbol_table_name, self.config_path + ) + + c = count() + + if versions.is_windows_8_or_later(self._context, kernel.symbol_table_name): + vollog.info("Finding shimcache entries for Windows 8.0+") + entries = self.find_shimcache_win_8_or_later( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + shimcache_table_name, + ) + + elif ( + versions.is_2003(self.context, kernel.symbol_table_name) + or versions.is_vista_or_later(self.context, kernel.symbol_table_name) + or versions.is_windows_7(self.context, kernel.symbol_table_name) + ): + vollog.info("Finding shimcache entries for Windows 2k3/Vista/7") + entries = self.find_shimcache_win_2k3_to_7( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + shimcache_table_name, + ) + + elif versions.is_windows_xp_sp2( + self._context, kernel.symbol_table_name + ) or versions.is_windows_xp_sp3(self.context, kernel.symbol_table_name): + vollog.info("Finding shimcache entries for WinXP") + entries = self.find_shimcache_win_xp( + self._context, + kernel.layer_name, + kernel.symbol_table_name, + shimcache_table_name, + ) + else: + vollog.warn("Cannot parse shimcache entries for this version of Windows") + return + + for entry in entries: + try: + vollog.debug(f"SHIM_CACHE_ENTRY type: {entry.__class__}") + shim_entry = ( + entry.last_modified, + entry.last_update, + entry.exec_flag, + ( + format_hints.Hex(entry.file_size) + if isinstance(entry.file_size, int) + else entry.file_size + ), + entry.file_path, + ) + except exceptions.InvalidAddressException: + continue + + yield ( + 0, + (next(c), *shim_entry), + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Order", int), + ("Last Modified", datetime), + ("Last Update", datetime), + ("Exec Flag", bool), + ("File Size", format_hints.Hex), + ("File Path", str), + ], + self._generator(), + ) + + @classmethod + def get_module_section_range( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + layer_name: str, + symbol_table: str, + module_list: List[str], + section_name: str, + ) -> Optional[Tuple[int, int]]: + """Locates the size and offset of the first found module section + specified by name from the list of modules. + + :param context: The context to operate on + :param layer_name: The memory layer to read from + :param module_list: A list of module names to search for the given section + :param section_name: The name of the section to search for. + + :return: The offset and size of the module, if found; Otherwise, returns `None` + """ + + try: + krnl_mod = next( + module + for module in modules.Modules.list_modules( + context, layer_name, symbol_table + ) + if module.BaseDllName.String in module_list + ) + except StopIteration: + return None + + pe_table_name = intermed.IntermediateSymbolTable.create( + context, + interfaces.configuration.path_join(config_path, "pe"), + "windows", + "pe", + class_types=pe.class_types, + ) + + # code taken from Win32KBase._section_chunks (win32_core.py) + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + layer_name, + offset=krnl_mod.DllBase, + ) + + if not dos_header: + return None + + nt_header = dos_header.get_nt_header() + + try: + section = next( + sec + for sec in nt_header.get_sections() + if section_name.lower() == array_to_string(sec.Name).lower() + ) + except StopIteration: + return None + + section_offset = krnl_mod.DllBase + section.VirtualAddress + section_size = section.Misc.VirtualSize + + return section_offset, section_size From 1508a414992ec958859ec5de84c3fbb51209d55f Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Thu, 18 Jul 2024 11:33:30 -0500 Subject: [PATCH 237/979] Move registry table init into the generator function --- .../framework/plugins/windows/registry/userassist.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index cf345c901..bd832b20c 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -286,6 +286,10 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac hive_offsets = [self.config.get("offset", None)] kernel = self.context.modules[self.config["kernel"]] + self._reg_table_name = intermed.IntermediateSymbolTable.create( + self.context, self._config_path, "windows", "registry" + ) + # get all the user hive offsets or use the one specified for hive in hivelist.HiveList.list_hives( context=self.context, @@ -337,10 +341,6 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac yield result def generate_timeline(self): - self._reg_table_name = intermed.IntermediateSymbolTable.create( - self.context, self._config_path, "windows", "registry" - ) - for row in self._generator(): _depth, row_data = row # check the name and the timestamp to not be empty @@ -351,10 +351,6 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac yield (description, timeliner.TimeLinerType.MODIFIED, row_data[10]) def run(self): - self._reg_table_name = intermed.IntermediateSymbolTable.create( - self.context, self._config_path, "windows", "registry" - ) - return renderers.TreeGrid( [ ("Hive Offset", renderers.format_hints.Hex), From 8da046530fd7ccd3f8a75fbe4e0b63d213207117 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 14:57:31 -0500 Subject: [PATCH 238/979] Address feedback --- .../framework/plugins/windows/svcdiff.py | 22 ++++--- .../framework/plugins/windows/svclist.py | 58 +++++++++++-------- .../framework/plugins/windows/svcscan.py | 43 ++++++++------ 3 files changed, 69 insertions(+), 54 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py index 809064946..c41a7b86c 100644 --- a/volatility3/framework/plugins/windows/svcdiff.py +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -20,8 +20,7 @@ from volatility3.framework.symbols.windows import versions vollog = logging.getLogger(__name__) - -class SvcDiff(svclist.SvcList, svcscan.SvcScan): +class SvcDiff(svcscan.SvcScan): """Compares services found through list walking versus scanning to find rootkits""" _required_framework_version = (2, 4, 0) @@ -39,22 +38,23 @@ class SvcDiff(svclist.SvcList, svcscan.SvcScan): name="svclist", component=svclist.SvcList, version=(1, 0, 0) ), requirements.VersionRequirement( - name="svcscan", component=svcscan.SvcScan, version=(2, 0, 0) + name="svcscan", component=svcscan.SvcScan, version=(3, 0, 0) ), ] def _generator(self): """ - Finds services by walking the services.exe list on supported Windows 10 versions + On Windows 10 version 15063+ 64bit Windows memory samples, walk the services list + and scan for services then report differences """ - kernel = self.context.modules[self.config["kernel"]] + kernel, service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() if not symbols.symbol_table_is_64bit( self.context, kernel.symbol_table_name ) or not versions.is_win10_15063_or_later( context=self.context, symbol_table=kernel.symbol_table_name ): - vollog.info( + vollog.warning( "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" ) return @@ -63,18 +63,16 @@ class SvcDiff(svclist.SvcList, svcscan.SvcScan): from_list = set() records = {} - service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - # collect unique service names from scanning - for service in self.service_scan( - service_table_name, service_binary_dll_map, filter_func + for service in svcscan.SvcScan.service_scan( + self.context, kernel, service_table_name, service_binary_dll_map, filter_func ): from_scan.add(service[6]) records[service[6]] = service # collect services from listing walking - for service in self.service_list( - service_table_name, service_binary_dll_map, filter_func + for service in svclist.SvcList.service_list( + self.context, kernel, service_table_name, service_binary_dll_map, filter_func ): from_list.add(service[6]) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index 1938dd182..832b3d129 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -4,7 +4,7 @@ import logging -from typing import List +from typing import List, Optional, Tuple from volatility3.framework import interfaces, exceptions, symbols from volatility3.framework.configuration import requirements @@ -20,16 +20,26 @@ class SvcList(svcscan.SvcScan): _version = (1, 0, 0) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._enumeration_method = self.service_list + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.PluginRequirement( - name="svcscan", plugin=svcscan.SvcScan, version=(2, 0, 0) + name="svcscan", plugin=svcscan.SvcScan, version=(3, 0, 0) + ), + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], ), ] - def _get_exe_range(self, proc): + @classmethod + def _get_exe_range(cls, proc) -> Optional[Tuple[int, int]]: """ Returns a tuple of starting,ending address for the VAD containing services.exe @@ -45,21 +55,27 @@ class SvcList(svcscan.SvcScan): return None - def service_list(self, service_table_name, service_binary_dll_map, filter_func): - kernel = self.context.modules[self.config["kernel"]] - + @classmethod + def service_list( + cls, + context: interfaces.context.ContextInterface, + kernel, + service_table_name: str, + service_binary_dll_map, + filter_func, + ): if not symbols.symbol_table_is_64bit( - self.context, kernel.symbol_table_name + context, kernel.symbol_table_name ) or not versions.is_win10_15063_or_later( - context=self.context, symbol_table=kernel.symbol_table_name + context=context, symbol_table=kernel.symbol_table_name ): - vollog.info( + vollog.warning( "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" ) return for proc in pslist.PsList.list_processes( - context=self.context, + context=context, layer_name=kernel.layer_name, symbol_table=kernel.symbol_table_name, filter_func=filter_func, @@ -74,9 +90,9 @@ class SvcList(svcscan.SvcScan): ) continue - layer = self.context.layers[layer_name] + layer = context.layers[layer_name] - exe_range = self._get_exe_range(proc) + exe_range = cls._get_exe_range(proc) if not exe_range: vollog.warning( "Could not find the application executable VAD for services.exe. Unable to proceed." @@ -84,19 +100,15 @@ class SvcList(svcscan.SvcScan): continue for offset in layer.scan( - context=self.context, + context=context, scanner=scanners.BytesScanner(needle=b"Sc27"), sections=exe_range, ): - for record in self.enumerate_vista_or_later_header( - service_table_name, service_binary_dll_map, layer_name, offset + for record in cls.enumerate_vista_or_later_header( + context, + service_table_name, + service_binary_dll_map, + layer_name, + offset, ): yield record - - def _generator(self): - service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - - for record in self.service_list( - service_table_name, service_binary_dll_map, filter_func - ): - yield (0, record) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 07274f03d..4368b83ce 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -39,7 +39,11 @@ class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (3, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._enumeration_method = self.service_scan @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -232,13 +236,14 @@ class SvcScan(interfaces.plugins.PluginInterface): for service_key in services } + @classmethod def enumerate_vista_or_later_header( - self, service_table_name, service_binary_dll_map, proc_layer_name, offset + cls, context, service_table_name, service_binary_dll_map, proc_layer_name, offset ): if offset % 8: return - service_header = self.context.object( + service_header =context.object( service_table_name + constants.BANG + "_SERVICE_HEADER", offset=offset, layer_name=proc_layer_name, @@ -257,17 +262,16 @@ class SvcScan(interfaces.plugins.PluginInterface): renderers.UnreadableValue(), renderers.UnreadableValue() ), ) - yield self.get_record_tuple(service_record, service_info) + yield cls.get_record_tuple(service_record, service_info) - def service_scan(self, service_table_name, service_binary_dll_map, filter_func): - kernel = self.context.modules[self.config["kernel"]] - - relative_tag_offset = self.context.symbol_space.get_type( + @classmethod + def service_scan(cls, context: interfaces.context.ContextInterface, kernel, service_table_name: str, service_binary_dll_map, filter_func): + relative_tag_offset = context.symbol_space.get_type( service_table_name + constants.BANG + "_SERVICE_RECORD" ).relative_child_offset("Tag") is_vista_or_later = versions.is_vista_or_later( - context=self.context, symbol_table=kernel.symbol_table_name + context=context, symbol_table=kernel.symbol_table_name ) if is_vista_or_later: @@ -278,7 +282,7 @@ class SvcScan(interfaces.plugins.PluginInterface): seen = [] for task in pslist.PsList.list_processes( - context=self.context, + context=context, layer_name=kernel.layer_name, symbol_table=kernel.symbol_table_name, filter_func=filter_func, @@ -295,15 +299,15 @@ class SvcScan(interfaces.plugins.PluginInterface): ) continue - layer = self.context.layers[proc_layer_name] + layer = context.layers[proc_layer_name] for offset in layer.scan( - context=self.context, + context=context, scanner=scanners.BytesScanner(needle=service_tag), sections=vadyarascan.VadYaraScan.get_vad_maps(task), ): if not is_vista_or_later: - service_record = self.context.object( + service_record = context.object( service_table_name + constants.BANG + "_SERVICE_RECORD", offset=offset - relative_tag_offset, layer_name=proc_layer_name, @@ -318,9 +322,10 @@ class SvcScan(interfaces.plugins.PluginInterface): renderers.UnreadableValue(), renderers.UnreadableValue() ), ) - yield self.get_record_tuple(service_record, service_info) + yield cls.get_record_tuple(service_record, service_info) else: - for service_record in self.enumerate_vista_or_later_header( + for service_record in cls.enumerate_vista_or_later_header( + context, service_table_name, service_binary_dll_map, proc_layer_name, @@ -351,13 +356,13 @@ class SvcScan(interfaces.plugins.PluginInterface): filter_func = pslist.PsList.create_name_filter(["services.exe"]) - return service_table_name, service_binary_dll_map, filter_func + return kernel, service_table_name, service_binary_dll_map, filter_func def _generator(self): - service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + kernel, service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() - for record in self.service_scan( - service_table_name, service_binary_dll_map, filter_func + for record in self._enumeration_method( + self.context, kernel, service_table_name, service_binary_dll_map, filter_func ): yield (0, record) From a7af052509417c08eae702cd59fbfaa5306cc686 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 14:59:40 -0500 Subject: [PATCH 239/979] Black fixes --- .../framework/plugins/windows/svcdiff.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py index c41a7b86c..4325771db 100644 --- a/volatility3/framework/plugins/windows/svcdiff.py +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -20,6 +20,7 @@ from volatility3.framework.symbols.windows import versions vollog = logging.getLogger(__name__) + class SvcDiff(svcscan.SvcScan): """Compares services found through list walking versus scanning to find rootkits""" @@ -47,7 +48,9 @@ class SvcDiff(svcscan.SvcScan): On Windows 10 version 15063+ 64bit Windows memory samples, walk the services list and scan for services then report differences """ - kernel, service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + kernel, service_table_name, service_binary_dll_map, filter_func = ( + self.get_prereq_info() + ) if not symbols.symbol_table_is_64bit( self.context, kernel.symbol_table_name @@ -65,14 +68,22 @@ class SvcDiff(svcscan.SvcScan): # collect unique service names from scanning for service in svcscan.SvcScan.service_scan( - self.context, kernel, service_table_name, service_binary_dll_map, filter_func + self.context, + kernel, + service_table_name, + service_binary_dll_map, + filter_func, ): from_scan.add(service[6]) records[service[6]] = service # collect services from listing walking for service in svclist.SvcList.service_list( - self.context, kernel, service_table_name, service_binary_dll_map, filter_func + self.context, + kernel, + service_table_name, + service_binary_dll_map, + filter_func, ): from_list.add(service[6]) From 2a5e94a0946e4c74f92588df6ceafdd89acd0be4 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 15:01:10 -0500 Subject: [PATCH 240/979] Black fixes --- .../framework/plugins/windows/svcscan.py | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 4368b83ce..8b9706625 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -238,12 +238,17 @@ class SvcScan(interfaces.plugins.PluginInterface): @classmethod def enumerate_vista_or_later_header( - cls, context, service_table_name, service_binary_dll_map, proc_layer_name, offset + cls, + context, + service_table_name, + service_binary_dll_map, + proc_layer_name, + offset, ): if offset % 8: return - service_header =context.object( + service_header = context.object( service_table_name + constants.BANG + "_SERVICE_HEADER", offset=offset, layer_name=proc_layer_name, @@ -265,7 +270,14 @@ class SvcScan(interfaces.plugins.PluginInterface): yield cls.get_record_tuple(service_record, service_info) @classmethod - def service_scan(cls, context: interfaces.context.ContextInterface, kernel, service_table_name: str, service_binary_dll_map, filter_func): + def service_scan( + cls, + context: interfaces.context.ContextInterface, + kernel, + service_table_name: str, + service_binary_dll_map, + filter_func, + ): relative_tag_offset = context.symbol_space.get_type( service_table_name + constants.BANG + "_SERVICE_RECORD" ).relative_child_offset("Tag") @@ -359,10 +371,16 @@ class SvcScan(interfaces.plugins.PluginInterface): return kernel, service_table_name, service_binary_dll_map, filter_func def _generator(self): - kernel, service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info() + kernel, service_table_name, service_binary_dll_map, filter_func = ( + self.get_prereq_info() + ) for record in self._enumeration_method( - self.context, kernel, service_table_name, service_binary_dll_map, filter_func + self.context, + kernel, + service_table_name, + service_binary_dll_map, + filter_func, ): yield (0, record) From b0a89f210977663c8d106c50ac4602eb48b7b211 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 15:47:37 -0500 Subject: [PATCH 241/979] Address feedback --- volatility3/framework/plugins/windows/processghosting.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/processghosting.py b/volatility3/framework/plugins/windows/processghosting.py index b29ff04f0..dda0d7675 100644 --- a/volatility3/framework/plugins/windows/processghosting.py +++ b/volatility3/framework/plugins/windows/processghosting.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging +import contextlib from volatility3.framework import interfaces, exceptions from volatility3.framework import renderers @@ -57,14 +58,15 @@ class ProcessGhosting(interfaces.plugins.PluginInterface): else: file_object = 0 + if isinstance(delete_pending, int) and delete_pending not in [0, 1]: + vollog.debug(f"Invalid delete_pending value {delete_pending} found for {process_name} {proc.UniqueProcessId}") + # delete_pending besides 0 or 1 = smear if file_object == 0 or delete_pending == 1: path = renderers.UnreadableValue() if file_object: - try: + with contextlib.suppress(exceptions.InvalidAddressException): path = file_object.FileName.String - except exceptions.InvalidAddressException: - path = renderers.UnreadableValue() yield ( 0, From 920b3ec615b91ec4dcc4bdf0ef9d15715a5b1c8d Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 15:48:21 -0500 Subject: [PATCH 242/979] Black fix --- volatility3/framework/plugins/windows/processghosting.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/processghosting.py b/volatility3/framework/plugins/windows/processghosting.py index dda0d7675..50f02c926 100644 --- a/volatility3/framework/plugins/windows/processghosting.py +++ b/volatility3/framework/plugins/windows/processghosting.py @@ -59,7 +59,9 @@ class ProcessGhosting(interfaces.plugins.PluginInterface): file_object = 0 if isinstance(delete_pending, int) and delete_pending not in [0, 1]: - vollog.debug(f"Invalid delete_pending value {delete_pending} found for {process_name} {proc.UniqueProcessId}") + vollog.debug( + f"Invalid delete_pending value {delete_pending} found for {process_name} {proc.UniqueProcessId}" + ) # delete_pending besides 0 or 1 = smear if file_object == 0 or delete_pending == 1: From 75ccf1bfab5ee6bc3d0f84baf427c11b8e61137c Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 17:54:33 -0500 Subject: [PATCH 243/979] Add dedicated plugin and API for extracting PE files from kernel and process memory --- .../framework/plugins/windows/dlllist.py | 76 +----- .../framework/plugins/windows/modscan.py | 11 +- .../framework/plugins/windows/modules.py | 12 +- .../framework/plugins/windows/pedump.py | 239 ++++++++++++++++++ 4 files changed, 258 insertions(+), 80 deletions(-) create mode 100644 volatility3/framework/plugins/windows/pedump.py diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index d48a53663..eef826ed5 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -1,10 +1,9 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import contextlib import datetime import logging -import ntpath import re from typing import List, Optional, Type @@ -14,7 +13,7 @@ from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins import timeliner -from volatility3.plugins.windows import info, pslist, psscan +from volatility3.plugins.windows import info, pslist, psscan, pedump vollog = logging.getLogger(__name__) @@ -23,7 +22,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the loaded modules in a particular windows memory image.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (3, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -76,66 +75,10 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): default=False, optional=True, ), - ] - - @classmethod - def dump_pe( - cls, - context: interfaces.context.ContextInterface, - pe_table_name: str, - dll_entry: interfaces.objects.ObjectInterface, - open_method: Type[interfaces.plugins.FileHandlerInterface], - layer_name: str = None, - prefix: str = "", - ) -> Optional[interfaces.plugins.FileHandlerInterface]: - """Extracts the complete data for a process as a FileInterface - - Args: - context: the context to operate upon - pe_table_name: the name for the symbol table containing the PE format symbols - dll_entry: the object representing the module - layer_name: the layer that the DLL lives within - open_method: class for constructing output files - - Returns: - An open FileHandlerInterface object containing the complete data for the DLL or None in the case of failure - """ - try: - try: - name = dll_entry.FullDllName.get_string() - except exceptions.InvalidAddressException: - name = "UnreadableDLLName" - - if layer_name is None: - layer_name = dll_entry.vol.layer_name - - file_handle = open_method( - "{}{}.{:#x}.{:#x}.dmp".format( - prefix, - ntpath.basename(name), - dll_entry.vol.offset, - dll_entry.DllBase, - ) - ) - - dos_header = context.object( - pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset=dll_entry.DllBase, - layer_name=layer_name, - ) - - for offset, data in dos_header.reconstruct(): - file_handle.seek(offset) - file_handle.write(data) - except ( - IOError, - exceptions.VolatilityException, - OverflowError, - ValueError, - ) as excp: - vollog.debug(f"Unable to dump dll at offset {dll_entry.DllBase}: {excp}") - return None - return file_handle + requirements.VersionRequirement( + name="pedump", component=pedump.PEDump, version=(1, 0, 0) + ), + ] def _generator(self, procs): pe_table_name = intermed.IntermediateSymbolTable.create( @@ -204,7 +147,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_output = "Disabled" if self.config["dump"]: - file_handle = self.dump_pe( + file_handle = pedump.PEDump.dump_ldr_entry( self.context, pe_table_name, entry, @@ -214,8 +157,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) file_output = "Error outputting file" if file_handle: - file_handle.close() - file_output = file_handle.preferred_filename + file_output = file_handle try: dllbase = format_hints.Hex(entry.DllBase) except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index 98546bc9c..fc45e6913 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -6,7 +6,7 @@ from typing import Iterable from volatility3.framework import interfaces from volatility3.framework.configuration import requirements -from volatility3.plugins.windows import poolscanner, dlllist, pslist, modules +from volatility3.plugins.windows import poolscanner, modules, pedump vollog = logging.getLogger(__name__) @@ -35,12 +35,6 @@ class ModScan(modules.Modules): requirements.VersionRequirement( name="modules", component=modules.Modules, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="pslist", component=pslist.PsList, version=(2, 0, 0) - ), - requirements.VersionRequirement( - name="dlllist", component=dlllist.DllList, version=(2, 0, 0) - ), requirements.BooleanRequirement( name="dump", description="Extract listed modules", @@ -58,6 +52,9 @@ class ModScan(modules.Modules): optional=True, default=None, ), + requirements.VersionRequirement( + name="pedump", component=pedump.PEDump, version=(1, 0, 0) + ), ] @classmethod diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 79eea1cd7..283d4dcb9 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -9,7 +9,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import pslist, dlllist +from volatility3.plugins.windows import pslist, pedump vollog = logging.getLogger(__name__) @@ -35,9 +35,6 @@ class Modules(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="dlllist", component=dlllist.DllList, version=(2, 0, 0) - ), requirements.BooleanRequirement( name="dump", description="Extract listed modules", @@ -55,6 +52,9 @@ class Modules(interfaces.plugins.PluginInterface): optional=True, default=None, ), + requirements.VersionRequirement( + name="pedump", component=pedump.PEDump, version=(1, 0, 0) + ), ] def dump_module(self, session_layers, pe_table_name, mod): @@ -63,7 +63,7 @@ class Modules(interfaces.plugins.PluginInterface): ) file_output = f"Cannot find a viable session layer for {mod.DllBase:#x}" if session_layer_name: - file_handle = dlllist.DllList.dump_pe( + file_handle = pedump.PEDump.dump_ldr_entry( self.context, pe_table_name, mod, @@ -72,7 +72,7 @@ class Modules(interfaces.plugins.PluginInterface): ) file_output = "Error outputting file" if file_handle: - file_output = file_handle.preferred_filename + file_output = file_handle return file_output diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py new file mode 100644 index 000000000..395d28ce5 --- /dev/null +++ b/volatility3/framework/plugins/windows/pedump.py @@ -0,0 +1,239 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +import ntpath +from typing import List, Type, Optional + +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import pe +from volatility3.plugins.windows import pslist, modules + +vollog = logging.getLogger(__name__) + + +class PEDump(interfaces.plugins.PluginInterface): + """Allows extracting PE Files from a specific address in a specific address space""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), + requirements.IntRequirement( + name="base", + description="Base address to reconstruct a PE file", + optional=False, + ), + requirements.BooleanRequirement( + name="kernel_module", + description="Extract from kernel address space.", + default=False, + optional=True, + ), + ] + + @classmethod + def dump_pe( + cls, + context: interfaces.context.ContextInterface, + pe_table_name: str, + layer_name: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], + file_name: str, + base: int + ) -> Optional[str]: + """ + Returns the filename of the dump file or None + """ + try: + file_handle = open_method(file_name) + + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base, + layer_name=layer_name, + ) + + for offset, data in dos_header.reconstruct(): + file_handle.seek(offset) + file_handle.write(data) + except ( + IOError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: + vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") + return None + finally: + file_handle.close() + + return file_handle.preferred_filename + + @classmethod + def dump_ldr_entry( + cls, + context: interfaces.context.ContextInterface, + pe_table_name: str, + ldr_entry: interfaces.objects.ObjectInterface, + open_method: Type[interfaces.plugins.FileHandlerInterface], + layer_name: str = None, + prefix: str = "", + ) -> Optional[str]: + """Extracts the PE file referenced an LDR_DATA_TABLE_ENTRY (DLL, kernel module) instance + + Args: + context: the context to operate upon + pe_table_name: the name for the symbol table containing the PE format symbols + ldr_entry: the object representing the module + open_method: class for constructing output files + layer_name: the layer that the DLL lives within + prefix: optional string to prepend to filename + Returns: + The output file name or None in the case of failure + """ + try: + name = ldr_entry.FullDllName.get_string() + except exceptions.InvalidAddressException: + name = "UnreadableDLLName" + + if layer_name is None: + layer_name = ldr_entry.vol.layer_name + + file_name = "{}{}.{:#x}.{:#x}.dmp".format( + prefix, + ntpath.basename(name), + ldr_entry.vol.offset, + ldr_entry.DllBase, + ) + + return PEDump.dump_pe(context, pe_table_name, layer_name, open_method, file_name, ldr_entry.DllBase) + + @classmethod + def dump_pe_at_base( + cls, + context: interfaces.context.ContextInterface, + pe_table_name: str, + layer_name: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], + proc_offset: int, + pid: int, + base: int, + ) -> Optional[str]: + file_name = "PE.{:#x}.{:d}.{:#x}.dmp".format( + proc_offset, + pid, + base, + ) + + return PEDump.dump_pe(context, pe_table_name, layer_name, open_method, file_name, base) + + @classmethod + def dump_kernel_pe_at_base(cls, context, kernel, pe_table_name, open_method, base): + session_layers = modules.Modules.get_session_layers( + context, kernel.layer_name, kernel.symbol_table_name + ) + + session_layer_name = modules.Modules.find_session_layer( + context, session_layers, base + ) + + if session_layer_name: + system_pid = 4 + + file_output = PEDump.dump_pe_at_base( + context, pe_table_name, session_layer_name, open_method, 0, system_pid, base + ) + + if file_output: + yield system_pid, "Kernel", file_output + else: + vollog.warning( + "Unable to find a session layer with the provided base address mapped in the kernel." + ) + + @classmethod + def dump_processes(cls, context, kernel, pe_table_name, open_method, filter_func, base): + """ + """ + + for proc in pslist.PsList.list_processes( + context=context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ): + pid = proc.UniqueProcessId + proc_name = proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ) + proc_layer_name = proc.add_process_layer() + + file_output = PEDump.dump_pe_at_base( + context, pe_table_name, proc_layer_name, open_method, proc.vol.offset, pid, base + ) + + if file_output: + yield pid, proc_name, file_output + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "windows", "pe", class_types=pe.class_types + ) + + if self.config["kernel_module"] and self.config["pid"]: + vollog.error("Only --kernel_module or --pid should be set. Not both") + return + + if not self.config["kernel_module"] and not self.config["pid"]: + vollog.error("--kernel_module or --pid must be set") + return + + if self.config["kernel_module"]: + pe_files = self.dump_kernel_pe_at_base(self.context, kernel, pe_table_name, self.open, self.config["base"]) + else: + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + pe_files = self.dump_processes(self.context, kernel, pe_table_name, self.open, filter_func, self.config["base"]) + + for pid, proc_name, file_output in pe_files: + yield ( + 0, + ( + pid, + proc_name, + file_output, + ), + ) + + def run(self): + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("File output", str), + ], + self._generator(), + ) From 12f3beacfcb3ab16f8418cee2821ad74c3738045 Mon Sep 17 00:00:00 2001 From: atcuno Date: Thu, 18 Jul 2024 17:56:24 -0500 Subject: [PATCH 244/979] Add dedicated plugin and API for extracting PE files from kernel and process memory --- .../framework/plugins/windows/dlllist.py | 2 +- .../framework/plugins/windows/pedump.py | 69 ++++++++++++++----- 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index eef826ed5..6c8c96dc3 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -78,7 +78,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="pedump", component=pedump.PEDump, version=(1, 0, 0) ), - ] + ] def _generator(self, procs): pe_table_name = intermed.IntermediateSymbolTable.create( diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 395d28ce5..0b4649abb 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -59,7 +59,7 @@ class PEDump(interfaces.plugins.PluginInterface): layer_name: str, open_method: Type[interfaces.plugins.FileHandlerInterface], file_name: str, - base: int + base: int, ) -> Optional[str]: """ Returns the filename of the dump file or None @@ -120,13 +120,20 @@ class PEDump(interfaces.plugins.PluginInterface): layer_name = ldr_entry.vol.layer_name file_name = "{}{}.{:#x}.{:#x}.dmp".format( - prefix, - ntpath.basename(name), - ldr_entry.vol.offset, - ldr_entry.DllBase, - ) + prefix, + ntpath.basename(name), + ldr_entry.vol.offset, + ldr_entry.DllBase, + ) - return PEDump.dump_pe(context, pe_table_name, layer_name, open_method, file_name, ldr_entry.DllBase) + return PEDump.dump_pe( + context, + pe_table_name, + layer_name, + open_method, + file_name, + ldr_entry.DllBase, + ) @classmethod def dump_pe_at_base( @@ -140,12 +147,14 @@ class PEDump(interfaces.plugins.PluginInterface): base: int, ) -> Optional[str]: file_name = "PE.{:#x}.{:d}.{:#x}.dmp".format( - proc_offset, - pid, - base, - ) + proc_offset, + pid, + base, + ) - return PEDump.dump_pe(context, pe_table_name, layer_name, open_method, file_name, base) + return PEDump.dump_pe( + context, pe_table_name, layer_name, open_method, file_name, base + ) @classmethod def dump_kernel_pe_at_base(cls, context, kernel, pe_table_name, open_method, base): @@ -161,7 +170,13 @@ class PEDump(interfaces.plugins.PluginInterface): system_pid = 4 file_output = PEDump.dump_pe_at_base( - context, pe_table_name, session_layer_name, open_method, 0, system_pid, base + context, + pe_table_name, + session_layer_name, + open_method, + 0, + system_pid, + base, ) if file_output: @@ -172,9 +187,10 @@ class PEDump(interfaces.plugins.PluginInterface): ) @classmethod - def dump_processes(cls, context, kernel, pe_table_name, open_method, filter_func, base): - """ - """ + def dump_processes( + cls, context, kernel, pe_table_name, open_method, filter_func, base + ): + """ """ for proc in pslist.PsList.list_processes( context=context, @@ -191,7 +207,13 @@ class PEDump(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() file_output = PEDump.dump_pe_at_base( - context, pe_table_name, proc_layer_name, open_method, proc.vol.offset, pid, base + context, + pe_table_name, + proc_layer_name, + open_method, + proc.vol.offset, + pid, + base, ) if file_output: @@ -213,10 +235,19 @@ class PEDump(interfaces.plugins.PluginInterface): return if self.config["kernel_module"]: - pe_files = self.dump_kernel_pe_at_base(self.context, kernel, pe_table_name, self.open, self.config["base"]) + pe_files = self.dump_kernel_pe_at_base( + self.context, kernel, pe_table_name, self.open, self.config["base"] + ) else: filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - pe_files = self.dump_processes(self.context, kernel, pe_table_name, self.open, filter_func, self.config["base"]) + pe_files = self.dump_processes( + self.context, + kernel, + pe_table_name, + self.open, + filter_func, + self.config["base"], + ) for pid, proc_name, file_output in pe_files: yield ( From bca4cdda744120a5ca868bdfd39119ebf0819201 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Fri, 19 Jul 2024 11:09:52 +0200 Subject: [PATCH 245/979] replace yara-python with yara-x --- doc/requirements.txt | 2 +- test/requirements-testing.txt | 2 +- volatility3/framework/plugins/yarascan.py | 74 ++++++----------------- 3 files changed, 22 insertions(+), 56 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index b715e59f5..c04f71219 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,6 +3,6 @@ sphinx>=4.0.0,<7 sphinx_autodoc_typehints>=1.4.0 sphinx-rtd-theme>=0.4.3 -yara-python +yara-x pycryptodome pefile diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index 7afe19b94..0955d1939 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -5,6 +5,6 @@ pefile>=2017.8.1 #foo # If certain packages are not necessary, place a comment (#) at the start of the line. # This is required for the yara plugins -yara-python>=3.8.0 +yara-x>=0.5.0 pytest>=7.0.0 diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 496ec1844..32a400f9e 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -14,14 +14,9 @@ from volatility3.framework.renderers import format_hints vollog = logging.getLogger(__name__) try: - import yara + import yara_x - if tuple([int(x) for x in yara.__version__.split(".")]) < (3, 8): - raise ImportError except ImportError: - vollog.info( - "Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available" - ) raise @@ -34,27 +29,20 @@ class YaraScanner(interfaces.layers.ScannerInterface): if rules is None: raise ValueError("No rules provided to YaraScanner") self._rules = rules - self.st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( - 4, - 3, - ) def __call__( self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: - for match in self._rules.match(data=data): - if YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: - yield ( - instance.offset + data_offset, - match.rule, - match_string.identifier, - instance.matched_data, - ) - else: - for offset, name, value in match.strings: - yield (offset + data_offset, match.rule, name, value) + results = self._rules.scan(data) + for match in results.matching_rules: + for match_string in match.patterns: + for instance in match_string.matches: + yield ( + instance.offset + data_offset, + f"{match.namespace}.{match.identifier}", + match_string.identifier, + data[instance.offset : instance.offset + instance.length], + ) class YaraScan(plugins.PluginInterface): @@ -63,9 +51,6 @@ class YaraScan(plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (1, 3, 0) - # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string - # or something that makes more sense - @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: """Returns the requirements needed to run yarascan directly, combining the TranslationLayerRequirement @@ -99,16 +84,13 @@ class YaraScan(plugins.PluginInterface): optional=True, ), requirements.StringRequirement( - name="yara_rules", description="Yara rules (as a string)", optional=True + name="yara_string", + description="Yara rules (as a string)", + optional=True, ), - requirements.URIRequirement( - name="yara_file", description="Yara rules (as a file)", optional=True - ), - # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code - # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later requirements.URIRequirement( name="yara_compiled_file", - description="Yara compiled rules (as a file)", + description="Yara-x compiled rules (as a file)", optional=True, ), requirements.IntRequirement( @@ -119,36 +101,20 @@ class YaraScan(plugins.PluginInterface): ), ] - @classmethod - def yara_returns_instances(cls) -> bool: - st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < ( - 4, - 3, - ) - return st_object - @classmethod def process_yara_options(cls, config: Dict[str, Any]): rules = None - if config.get("yara_rules", None) is not None: - rule = config["yara_rules"] + if config.get("yara_string") is not None: + rule = config["yara_string"] if rule[0] not in ["{", "/"]: rule = f'"{rule}"' if config.get("case", False): rule += " nocase" if config.get("wide", False): rule += " wide ascii" - rules = yara.compile( - sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} - ) - elif config.get("yara_source", None) is not None: - rules = yara.compile(source=config["yara_source"]) - elif config.get("yara_file", None) is not None: - rules = yara.compile( - file=resources.ResourceAccessor().open(config["yara_file"], "rb") - ) - elif config.get("yara_compiled_file", None) is not None: - rules = yara.load( + rules = yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") + elif config.get("yara_compiled_file") is not None: + rules = yara_x.Rules.deserialize_from( file=resources.ResourceAccessor().open( config["yara_compiled_file"], "rb" ) From 9454181b9892f5c4245435bb868f24aaee23ae01 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 21 Jul 2024 09:42:19 -0500 Subject: [PATCH 246/979] Address feedback --- volatility3/framework/plugins/windows/dlllist.py | 9 +++++---- volatility3/framework/plugins/windows/modules.py | 7 +++---- volatility3/framework/plugins/windows/pedump.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 6c8c96dc3..5a1b37fcf 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -147,7 +147,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_output = "Disabled" if self.config["dump"]: - file_handle = pedump.PEDump.dump_ldr_entry( + file_output = pedump.PEDump.dump_ldr_entry( self.context, pe_table_name, entry, @@ -155,9 +155,10 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): proc_layer_name, prefix=f"pid.{proc_id}.", ) - file_output = "Error outputting file" - if file_handle: - file_output = file_handle + + if not file_output: + file_output = "Error outputting file" + try: dllbase = format_hints.Hex(entry.DllBase) except exceptions.InvalidAddressException: diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 283d4dcb9..ba45834d5 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -63,16 +63,15 @@ class Modules(interfaces.plugins.PluginInterface): ) file_output = f"Cannot find a viable session layer for {mod.DllBase:#x}" if session_layer_name: - file_handle = pedump.PEDump.dump_ldr_entry( + file_output = pedump.PEDump.dump_ldr_entry( self.context, pe_table_name, mod, self.open, layer_name=session_layer_name, ) - file_output = "Error outputting file" - if file_handle: - file_output = file_handle + if not file_output: + file_output = "Error outputting file" return file_output diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 0b4649abb..858d0615a 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -126,7 +126,7 @@ class PEDump(interfaces.plugins.PluginInterface): ldr_entry.DllBase, ) - return PEDump.dump_pe( + return cls.dump_pe( context, pe_table_name, layer_name, From 21396185d0faaab15a217cff2072aa7de2652b67 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 21 Jul 2024 11:10:38 -0500 Subject: [PATCH 247/979] Address feedback --- .../framework/plugins/windows/svcdiff.py | 37 ++++++++++----- .../framework/plugins/windows/svclist.py | 11 +++-- .../framework/plugins/windows/svcscan.py | 47 +++++++++++-------- 3 files changed, 58 insertions(+), 37 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcdiff.py b/volatility3/framework/plugins/windows/svcdiff.py index 4325771db..84d06a695 100644 --- a/volatility3/framework/plugins/windows/svcdiff.py +++ b/volatility3/framework/plugins/windows/svcdiff.py @@ -13,7 +13,7 @@ import logging -from volatility3.framework import symbols +from volatility3.framework import symbols, interfaces from volatility3.framework.configuration import requirements from volatility3.plugins.windows import svclist, svcscan from volatility3.framework.symbols.windows import versions @@ -26,6 +26,10 @@ class SvcDiff(svcscan.SvcScan): _required_framework_version = (2, 4, 0) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._enumeration_method = self.service_diff + @classmethod def get_requirements(cls): # Since we're calling the plugin, make sure we have the plugin's requirements @@ -43,19 +47,24 @@ class SvcDiff(svcscan.SvcScan): ), ] - def _generator(self): + @classmethod + def service_diff( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + service_table_name: str, + service_binary_dll_map, + filter_func, + ): """ On Windows 10 version 15063+ 64bit Windows memory samples, walk the services list and scan for services then report differences """ - kernel, service_table_name, service_binary_dll_map, filter_func = ( - self.get_prereq_info() - ) - if not symbols.symbol_table_is_64bit( - self.context, kernel.symbol_table_name + context, symbol_table ) or not versions.is_win10_15063_or_later( - context=self.context, symbol_table=kernel.symbol_table_name + context=context, symbol_table=symbol_table ): vollog.warning( "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" @@ -68,8 +77,9 @@ class SvcDiff(svcscan.SvcScan): # collect unique service names from scanning for service in svcscan.SvcScan.service_scan( - self.context, - kernel, + context, + layer_name, + symbol_table, service_table_name, service_binary_dll_map, filter_func, @@ -79,8 +89,9 @@ class SvcDiff(svcscan.SvcScan): # collect services from listing walking for service in svclist.SvcList.service_list( - self.context, - kernel, + context, + layer_name, + symbol_table, service_table_name, service_binary_dll_map, filter_func, @@ -89,4 +100,4 @@ class SvcDiff(svcscan.SvcScan): # report services found from scanning but not list walking for hidden_service in from_scan - from_list: - yield (0, records[hidden_service]) + yield records[hidden_service] diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index 832b3d129..a59581063 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -59,15 +59,16 @@ class SvcList(svcscan.SvcScan): def service_list( cls, context: interfaces.context.ContextInterface, - kernel, + layer_name: str, + symbol_table: str, service_table_name: str, service_binary_dll_map, filter_func, ): if not symbols.symbol_table_is_64bit( - context, kernel.symbol_table_name + context, symbol_table ) or not versions.is_win10_15063_or_later( - context=context, symbol_table=kernel.symbol_table_name + context=context, symbol_table=symbol_table ): vollog.warning( "This plugin only supports Windows 10 version 15063+ 64bit Windows memory samples" @@ -76,8 +77,8 @@ class SvcList(svcscan.SvcScan): for proc in pslist.PsList.list_processes( context=context, - layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name, + layer_name=layer_name, + symbol_table=symbol_table, filter_func=filter_func, ): try: diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 8b9706625..bf676acb7 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -148,14 +148,17 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types=native_types, ) - def _get_service_key(self, kernel) -> Optional[objects.StructType]: + @classmethod + def _get_service_key( + cls, context, config_path: str, layer_name: str, symbol_table: str + ) -> Optional[objects.StructType]: for hive in hivelist.HiveList.list_hives( - context=self.context, + context=context, base_config_path=interfaces.configuration.path_join( - self.config_path, "hivelist" + config_path, "hivelist" ), - layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name, + layer_name=layer_name, + symbol_table=symbol_table, filter_string="machine\\system", ): # Get ControlSet\Services. @@ -273,7 +276,8 @@ class SvcScan(interfaces.plugins.PluginInterface): def service_scan( cls, context: interfaces.context.ContextInterface, - kernel, + layer_name: str, + symbol_table: str, service_table_name: str, service_binary_dll_map, filter_func, @@ -283,7 +287,7 @@ class SvcScan(interfaces.plugins.PluginInterface): ).relative_child_offset("Tag") is_vista_or_later = versions.is_vista_or_later( - context=context, symbol_table=kernel.symbol_table_name + context=context, symbol_table=symbol_table ) if is_vista_or_later: @@ -295,8 +299,8 @@ class SvcScan(interfaces.plugins.PluginInterface): for task in pslist.PsList.list_processes( context=context, - layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name, + layer_name=layer_name, + symbol_table=symbol_table, filter_func=filter_func, ): proc_id = "Unknown" @@ -348,36 +352,41 @@ class SvcScan(interfaces.plugins.PluginInterface): seen.append(service_record) yield service_record - def get_prereq_info(self): + @classmethod + def get_prereq_info(cls, context, config_path, layer_name: str, symbol_table: str): """ Data structures and information needed to analyze service information """ - kernel = self.context.modules[self.config["kernel"]] - service_table_name = self.create_service_table( - self.context, kernel.symbol_table_name, self.config_path + service_table_name = cls.create_service_table( + context, symbol_table, config_path ) - services_key = self._get_service_key(kernel) + services_key = cls._get_service_key( + context, config_path, layer_name, symbol_table + ) service_binary_dll_map = ( - self._get_service_binary_map(services_key) + cls._get_service_binary_map(services_key) if services_key is not None else {} ) filter_func = pslist.PsList.create_name_filter(["services.exe"]) - return kernel, service_table_name, service_binary_dll_map, filter_func + return service_table_name, service_binary_dll_map, filter_func def _generator(self): - kernel, service_table_name, service_binary_dll_map, filter_func = ( - self.get_prereq_info() + kernel = self.context.modules[self.config["kernel"]] + + service_table_name, service_binary_dll_map, filter_func = self.get_prereq_info( + self.context, self.config_path, kernel.layer_name, kernel.symbol_table_name ) for record in self._enumeration_method( self.context, - kernel, + kernel.layer_name, + kernel.symbol_table_name, service_table_name, service_binary_dll_map, filter_func, From 111873e173b1bc639754b851ce46ff5056f8edb8 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 21 Jul 2024 11:54:06 -0500 Subject: [PATCH 248/979] Fix class vs static method and leading underscores --- volatility3/framework/plugins/windows/svcscan.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index bf676acb7..52ed5e759 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -110,7 +110,7 @@ class SvcScan(interfaces.plugins.PluginInterface): ] @staticmethod - def create_service_table( + def _create_service_table( context: interfaces.context.ContextInterface, symbol_table: str, config_path: str, @@ -148,9 +148,9 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types=native_types, ) - @classmethod + @staticmethod def _get_service_key( - cls, context, config_path: str, layer_name: str, symbol_table: str + context, config_path: str, layer_name: str, symbol_table: str ) -> Optional[objects.StructType]: for hive in hivelist.HiveList.list_hives( context=context, @@ -358,7 +358,7 @@ class SvcScan(interfaces.plugins.PluginInterface): Data structures and information needed to analyze service information """ - service_table_name = cls.create_service_table( + service_table_name = cls._create_service_table( context, symbol_table, config_path ) From 5e96327cb0851fe75d1c7a4cb9ef27641bf119f7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 21 Jul 2024 22:58:43 +0100 Subject: [PATCH 249/979] Add in threads that only provides an implmentation method --- .../framework/plugins/windows/thrdscan.py | 32 +++++++++++------ .../framework/plugins/windows/threads.py | 34 +++++++++++++------ 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index bbf65cd6c..be7097347 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -3,7 +3,7 @@ ## import logging import datetime -from typing import Iterable +from typing import Callable, Iterable from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements @@ -21,6 +21,10 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) _required_framework_version = (2, 6, 0) _version = (1, 0, 0) + def __init__(self, *args, **kwargs): + self.implementation = self.scan_threads + super().__init__(*args, **kwargs) + @classmethod def get_requirements(cls): return [ @@ -38,8 +42,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) def scan_threads( cls, context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table: str, + module_name: str, ) -> Iterable[interfaces.objects.ObjectInterface]: """Scans for threads using the poolscanner module and constraints. @@ -52,6 +55,10 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) A list of _ETHREAD objects found by scanning memory for the "Thre" / "Thr\\xE5" pool signatures """ + module = context.modules[module_name] + layer_name = module.layer_name + symbol_table = module.symbol_table_name + constraints = poolscanner.PoolScanner.builtin_constraints( symbol_table, [b"Thr\xe5", b"Thre"] ) @@ -76,7 +83,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ethread.get_exit_time() ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object except exceptions.InvalidAddressException: - vollog.debug("Thread invalid address {:#x}".format(thread.vol.offset)) + vollog.debug("Thread invalid address {:#x}".format(ethread.vol.offset)) return None return ( @@ -88,12 +95,10 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) thread_exit_time, ) - def _generator(self): - kernel = self.context.modules[self.config["kernel"]] + def _generator(self, filter_func: Callable): + kernel_name = self.config["kernel"] - for ethread in self.scan_threads( - self.context, kernel.layer_name, kernel.symbol_table_name - ): + for ethread in self.implementation(self.context, kernel_name): info = self.gather_thread_info(ethread) if info: yield (0, info) @@ -126,7 +131,14 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) row_dict["ExitTime"], ) + @classmethod + def filter_func(cls, config: interfaces.configuration.HierarchicalDict) -> Callable: + """Returns a function that can filter this plugin's implementation method based on the config""" + return lambda x: False + def run(self): + filt_func = self.filter_func(self.config) + return renderers.TreeGrid( [ ("Offset", format_hints.Hex), @@ -136,5 +148,5 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime), ], - self._generator(), + self._generator(filt_func), ) diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index 83d231abb..55720c6f3 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -3,7 +3,7 @@ # import logging -from typing import List, Generator +from typing import Callable, Iterable, List, Generator from volatility3.framework import interfaces, constants from volatility3.framework.configuration import requirements @@ -18,6 +18,9 @@ class Threads(thrdscan.ThrdScan): _required_framework_version = (2, 4, 0) _version = (1, 0, 0) + def __init__(self): + self.implementation = self.list_process_threads + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements @@ -60,18 +63,27 @@ class Threads(thrdscan.ThrdScan): seen.add(thread.vol.offset) yield thread - def _generator(self): - kernel = self.context.modules[self.config["kernel"]] + @classmethod + def filter_func(cls, config: interfaces.configuration.HierarchicalDict) -> Callable: + return pslist.PsList.create_pid_filter(config.get("pid", None)) - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + @classmethod + def list_process_threads( + cls, + context: interfaces.context.ContextInterface, + module_name: str, + filter_func: Callable, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Runs through all processes and lists threads for each process""" + module = context.modules[module_name] + layer_name = module.layer_name + symbol_table_name = module.symbol_table_name for proc in pslist.PsList.list_processes( - context=self.context, - layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name, + context=context, + layer_name=layer_name, + symbol_table=symbol_table_name, filter_func=filter_func, ): - for thread in self.list_threads(kernel, proc): - info = self.gather_thread_info(thread) - if info: - yield (0, info) + for thread in cls.list_threads(module, proc): + yield thread From ce9832a2b3d524637a7bfaa49c7726ff234c121e Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Mon, 22 Jul 2024 13:58:33 +0200 Subject: [PATCH 250/979] both yara-python and yara-x support --- doc/requirements.txt | 1 + test/requirements-testing.txt | 1 + volatility3/framework/plugins/yarascan.py | 101 ++++++++++++++++++++-- 3 files changed, 95 insertions(+), 8 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index c04f71219..d3ba51224 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,6 +3,7 @@ sphinx>=4.0.0,<7 sphinx_autodoc_typehints>=1.4.0 sphinx-rtd-theme>=0.4.3 +yara-python yara-x pycryptodome pefile diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt index 0955d1939..51c8f602c 100644 --- a/test/requirements-testing.txt +++ b/test/requirements-testing.txt @@ -5,6 +5,7 @@ pefile>=2017.8.1 #foo # If certain packages are not necessary, place a comment (#) at the start of the line. # This is required for the yara plugins +yara-python>=3.8.0 yara-x>=0.5.0 pytest>=7.0.0 diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 32a400f9e..03a45b8df 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -13,14 +13,73 @@ from volatility3.framework.renderers import format_hints vollog = logging.getLogger(__name__) +USE_YARA_X = False + try: import yara_x + USE_YARA_X = True + except ImportError: - raise + try: + import yara + + if tuple(int(x) for x in yara.__version__.split(".")) < (3, 8): + raise ImportError + + vollog.info("Using yara-python module") + + except ImportError: + vollog.info( + "Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available" + ) + raise -class YaraScanner(interfaces.layers.ScannerInterface): +class YaraPythonScanner(interfaces.layers.ScannerInterface): + _version = (2, 0, 0) + + # yara.Rules isn't exposed, so we can't type this properly + def __init__(self, rules) -> None: + super().__init__() + if rules is None: + raise ValueError("No rules provided to YaraScanner") + self._rules = rules + self.st_object = not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) + + def __call__( + self, data: bytes, data_offset: int + ) -> Iterable[Tuple[int, str, str, bytes]]: + for match in self._rules.match(data=data): + if self.st_object: + for match_string in match.strings: + for instance in match_string.instances: + yield ( + instance.offset + data_offset, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield (offset + data_offset, match.rule, name, value) + + @staticmethod + def get_rule(rule): + return yara.compile( + sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} + ) + + @staticmethod + def from_compiled_file(filepath): + return yara.load(file=resources.ResourceAccessor().open(filepath, "rb")) + + @staticmethod + def from_file(filepath): + return yara.compile(file=resources.ResourceAccessor().open(filepath, "rb")) + + +class YaraXScanner(interfaces.layers.ScannerInterface): _version = (2, 0, 0) # yara.Rules isn't exposed, so we can't type this properly @@ -44,6 +103,25 @@ class YaraScanner(interfaces.layers.ScannerInterface): data[instance.offset : instance.offset + instance.length], ) + @staticmethod + def get_rule(rule): + return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") + + @staticmethod + def from_compiled_file(filepath): + return yara_x.Rules.deserialize_from( + file=resources.ResourceAccessor().open(filepath, "rb") + ) + + @staticmethod + def from_file(filepath): + return yara_x.compile( + resources.ResourceAccessor().open(filepath, "rb").read().decode() + ) + + +YaraScanner = YaraXScanner if USE_YARA_X else YaraPythonScanner + class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" @@ -88,9 +166,14 @@ class YaraScan(plugins.PluginInterface): description="Yara rules (as a string)", optional=True, ), + requirements.URIRequirement( + name="yara_file", + description="Yara rules (as a file)", + optional=True, + ), requirements.URIRequirement( name="yara_compiled_file", - description="Yara-x compiled rules (as a file)", + description="Yara compiled rules (as a file)", optional=True, ), requirements.IntRequirement( @@ -112,13 +195,15 @@ class YaraScan(plugins.PluginInterface): rule += " nocase" if config.get("wide", False): rule += " wide ascii" - rules = yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") + rules = YaraScanner.get_rule(rule) + elif config.get("yara_file") is not None: + vollog.debug(f"Plain file: {config["yara_file"]} - yara-x: {USE_YARA_X}") + rules = YaraScanner.from_file(config["yara_file"]) elif config.get("yara_compiled_file") is not None: - rules = yara_x.Rules.deserialize_from( - file=resources.ResourceAccessor().open( - config["yara_compiled_file"], "rb" - ) + vollog.debug( + f"Compiled file: {config["yara_compiled_file"]} - yara-x: {USE_YARA_X}" ) + rules = YaraScanner.from_compiled_file(config["yara_compiled_file"]) else: vollog.error("No yara rules, nor yara rules file were specified") return rules From af2d6206763a661ced9687f1c7b31b7888d3d0ce Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Tue, 23 Jul 2024 22:02:34 +0200 Subject: [PATCH 251/979] Improving lsof --- volatility3/framework/plugins/linux/lsof.py | 53 +++++++++++++++++-- .../framework/symbols/linux/__init__.py | 21 +++++++- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index d970ad8a9..f9aeafe14 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -3,7 +3,7 @@ # """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" -import logging +import logging, datetime from typing import List, Callable from volatility3.framework import renderers, interfaces, constants @@ -76,14 +76,59 @@ class Lsof(plugins.PluginInterface): ) for pid, task_comm, _task, fd_fields in fds_generator: - fd_num, _filp, full_path = fd_fields + ( + fd_num, + _filp, + full_path, + inode_num, + imode, + ctime, + mtime, + atime, + file_size, + ) = fd_fields - fields = (pid, task_comm, fd_num, full_path) + fields = ( + pid, + task_comm, + fd_num, + full_path, + inode_num, + imode, + ctime, + mtime, + atime, + file_size, + ) yield (0, fields) def run(self): pids = self.config.get("pid", None) symbol_table = self.config["kernel"] - tree_grid_args = [("PID", int), ("Process", str), ("FD", int), ("Path", str)] + tree_grid_args = [ + ("PID", int), + ("Process", str), + ("FD", int), + ("Path", str), + ("Inode", int), + ("Mode", str), + ("LastChange", datetime.datetime), + ("LastModify", datetime.datetime), + ("LastAccessed", datetime.datetime), + ("Size", int), + ] return renderers.TreeGrid(tree_grid_args, self._generator(pids, symbol_table)) + + def generate_timeline(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + for row in self._generator( + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ) + ): + _depth, row_data = row + description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[4]}"' + yield description, timeliner.TimeLinerType.CHANGED, row_data[5] + yield description, timeliner.TimeLinerType.MODIFIED, row_data[6] + yield description, timeliner.TimeLinerType.ACCESSED, row_data[7] diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index c4e2587f4..b9321f369 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,6 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import stat, datetime from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework @@ -265,8 +266,26 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): for fd_num, filp in enumerate(fds): if filp != 0: full_path = LinuxUtilities.path_for_file(context, task, filp) + dentry = filp.get_dentry() + if dentry != 0: + inode_object = dentry.d_inode + inode_num = inode_object.i_ino + file_size = inode_object.i_size # file size in bytes + imode = stat.filemode( + inode_object.i_mode + ) # file type & Permissions - yield fd_num, filp, full_path + # Timestamps + ctime = datetime.datetime.fromtimestamp( + inode_object.i_ctime.tv_sec + ) # last change time + mtime = datetime.datetime.fromtimestamp( + inode_object.i_mtime.tv_sec + ) # last modify time + atime = datetime.datetime.fromtimestamp( + inode_object.i_atime.tv_sec + ) # last access time + yield fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size @classmethod def mask_mods_list( From b8b146a4441f054bccf5697868c26250ea99d86d Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 24 Jul 2024 10:59:52 +0100 Subject: [PATCH 252/979] Windows: update handles plugin to use a default SAR value of 0x10 if decoding fails. Produce warnings when this happens. Ref issue #1147 --- volatility3/framework/plugins/windows/handles.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index a19e7a397..2edf28e86 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -25,7 +25,7 @@ class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -175,10 +175,11 @@ class Handles(interfaces.plugins.PluginInterface): virtual_layer_name, func_addr_to_read, num_bytes_to_read ) except exceptions.InvalidAddressException: - vollog.debug( - f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}" + vollog.warning( + f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of 0x10" ) - return None + self._sar_value = 0x10 + return self._sar_value md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) @@ -198,9 +199,10 @@ class Handles(interfaces.plugins.PluginInterface): break if self._sar_value is None: - vollog.debug( - f"Failed to to locate SAR value having parsed {instruction_count} instructions" + vollog.warning( + f"Failed to to locate SAR value having parsed {instruction_count} instructions, failing back to a common value of 0x10" ) + self._sar_value = 0x10 return self._sar_value From 7d52f7992d187e4c18425ca71171dc27a9710903 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 24 Jul 2024 11:06:26 +0100 Subject: [PATCH 253/979] Windows: Make the default sar value used in handles plugin a variable so if it needs to be changed it gets updated in one place only --- volatility3/framework/plugins/windows/handles.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 2edf28e86..abe154fa1 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -142,6 +142,7 @@ class Handles(interfaces.plugins.PluginInterface): pointers in the _HANDLE_TABLE_ENTRY which allows us to find the associated _OBJECT_HEADER. """ + DEFAULT_SAR_VALUE = 0x10 # to be used only when decoding fails if self._sar_value is None: if not has_capstone: @@ -178,7 +179,7 @@ class Handles(interfaces.plugins.PluginInterface): vollog.warning( f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of 0x10" ) - self._sar_value = 0x10 + self._sar_value = DEFAULT_SAR_VALUE return self._sar_value md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) @@ -202,7 +203,7 @@ class Handles(interfaces.plugins.PluginInterface): vollog.warning( f"Failed to to locate SAR value having parsed {instruction_count} instructions, failing back to a common value of 0x10" ) - self._sar_value = 0x10 + self._sar_value = DEFAULT_SAR_VALUE return self._sar_value From e1065f9e788322faded83520da29cd5cbcd5d5ed Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Jul 2024 20:32:36 +0100 Subject: [PATCH 254/979] Fix up a missing super which @atcuno spotted --- volatility3/framework/plugins/windows/threads.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index 55720c6f3..d57911650 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -20,6 +20,7 @@ class Threads(thrdscan.ThrdScan): def __init__(self): self.implementation = self.list_process_threads + super().__init__() @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From f4cbed856bcfb011707588d5563cd7144450ba1e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Jul 2024 20:47:05 +0100 Subject: [PATCH 255/979] Fix up missing filter parameter --- volatility3/framework/plugins/windows/thrdscan.py | 7 +++++-- volatility3/framework/plugins/windows/threads.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index be7097347..6e664e052 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -19,7 +19,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) # version 2.6.0 adds support for scanning for 'Ethread' structures by pool tags _required_framework_version = (2, 6, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) def __init__(self, *args, **kwargs): self.implementation = self.scan_threads @@ -100,11 +100,14 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) for ethread in self.implementation(self.context, kernel_name): info = self.gather_thread_info(ethread) + if info: yield (0, info) def generate_timeline(self): - for row in self._generator(): + filt_func = self.filter_func(self.config) + + for row in self._generator(filt_func): _depth, row_data = row row_dict = {} ( diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index d57911650..ae70e717b 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -38,7 +38,7 @@ class Threads(thrdscan.ThrdScan): optional=True, ), requirements.PluginRequirement( - name="thrdscan", plugin=thrdscan.ThrdScan, version=(1, 0, 0) + name="thrdscan", plugin=thrdscan.ThrdScan, version=(1, 1, 0) ), ] From fceb79ba8b86e95919451ee29ca9fdfccd3ebc3e Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 08:34:04 +0200 Subject: [PATCH 256/979] use with context, bump release --- volatility3/framework/plugins/yarascan.py | 29 +++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 03a45b8df..7957aa0c5 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -27,11 +27,11 @@ except ImportError: if tuple(int(x) for x in yara.__version__.split(".")) < (3, 8): raise ImportError - vollog.info("Using yara-python module") + vollog.debug("Using yara-python module") except ImportError: vollog.info( - "Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available" + "Neither yara-x nor yara-python (>3.8.0) module not found, plugin (and dependent plugins) not available" ) raise @@ -51,7 +51,7 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: for match in self._rules.match(data=data): - if self.st_object: + if YaraScan.yara_returns_instances(): for match_string in match.strings: for instance in match_string.instances: yield ( @@ -72,11 +72,13 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): @staticmethod def from_compiled_file(filepath): - return yara.load(file=resources.ResourceAccessor().open(filepath, "rb")) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara.load(file=fp) @staticmethod def from_file(filepath): - return yara.compile(file=resources.ResourceAccessor().open(filepath, "rb")) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara.compile(file=fp) class YaraXScanner(interfaces.layers.ScannerInterface): @@ -109,15 +111,13 @@ class YaraXScanner(interfaces.layers.ScannerInterface): @staticmethod def from_compiled_file(filepath): - return yara_x.Rules.deserialize_from( - file=resources.ResourceAccessor().open(filepath, "rb") - ) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara_x.Rules.deserialize_from(file=fp) @staticmethod def from_file(filepath): - return yara_x.compile( - resources.ResourceAccessor().open(filepath, "rb").read().decode() - ) + with resources.ResourceAccessor().open(filepath, "rb") as fp: + return yara_x.compile(fp.read().decode()) YaraScanner = YaraXScanner if USE_YARA_X else YaraPythonScanner @@ -127,7 +127,7 @@ class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" _required_framework_version = (2, 0, 0) - _version = (1, 3, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -184,6 +184,11 @@ class YaraScan(plugins.PluginInterface): ), ] + @classmethod + def yara_returns_instances(cls) -> bool: + st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < (4, 3) + return st_object + @classmethod def process_yara_options(cls, config: Dict[str, Any]): rules = None From 1616a898ae5ddfe7369c0ea90306546abd4d929e Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 08:37:27 +0200 Subject: [PATCH 257/979] restore comment --- volatility3/framework/plugins/yarascan.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 7957aa0c5..3bf894d06 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -171,6 +171,8 @@ class YaraScan(plugins.PluginInterface): description="Yara rules (as a file)", optional=True, ), + # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code + # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later requirements.URIRequirement( name="yara_compiled_file", description="Yara compiled rules (as a file)", From c6727ffb701d31067e6617d49904614bedb63ac9 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 08:43:21 +0200 Subject: [PATCH 258/979] fix f-string for previous python release --- volatility3/framework/plugins/yarascan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 3bf894d06..8d2f242d9 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -204,11 +204,11 @@ class YaraScan(plugins.PluginInterface): rule += " wide ascii" rules = YaraScanner.get_rule(rule) elif config.get("yara_file") is not None: - vollog.debug(f"Plain file: {config["yara_file"]} - yara-x: {USE_YARA_X}") + vollog.debug(f"Plain file: {config['yara_file']} - yara-x: {USE_YARA_X}") rules = YaraScanner.from_file(config["yara_file"]) elif config.get("yara_compiled_file") is not None: vollog.debug( - f"Compiled file: {config["yara_compiled_file"]} - yara-x: {USE_YARA_X}" + f"Compiled file: {config['yara_compiled_file']} - yara-x: {USE_YARA_X}" ) rules = YaraScanner.from_compiled_file(config["yara_compiled_file"]) else: From dc8dc9b078d42e41698ff93f2e65c5ee99aa2172 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 25 Jul 2024 14:37:18 +0200 Subject: [PATCH 259/979] fix yara depending plugins --- .../framework/plugins/linux/vmayarascan.py | 4 +- .../framework/plugins/windows/mftscan.py | 6 +-- .../framework/plugins/windows/vadyarascan.py | 49 +++++++++++++------ volatility3/framework/plugins/yarascan.py | 24 ++++----- 4 files changed, 48 insertions(+), 35 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index eda0d7dca..6d055707c 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -31,10 +31,10 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.PluginRequirement( - name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) ), requirements.ModuleRequirement( name="kernel", diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 1b1a1b45e..9e6585345 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -29,7 +29,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) ), ] @@ -38,7 +38,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\\*|BAAD/"} + {"yara_string": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File @@ -197,7 +197,7 @@ class ADS(interfaces.plugins.PluginInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\\*|BAAD/"} + {"yara_string": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index dc318dd93..4a84a1285 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -33,7 +33,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.PluginRequirement( - name="yarascan", plugin=yarascan.YaraScan, version=(1, 3, 0) + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), requirements.ListRequirement( name="pid", @@ -73,26 +73,43 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ) continue - for match in rules.match(data=layer.read(start, size, True)): - if yarascan.YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: + if not yarascan.YaraScan._yara_x: + for match in rules.match(data=layer.read(start, size, True)): + if yarascan.YaraScan.yara_returns_instances(): + for match_string in match.strings: + for instance in match_string.instances: + yield 0, ( + format_hints.Hex(instance.offset + start), + task.UniqueProcessId, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield 0, ( + format_hints.Hex(offset + start), + task.UniqueProcessId, + match.rule, + name, + value, + ) + else: + data = layer.read(start, size, True) + results = rules.scan(data) + for match in results.matching_rules: + for match_string in match.patterns: + for instance in match_string.matches: yield 0, ( format_hints.Hex(instance.offset + start), task.UniqueProcessId, - match.rule, + f"{match.namespace}.{match.identifier}", match_string.identifier, - instance.matched_data, + data[ + instance.offset : instance.offset + + instance.length + ], ) - else: - for offset, name, value in match.strings: - yield 0, ( - format_hints.Hex(offset + start), - task.UniqueProcessId, - match.rule, - name, - value, - ) @staticmethod def get_vad_maps( diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 8d2f242d9..6a4dd9251 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -36,8 +36,8 @@ except ImportError: raise -class YaraPythonScanner(interfaces.layers.ScannerInterface): - _version = (2, 0, 0) +class BaseYaraScanner(interfaces.layers.ScannerInterface): + _version = (2, 1, 0) # yara.Rules isn't exposed, so we can't type this properly def __init__(self, rules) -> None: @@ -45,6 +45,11 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): if rules is None: raise ValueError("No rules provided to YaraScanner") self._rules = rules + + +class YaraPythonScanner(BaseYaraScanner): + def __init__(self, rules) -> None: + super().__init__(rules) self.st_object = not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) def __call__( @@ -81,16 +86,7 @@ class YaraPythonScanner(interfaces.layers.ScannerInterface): return yara.compile(file=fp) -class YaraXScanner(interfaces.layers.ScannerInterface): - _version = (2, 0, 0) - - # yara.Rules isn't exposed, so we can't type this properly - def __init__(self, rules) -> None: - super().__init__() - if rules is None: - raise ValueError("No rules provided to YaraScanner") - self._rules = rules - +class YaraXScanner(BaseYaraScanner): def __call__( self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: @@ -128,6 +124,7 @@ class YaraScan(plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (2, 0, 0) + _yara_x = USE_YARA_X @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -188,8 +185,7 @@ class YaraScan(plugins.PluginInterface): @classmethod def yara_returns_instances(cls) -> bool: - st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < (4, 3) - return st_object + return not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) @classmethod def process_yara_options(cls, config: Dict[str, Any]): From 6f3f645dbcfb94b797f2e244e56e43dd8e4cba0a Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 25 Jul 2024 17:59:10 +0100 Subject: [PATCH 260/979] Windows: update handles plugin sar warnings to use DEFAULT_SAR_VALUE var --- volatility3/framework/plugins/windows/handles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index abe154fa1..6010b4c72 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -177,7 +177,7 @@ class Handles(interfaces.plugins.PluginInterface): ) except exceptions.InvalidAddressException: vollog.warning( - f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of 0x10" + f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of {hex(DEFAULT_SAR_VALUE)}}" ) self._sar_value = DEFAULT_SAR_VALUE return self._sar_value @@ -201,7 +201,7 @@ class Handles(interfaces.plugins.PluginInterface): if self._sar_value is None: vollog.warning( - f"Failed to to locate SAR value having parsed {instruction_count} instructions, failing back to a common value of 0x10" + f"Failed to to locate SAR value having parsed {instruction_count} instructions, failing back to a common value of {hex(DEFAULT_SAR_VALUE)}" ) self._sar_value = DEFAULT_SAR_VALUE From 799afe6e51558dc4fe3828276f28b888e866e708 Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 25 Jul 2024 18:02:30 +0100 Subject: [PATCH 261/979] Windows: fix type in handles plugin --- volatility3/framework/plugins/windows/handles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 6010b4c72..3e5a2fd82 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -177,7 +177,7 @@ class Handles(interfaces.plugins.PluginInterface): ) except exceptions.InvalidAddressException: vollog.warning( - f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of {hex(DEFAULT_SAR_VALUE)}}" + f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of {hex(DEFAULT_SAR_VALUE)}" ) self._sar_value = DEFAULT_SAR_VALUE return self._sar_value From 7a03e9deabc6f743eb8e201fe75dcdd810339635 Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 25 Jul 2024 18:21:12 +0100 Subject: [PATCH 262/979] Windows: Add a _version to the filescan plugin --- volatility3/framework/plugins/windows/filescan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/filescan.py b/volatility3/framework/plugins/windows/filescan.py index 34c0c60d3..82566361d 100644 --- a/volatility3/framework/plugins/windows/filescan.py +++ b/volatility3/framework/plugins/windows/filescan.py @@ -14,6 +14,7 @@ class FileScan(interfaces.plugins.PluginInterface): """Scans for file objects present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): From 650dd06245918f1b14d8477eff85a743c9e42c4f Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 27 Jul 2024 16:03:59 +0200 Subject: [PATCH 263/979] Modifications following the review --- volatility3/framework/plugins/linux/lsof.py | 74 +++++++++++-------- .../framework/symbols/linux/__init__.py | 51 +++++++------ 2 files changed, 70 insertions(+), 55 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index f9aeafe14..3bbc855f9 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # """A module containing a collection of plugins that produce data typically @@ -12,16 +12,17 @@ from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.framework.symbols import linux from volatility3.plugins.linux import pslist +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class Lsof(plugins.PluginInterface): +class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -46,7 +47,7 @@ class Lsof(plugins.PluginInterface): ] @classmethod - def list_fds( + def list_fds_and_inodes( cls, context: interfaces.context.ContextInterface, symbol_table: str, @@ -67,27 +68,38 @@ class Lsof(plugins.PluginInterface): ) for fd_fields in fd_generator: - yield pid, task_comm, task, fd_fields + fd_num, filp, full_path = fd_fields + inode_metadata = linux.LinuxUtilities.get_inode_metadata(context, filp) + try: + inode_num, file_size, imode, ctime, mtime, atime = next( + inode_metadata + ) + except Exception as e: + vollog.warning( + f"Can't get inode metadata for file descriptor {fd_num}: {e}" + ) + continue + yield pid, task_comm, task, fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size def _generator(self, pids, symbol_table): filter_func = pslist.PsList.create_pid_filter(pids) - fds_generator = self.list_fds( + fds_generator = self.list_fds_and_inodes( self.context, symbol_table, filter_func=filter_func ) - - for pid, task_comm, _task, fd_fields in fds_generator: - ( - fd_num, - _filp, - full_path, - inode_num, - imode, - ctime, - mtime, - atime, - file_size, - ) = fd_fields - + for ( + pid, + task_comm, + task, + fd_num, + filp, + full_path, + inode_num, + imode, + ctime, + mtime, + atime, + file_size, + ) in fds_generator: fields = ( pid, task_comm, @@ -113,22 +125,20 @@ class Lsof(plugins.PluginInterface): ("Path", str), ("Inode", int), ("Mode", str), - ("LastChange", datetime.datetime), - ("LastModify", datetime.datetime), - ("LastAccessed", datetime.datetime), + ("Changed", datetime.datetime), + ("Modified", datetime.datetime), + ("Accessed", datetime.datetime), ("Size", int), ] return renderers.TreeGrid(tree_grid_args, self._generator(pids, symbol_table)) def generate_timeline(self): + pids = self.config.get("pid", None) + symbol_table = self.config["kernel"] filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - for row in self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ): + for row in self._generator(pids, symbol_table): _depth, row_data = row - description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[4]}"' - yield description, timeliner.TimeLinerType.CHANGED, row_data[5] - yield description, timeliner.TimeLinerType.MODIFIED, row_data[6] - yield description, timeliner.TimeLinerType.ACCESSED, row_data[7] + description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[3]}"' + yield description, timeliner.TimeLinerType.CHANGED, row_data[6] + yield description, timeliner.TimeLinerType.MODIFIED, row_data[7] + yield description, timeliner.TimeLinerType.ACCESSED, row_data[8] diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index b9321f369..2b97bc5f4 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,9 +1,8 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import stat, datetime from typing import Iterator, List, Tuple, Optional, Union - +import logging, datetime, stat from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility @@ -62,7 +61,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 1, 0) + _version = (2, 2, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -266,26 +265,32 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): for fd_num, filp in enumerate(fds): if filp != 0: full_path = LinuxUtilities.path_for_file(context, task, filp) - dentry = filp.get_dentry() - if dentry != 0: - inode_object = dentry.d_inode - inode_num = inode_object.i_ino - file_size = inode_object.i_size # file size in bytes - imode = stat.filemode( - inode_object.i_mode - ) # file type & Permissions - # Timestamps - ctime = datetime.datetime.fromtimestamp( - inode_object.i_ctime.tv_sec - ) # last change time - mtime = datetime.datetime.fromtimestamp( - inode_object.i_mtime.tv_sec - ) # last modify time - atime = datetime.datetime.fromtimestamp( - inode_object.i_atime.tv_sec - ) # last access time - yield fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size + yield fd_num, filp, full_path + + @classmethod + def get_inode_metadata(cls, context: interfaces.context.ContextInterface, filp): + """ + A helper function that gets the inodes metadata from a file descriptor + """ + dentry = filp.get_dentry() + if dentry != 0: + inode_object = dentry.d_inode + inode_num = inode_object.i_ino + file_size = inode_object.i_size # file size in bytes + imode = stat.filemode(inode_object.i_mode) # file type & Permissions + + # Timestamps + ctime = datetime.datetime.fromtimestamp( + inode_object.i_ctime.tv_sec + ) # last change time + mtime = datetime.datetime.fromtimestamp( + inode_object.i_mtime.tv_sec + ) # last modify time + atime = datetime.datetime.fromtimestamp( + inode_object.i_atime.tv_sec + ) # last access time + yield inode_num, file_size, imode, ctime, mtime, atime @classmethod def mask_mods_list( From 7024588076adf95c2d6667c851cddf87e8c68555 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 27 Jul 2024 16:09:52 +0200 Subject: [PATCH 264/979] Code clean --- volatility3/framework/plugins/linux/lsof.py | 1 - volatility3/framework/symbols/linux/__init__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 3bbc855f9..98a215ecf 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -135,7 +135,6 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): def generate_timeline(self): pids = self.config.get("pid", None) symbol_table = self.config["kernel"] - filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) for row in self._generator(pids, symbol_table): _depth, row_data = row description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[3]}"' diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 2b97bc5f4..1b3f75e98 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # from typing import Iterator, List, Tuple, Optional, Union -import logging, datetime, stat +import datetime, stat from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility From 815252c9ba31913d22e8836183828059217c4e9d Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 27 Jul 2024 16:35:22 +0200 Subject: [PATCH 265/979] Adding watchdogs --- volatility3/framework/plugins/linux/lsof.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 98a215ecf..c1de48c1a 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -78,7 +78,13 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): vollog.warning( f"Can't get inode metadata for file descriptor {fd_num}: {e}" ) - continue + # Yield NotAvailableValue for each field in case of an exception + inode_num = renderers.NotAvailableValue() + file_size = renderers.NotAvailableValue() + imode = renderers.NotAvailableValue() + ctime = renderers.NotAvailableValue() + mtime = renderers.NotAvailableValue() + atime = renderers.NotAvailableValue() yield pid, task_comm, task, fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size def _generator(self, pids, symbol_table): From f44ceb321d0c3b10249c1c697b3fb0c40f38ba39 Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Sat, 27 Jul 2024 20:11:39 -0500 Subject: [PATCH 266/979] Added psxview --- .../framework/plugins/windows/psxview.py | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 volatility3/framework/plugins/windows/psxview.py diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py new file mode 100644 index 000000000..3bf6c27f5 --- /dev/null +++ b/volatility3/framework/plugins/windows/psxview.py @@ -0,0 +1,188 @@ +import datetime, logging + +from volatility3.framework import constants, exceptions +from volatility3.framework.interfaces import plugins +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, TreeGrid +from volatility3.plugins.windows import handles, info, pslist, psscan, sessions, thrdscan + +vollog = logging.getLogger(__name__) + +class PsXView(plugins.PluginInterface): + """Lists all processes found via 6 of the methods described in \"The Art of Memory Forensics,\" which may help + identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this + plugin's output in a terminal.""" + # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality + # which the original plugin to do it. + + # I don't think it's worth including the sessions method either because both the original psxview plugin + # and Volatility3's sessions plugin begin with the list of processes found by PsList. + # The original psxview plugin's session code essentially just filters the pslist for processes + # whose session ID is not None. I've matched this in my code, but again, it doesn't seem worth including. + + # Lastly, I've omitted the pspcid method because I could not for the life of me get it to work. I saved the + # code I do have from it, and will happily share it if anyone else wants to add it. + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [requirements.ModuleRequirement(name="kernel", description="Windows kernel", architectures=["Intel32", "Intel64"]), + requirements.VersionRequirement(name="info", component=info.Info, version=(1, 0, 0)), + requirements.VersionRequirement(name="pslist", component=pslist.PsList, version=(2, 0, 0)), + requirements.VersionRequirement(name="psscan", component=psscan.PsScan, version=(1, 0, 0)), + requirements.VersionRequirement(name="thrdscan", component=thrdscan.ThrdScan, version=(1, 0, 0)), + requirements.VersionRequirement(name="handles", component=handles.Handles, version=(1, 0, 0)), + requirements.VersionRequirement(name="sessions", component=sessions.Sessions, version=(0, 0, 0)), + requirements.BooleanRequirement(name="identify-expected", description="In the plugin's output, replace false with \ + normal where false is the expected result for a Windows machine running normally. \ + Keep in mind that this plugin uses simple checks to identify \"normal\" behavior, \ + so you may want to double-check the legitimacy of these processes yourself.", optional=True), + requirements.BooleanRequirement(name="physical-offsets", description="List processes with phyiscall offsets instead of virtual offsets.", optional=True)] + + def proc_name_to_string(self, proc): + return proc.ImageFileName.cast("string", max_length=proc.ImageFileName.vol.count, errors="replace") + + def is_ascii(self, str): + return str.split('.')[0].isalnum() + + def filter_garbage_procs(self, proc_list): + return [p for p in proc_list if p.is_valid() and self.is_ascii(self.proc_name_to_string(p))] + + def translate_offset(self, offset): + if self.config["physical-offsets"]: + return offset + + kernel = self.context.modules[self.config["kernel"]] + layer_name = kernel.layer_name + + try: + offset = list(self.context.layers[layer_name].mapping(offset=offset, length=0))[0][2] + except: + # already have physical address + pass + + return offset + + def proc_list_to_dict(self, tasks): + return {self.translate_offset(proc.vol.offset):proc for proc in tasks} + + def check_pslist(self, tasks): + res = self.filter_garbage_procs(tasks) + return self.proc_list_to_dict(tasks) + + def check_psscan(self, layer_name, symbol_table): + res = psscan.PsScan.scan_processes(context=self.context, layer_name=layer_name, symbol_table=symbol_table) + res = self.filter_garbage_procs(res) + + return self.proc_list_to_dict(res) + + def check_thrdscan(self): + ret = [] + + for ethread in thrdscan.ThrdScan.scan_threads(self.context, module_name='kernel'): + process = None + try: + process = ethread.owning_process() + if not process.is_valid(): + continue + + ret.append(process) + except AttributeError: + vollog.log(constants.LOGLEVEL_VVV, "Unable to find the owning process of ethread") + + return self.proc_list_to_dict(ret) + + def check_csrss_handles(self, tasks, layer_name, symbol_table): + ret = [] + + for p in tasks: + name = self.proc_name_to_string(p) + if name == 'csrss.exe': + try: + if p.has_member("ObjectTable"): + handles_plugin = handles.Handles(context=self.context, config_path=self.config_path) + hndls = list(handles_plugin.handles(p.ObjectTable)) + for h in hndls: + if (h.get_object_type(handles_plugin.get_type_map(self.context, layer_name, symbol_table)) == "Process"): + ret.append(h.Body.cast("_EPROCESS")) + + except exceptions.InvalidAddressException: + vollog.log(constants.LOGLEVEL_VVV, "Cannot access eprocess object table") + + ret = self.filter_garbage_procs(ret) + return self.proc_list_to_dict(ret) + + def check_session(self, pslist_procs): + procs = [p for p in pslist_procs if p.get_session_id() != None] + + return self.proc_list_to_dict(procs) + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + layer_name = kernel.layer_name + symbol_table = kernel.symbol_table_name + + kdbg_list_processes = list(pslist.PsList.list_processes(context=self.context, layer_name=layer_name, symbol_table=symbol_table)) + + processes = {} + + processes['pslist'] = self.check_pslist(kdbg_list_processes) + processes['psscan'] = self.check_psscan(layer_name, symbol_table) + processes['thrdscan'] = self.check_thrdscan() + processes['csrss'] = self.check_csrss_handles(kdbg_list_processes, layer_name, symbol_table) + processes['sessions'] = self.check_session(kdbg_list_processes) + + seen_offsets = set() + for source in processes: + for offset in processes[source]: + if offset not in seen_offsets: + seen_offsets.add(offset) + proc = processes[source][offset] + + pid = proc.UniqueProcessId + name = self.proc_name_to_string(proc) + + exit_time = proc.get_exit_time() + if (type(exit_time) != datetime.datetime): + exit_time = "" + else: + exit_time = str(exit_time) + + in_sources = {src:str(offset in processes[src]) for src in processes} + + if self.config["identify-expected"]: + f = "False" + n = "Normal" + + if in_sources["pslist"] == f: + if exit_time != "": + in_sources["pslist"] = n + + if in_sources["thrdscan"] == f: + if exit_time != "": + in_sources["thrdscan"] = n + + if in_sources["csrss"] == f: + if name.lower() in ["system", "smss.exe", "csrss.exe"]: + in_sources["csrss"] = n + elif exit_time != "": + in_sources["csrss"] = n + + if in_sources["sessions"] == f: + if name.lower() in ["system", "smss.exe"]: + in_sources["sessions"] = n + + yield (0, (format_hints.Hex(offset), name, pid, in_sources["pslist"], + in_sources["psscan"], in_sources["thrdscan"], in_sources["csrss"], + in_sources["sessions"], exit_time)) + + + def run(self): + offset_type = "(Physical)" if self.config["physical-offsets"] else "(Virtual)" + offset_str = "Offset" + offset_type + + return TreeGrid([(offset_str, format_hints.Hex), ("Name", str), ("PID", int), ("pslist", str), ("psscan", str), + ("thrdscan", str), ("csrss", str), ("sessions", str), ("Exit Time", str) ], self._generator()) \ No newline at end of file From 824b0599f20d2a1f0e7f16ca08f9b498898e4c52 Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Sat, 27 Jul 2024 20:35:08 -0500 Subject: [PATCH 267/979] formatted --- .../framework/plugins/windows/psxview.py | 203 +++++++++++++----- 1 file changed, 147 insertions(+), 56 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 3bf6c27f5..1fde5b12d 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -4,19 +4,28 @@ from volatility3.framework import constants, exceptions from volatility3.framework.interfaces import plugins from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints, TreeGrid -from volatility3.plugins.windows import handles, info, pslist, psscan, sessions, thrdscan +from volatility3.plugins.windows import ( + handles, + info, + pslist, + psscan, + sessions, + thrdscan, +) vollog = logging.getLogger(__name__) + class PsXView(plugins.PluginInterface): - """Lists all processes found via 6 of the methods described in \"The Art of Memory Forensics,\" which may help + """Lists all processes found via 6 of the methods described in \"The Art of Memory Forensics,\" which may help identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this plugin's output in a terminal.""" - # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality - # which the original plugin to do it. - # I don't think it's worth including the sessions method either because both the original psxview plugin - # and Volatility3's sessions plugin begin with the list of processes found by PsList. + # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality + # which the original plugin to do it. + + # I don't think it's worth including the sessions method either because both the original psxview plugin + # and Volatility3's sessions plugin begin with the list of processes found by PsList. # The original psxview plugin's session code essentially just filters the pslist for processes # whose session ID is not None. I've matched this in my code, but again, it doesn't seem worth including. @@ -28,52 +37,88 @@ class PsXView(plugins.PluginInterface): @classmethod def get_requirements(cls): - return [requirements.ModuleRequirement(name="kernel", description="Windows kernel", architectures=["Intel32", "Intel64"]), - requirements.VersionRequirement(name="info", component=info.Info, version=(1, 0, 0)), - requirements.VersionRequirement(name="pslist", component=pslist.PsList, version=(2, 0, 0)), - requirements.VersionRequirement(name="psscan", component=psscan.PsScan, version=(1, 0, 0)), - requirements.VersionRequirement(name="thrdscan", component=thrdscan.ThrdScan, version=(1, 0, 0)), - requirements.VersionRequirement(name="handles", component=handles.Handles, version=(1, 0, 0)), - requirements.VersionRequirement(name="sessions", component=sessions.Sessions, version=(0, 0, 0)), - requirements.BooleanRequirement(name="identify-expected", description="In the plugin's output, replace false with \ + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="info", component=info.Info, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="psscan", component=psscan.PsScan, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="thrdscan", component=thrdscan.ThrdScan, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="handles", component=handles.Handles, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="sessions", component=sessions.Sessions, version=(0, 0, 0) + ), + requirements.BooleanRequirement( + name="identify-expected", + description='In the plugin\'s output, replace false with \ normal where false is the expected result for a Windows machine running normally. \ - Keep in mind that this plugin uses simple checks to identify \"normal\" behavior, \ - so you may want to double-check the legitimacy of these processes yourself.", optional=True), - requirements.BooleanRequirement(name="physical-offsets", description="List processes with phyiscall offsets instead of virtual offsets.", optional=True)] - + Keep in mind that this plugin uses simple checks to identify "normal" behavior, \ + so you may want to double-check the legitimacy of these processes yourself.', + optional=True, + ), + requirements.BooleanRequirement( + name="physical-offsets", + description="List processes with phyiscall offsets instead of virtual offsets.", + optional=True, + ), + ] + def proc_name_to_string(self, proc): - return proc.ImageFileName.cast("string", max_length=proc.ImageFileName.vol.count, errors="replace") + return proc.ImageFileName.cast( + "string", max_length=proc.ImageFileName.vol.count, errors="replace" + ) def is_ascii(self, str): - return str.split('.')[0].isalnum() - + return str.split(".")[0].isalnum() + def filter_garbage_procs(self, proc_list): - return [p for p in proc_list if p.is_valid() and self.is_ascii(self.proc_name_to_string(p))] - + return [ + p + for p in proc_list + if p.is_valid() and self.is_ascii(self.proc_name_to_string(p)) + ] + def translate_offset(self, offset): if self.config["physical-offsets"]: return offset - + kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name try: - offset = list(self.context.layers[layer_name].mapping(offset=offset, length=0))[0][2] + offset = list( + self.context.layers[layer_name].mapping(offset=offset, length=0) + )[0][2] except: # already have physical address pass return offset - + def proc_list_to_dict(self, tasks): - return {self.translate_offset(proc.vol.offset):proc for proc in tasks} - + return {self.translate_offset(proc.vol.offset): proc for proc in tasks} + def check_pslist(self, tasks): res = self.filter_garbage_procs(tasks) return self.proc_list_to_dict(tasks) - + def check_psscan(self, layer_name, symbol_table): - res = psscan.PsScan.scan_processes(context=self.context, layer_name=layer_name, symbol_table=symbol_table) + res = psscan.PsScan.scan_processes( + context=self.context, layer_name=layer_name, symbol_table=symbol_table + ) res = self.filter_garbage_procs(res) return self.proc_list_to_dict(res) @@ -81,7 +126,9 @@ class PsXView(plugins.PluginInterface): def check_thrdscan(self): ret = [] - for ethread in thrdscan.ThrdScan.scan_threads(self.context, module_name='kernel'): + for ethread in thrdscan.ThrdScan.scan_threads( + self.context, module_name="kernel" + ): process = None try: process = ethread.owning_process() @@ -90,50 +137,70 @@ class PsXView(plugins.PluginInterface): ret.append(process) except AttributeError: - vollog.log(constants.LOGLEVEL_VVV, "Unable to find the owning process of ethread") + vollog.log( + constants.LOGLEVEL_VVV, + "Unable to find the owning process of ethread", + ) return self.proc_list_to_dict(ret) - + def check_csrss_handles(self, tasks, layer_name, symbol_table): ret = [] for p in tasks: name = self.proc_name_to_string(p) - if name == 'csrss.exe': + if name == "csrss.exe": try: if p.has_member("ObjectTable"): - handles_plugin = handles.Handles(context=self.context, config_path=self.config_path) + handles_plugin = handles.Handles( + context=self.context, config_path=self.config_path + ) hndls = list(handles_plugin.handles(p.ObjectTable)) for h in hndls: - if (h.get_object_type(handles_plugin.get_type_map(self.context, layer_name, symbol_table)) == "Process"): + if ( + h.get_object_type( + handles_plugin.get_type_map( + self.context, layer_name, symbol_table + ) + ) + == "Process" + ): ret.append(h.Body.cast("_EPROCESS")) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, "Cannot access eprocess object table") + vollog.log( + constants.LOGLEVEL_VVV, "Cannot access eprocess object table" + ) ret = self.filter_garbage_procs(ret) return self.proc_list_to_dict(ret) def check_session(self, pslist_procs): procs = [p for p in pslist_procs if p.get_session_id() != None] - + return self.proc_list_to_dict(procs) def _generator(self): kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name - symbol_table = kernel.symbol_table_name + symbol_table = kernel.symbol_table_name + + kdbg_list_processes = list( + pslist.PsList.list_processes( + context=self.context, layer_name=layer_name, symbol_table=symbol_table + ) + ) - kdbg_list_processes = list(pslist.PsList.list_processes(context=self.context, layer_name=layer_name, symbol_table=symbol_table)) - processes = {} - processes['pslist'] = self.check_pslist(kdbg_list_processes) - processes['psscan'] = self.check_psscan(layer_name, symbol_table) - processes['thrdscan'] = self.check_thrdscan() - processes['csrss'] = self.check_csrss_handles(kdbg_list_processes, layer_name, symbol_table) - processes['sessions'] = self.check_session(kdbg_list_processes) + processes["pslist"] = self.check_pslist(kdbg_list_processes) + processes["psscan"] = self.check_psscan(layer_name, symbol_table) + processes["thrdscan"] = self.check_thrdscan() + processes["csrss"] = self.check_csrss_handles( + kdbg_list_processes, layer_name, symbol_table + ) + processes["sessions"] = self.check_session(kdbg_list_processes) seen_offsets = set() for source in processes: @@ -146,12 +213,14 @@ class PsXView(plugins.PluginInterface): name = self.proc_name_to_string(proc) exit_time = proc.get_exit_time() - if (type(exit_time) != datetime.datetime): + if type(exit_time) != datetime.datetime: exit_time = "" else: exit_time = str(exit_time) - in_sources = {src:str(offset in processes[src]) for src in processes} + in_sources = { + src: str(offset in processes[src]) for src in processes + } if self.config["identify-expected"]: f = "False" @@ -173,16 +242,38 @@ class PsXView(plugins.PluginInterface): if in_sources["sessions"] == f: if name.lower() in ["system", "smss.exe"]: - in_sources["sessions"] = n + in_sources["sessions"] = n + + yield ( + 0, + ( + format_hints.Hex(offset), + name, + pid, + in_sources["pslist"], + in_sources["psscan"], + in_sources["thrdscan"], + in_sources["csrss"], + in_sources["sessions"], + exit_time, + ), + ) - yield (0, (format_hints.Hex(offset), name, pid, in_sources["pslist"], - in_sources["psscan"], in_sources["thrdscan"], in_sources["csrss"], - in_sources["sessions"], exit_time)) - - def run(self): offset_type = "(Physical)" if self.config["physical-offsets"] else "(Virtual)" offset_str = "Offset" + offset_type - return TreeGrid([(offset_str, format_hints.Hex), ("Name", str), ("PID", int), ("pslist", str), ("psscan", str), - ("thrdscan", str), ("csrss", str), ("sessions", str), ("Exit Time", str) ], self._generator()) \ No newline at end of file + return TreeGrid( + [ + (offset_str, format_hints.Hex), + ("Name", str), + ("PID", int), + ("pslist", str), + ("psscan", str), + ("thrdscan", str), + ("csrss", str), + ("sessions", str), + ("Exit Time", str), + ], + self._generator(), + ) From 44f26c928eddaaf6743ac07b22331ad082e86bc1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Jul 2024 12:35:02 +0100 Subject: [PATCH 268/979] Add in shtab autocompletion --- volatility3/cli/__init__.py | 22 +++++++++++++++++++--- volatility3/cli/volargparse.py | 6 ++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 6b17edac0..883b54ac4 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -22,6 +22,13 @@ import traceback from typing import Any, Dict, List, Tuple, Type, Union from urllib import parse, request +try: + import shtab + + HAS_SHTAB = True +except ImportError: + HAS_SHTAB = False + from volatility3.cli import text_filter import volatility3.plugins import volatility3.symbols @@ -106,6 +113,9 @@ class CommandLine: ] ) + # Argument for doing autocompletion + print_completion_arg = "--print-completion" + # Load up system defaults delayed_logs, default_config = self.load_system_defaults("vol.json") @@ -246,10 +256,12 @@ class CommandLine: # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != "--help" and arg != "-h"] - partial_args, _ = parser.parse_known_args(known_args) - + partial_args, unknown_args = parser.parse_known_args(known_args) banner_output = sys.stdout - if renderers[partial_args.renderer].structured_output: + if ( + renderers[partial_args.renderer].structured_output + or print_completion_arg in unknown_args + ): banner_output = sys.stderr banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") @@ -351,6 +363,10 @@ class CommandLine: # Hand the plugin requirements over to the CLI (us) and let it construct the config tree # Run the argparser + if HAS_SHTAB: + # The autocompletion line must be after the partial_arg handling, so that it doesn't trip it + # before all the plugins have been added + shtab.add_argument_to(parser, [print_completion_arg]) args = parser.parse_args() if args.plugin is None: parser.error("Please select a plugin to run") diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index 3048a0885..3e7eb6751 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -21,8 +21,6 @@ class HelpfulSubparserAction(argparse._SubParsersAction): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - # We don't want the action self-check to kick in, so we remove the choices list, the check happens in __call__ - self.choices = None def __call__( self, @@ -100,3 +98,7 @@ class HelpfulArgParser(argparse.ArgumentParser): # return the number of arguments matched return len(match.group(1)) + + def _check_value(self, action, value): + if not isinstance(action, HelpfulSubparserAction): + return super()._check_value(action, value) From e89e77637776b14c61516d2c47a7148a2f13860a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Jul 2024 12:41:31 +0100 Subject: [PATCH 269/979] Try out argcomplete as well --- vol.py | 1 + volatility3/cli/__init__.py | 21 ++++++++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/vol.py b/vol.py index ff420cad5..c49d5985d 100755 --- a/vol.py +++ b/vol.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# PYTHON_ARGCOMPLETE_OK # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 883b54ac4..1209d7cdc 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -23,11 +23,11 @@ from typing import Any, Dict, List, Tuple, Type, Union from urllib import parse, request try: - import shtab + import argcomplete - HAS_SHTAB = True + HAS_ARGCOMPLETE = True except ImportError: - HAS_SHTAB = False + HAS_ARGCOMPLETE = False from volatility3.cli import text_filter import volatility3.plugins @@ -113,9 +113,6 @@ class CommandLine: ] ) - # Argument for doing autocompletion - print_completion_arg = "--print-completion" - # Load up system defaults delayed_logs, default_config = self.load_system_defaults("vol.json") @@ -256,12 +253,10 @@ class CommandLine: # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != "--help" and arg != "-h"] - partial_args, unknown_args = parser.parse_known_args(known_args) + partial_args, _ = parser.parse_known_args(known_args) + banner_output = sys.stdout - if ( - renderers[partial_args.renderer].structured_output - or print_completion_arg in unknown_args - ): + if renderers[partial_args.renderer].structured_output: banner_output = sys.stderr banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") @@ -363,10 +358,10 @@ class CommandLine: # Hand the plugin requirements over to the CLI (us) and let it construct the config tree # Run the argparser - if HAS_SHTAB: + if HAS_ARGCOMPLETE: # The autocompletion line must be after the partial_arg handling, so that it doesn't trip it # before all the plugins have been added - shtab.add_argument_to(parser, [print_completion_arg]) + argcomplete.autocomplete(parser) args = parser.parse_args() if args.plugin is None: parser.error("Please select a plugin to run") From 0d8fb76b3a48599ac9beb8eea5d79e46fe7b877e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Jul 2024 19:57:18 +0100 Subject: [PATCH 270/979] Renderers: Allow BaseAbsentValues in value results Fixes #1216 --- volatility3/framework/renderers/format_hints.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/volatility3/framework/renderers/format_hints.py b/volatility3/framework/renderers/format_hints.py index 6120b77c9..194e38099 100644 --- a/volatility3/framework/renderers/format_hints.py +++ b/volatility3/framework/renderers/format_hints.py @@ -10,6 +10,8 @@ Text renderers should attempt to honour all hints provided in this module where """ from typing import Type, Union +from volatility3.framework import interfaces + class Bin(int): """A class to indicate that the integer value should be represented as a @@ -66,3 +68,17 @@ class MultiTypeData(bytes): and self.split_nulls == other.split_nulls and self.show_hex == other.show_hex ) + + +BinOrAbsent = lambda x: ( + Bin(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x +) +HexOrAbsent = lambda x: ( + Hex(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x +) +HexBytesOrAbsent = lambda x: ( + HexBytes(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x +) +MultiTypeDataOrAbsent = lambda x: ( + MultiTypeData(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x +) From b659a060bd6caf37eb0e51560986cc58aab067de Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Jul 2024 21:57:48 +0100 Subject: [PATCH 271/979] Renderers: Ensure the version is bumped so plugins can require the format_hints properly --- volatility3/framework/constants/_version.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 21c339a6e..f219fb0af 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,11 +1,9 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 7 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_MINOR = 8 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" -# TODO: At version 2.0.0, remove the symbol_shift feature - PACKAGE_VERSION = ( ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + VERSION_SUFFIX From 76414d3246717ba9dde2a3cdd94a8c1037a7374b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 29 Jul 2024 14:13:54 +1000 Subject: [PATCH 272/979] Renderers conversion fix: Create aware datetimes to represent times in UTC. Fix Python 3.12 datetime.utcfromtimestamp() deprecation. See warning note on https://docs.python.org/3/library/datetime.html#datetime.datetime.utcfromtimestamp: Because naive datetime objects are treated by many datetime methods as local times, it is preferred to use aware datetimes to represent times in UTC. As such, the recommended way to create an object representing a specific timestamp in UTC is by calling datetime.fromtimestamp(timestamp, tz=timezone.utc). Additionaly, datetime.utcfromtimestamp() is deprecated since 3.12 --- volatility3/framework/renderers/conversion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index bb18fcc8a..c833cc2cf 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -19,7 +19,7 @@ def wintime_to_datetime( return renderers.NotApplicableValue() unix_time = unix_time - 11644473600 try: - return datetime.datetime.utcfromtimestamp(unix_time) + return datetime.datetime.fromtimestamp(unix_time, datetime.timezone.utc) # Windows sometimes throws OSErrors rather than ValueErrors when it can't convert a value except (ValueError, OSError): return renderers.UnparsableValue() @@ -34,7 +34,7 @@ def unixtime_to_datetime( if unixtime > 0: with contextlib.suppress(ValueError): - ret = datetime.datetime.utcfromtimestamp(unixtime) + ret = datetime.datetime.fromtimestamp(unixtime, datetime.timezone.utc) return ret From ece15c914f27e6fbb651e8b3f3ae7e525aa50138 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 29 Jul 2024 14:18:10 +1000 Subject: [PATCH 273/979] Renderers conversion exceptions fix: Since version 3.3 utcfromtimestamp() and fromtimestamp() Python datimetime module raises OverflowError instead of ValueError. As of today, Volatility3 requires Python 3.7.3 so we should only include OverflowError --- volatility3/framework/renderers/conversion.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index c833cc2cf..c94201681 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -20,8 +20,8 @@ def wintime_to_datetime( unix_time = unix_time - 11644473600 try: return datetime.datetime.fromtimestamp(unix_time, datetime.timezone.utc) - # Windows sometimes throws OSErrors rather than ValueErrors when it can't convert a value - except (ValueError, OSError): + # Windows sometimes throws OSErrors rather than OverflowError when it can't convert a value + except (OverflowError, OSError): return renderers.UnparsableValue() @@ -33,7 +33,7 @@ def unixtime_to_datetime( ) if unixtime > 0: - with contextlib.suppress(ValueError): + with contextlib.suppress(OverflowError): ret = datetime.datetime.fromtimestamp(unixtime, datetime.timezone.utc) return ret From b343734bae0d3090a5477997d27885d237096e50 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 29 Jul 2024 15:16:34 +1000 Subject: [PATCH 274/979] Renderers conversion exceptions fix: Even though the documentation states that OverflowError should be raised starting from version 3.3, it has been observed that ValueError is still being triggered. Also, in Linux, we noticed that OSError is also being raised in some cases. --- volatility3/framework/renderers/conversion.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index c94201681..864794860 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -20,8 +20,10 @@ def wintime_to_datetime( unix_time = unix_time - 11644473600 try: return datetime.datetime.fromtimestamp(unix_time, datetime.timezone.utc) - # Windows sometimes throws OSErrors rather than OverflowError when it can't convert a value - except (OverflowError, OSError): + # Windows sometimes throws OSErrors rather than ValueError/OverflowError when it can't convert a value + # Since Python 3.3, this should raise OverflowError instead of ValueError. However, it was observed + # that even in Python 3.7.17, ValueError is still being raised. + except (ValueError, OverflowError, OSError): return renderers.UnparsableValue() @@ -33,7 +35,9 @@ def unixtime_to_datetime( ) if unixtime > 0: - with contextlib.suppress(OverflowError): + # Since Python 3.3, this should raise OverflowError instead of ValueError. However, it was observed + # that even in Python 3.7.17, ValueError is still being raised. OSError is also raised on Linux + with contextlib.suppress(ValueError, OverflowError, OSError): ret = datetime.datetime.fromtimestamp(unixtime, datetime.timezone.utc) return ret From ba7ec0059960fd6470254e8b3cd4ae15e89ea174 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 29 Jul 2024 17:55:56 -0500 Subject: [PATCH 275/979] Windows: Fixes bad structure member in callbacks This fixes a bug in the x64 callbacks symbols. The `NotificationRoutine` is currently an `unsigned int` instead of a void pointer. This prevents the correct mapping of the notification routine to the kernel module that contains it. --- volatility3/framework/symbols/windows/callbacks-x64.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/callbacks-x64.json b/volatility3/framework/symbols/windows/callbacks-x64.json index 3f891b94f..705f2361d 100644 --- a/volatility3/framework/symbols/windows/callbacks-x64.json +++ b/volatility3/framework/symbols/windows/callbacks-x64.json @@ -105,8 +105,11 @@ }, "NotificationRoutine": { "type": { - "kind": "base", - "name": "unsigned int" + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } }, "offset": 24 } From d97fd777f37d58fff19df48334546c2e709f0d27 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 29 Jul 2024 18:36:43 -0500 Subject: [PATCH 276/979] Windows: Bumps netstat module version requirement This is a bump of the version number for the netstat plugin's `modules` requirement - it didn't get updated after #1173 was merged. --- volatility3/framework/plugins/windows/netstat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 24eb02018..0908767fc 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -35,7 +35,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): name="netscan", component=netscan.NetScan, version=(1, 0, 0) ), requirements.VersionRequirement( - name="modules", component=modules.Modules, version=(1, 0, 0) + name="modules", component=modules.Modules, version=(2, 0, 0) ), requirements.VersionRequirement( name="pdbutil", component=pdbutil.PDBUtility, version=(1, 0, 0) From 59fec176aa5377e7324c6996770d27a46fdebd9a Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 29 Jul 2024 18:49:48 -0500 Subject: [PATCH 277/979] Fix bugs in thrdscan and threads. Add orphan kernel threads plugin --- .../plugins/windows/orphan_kernel_threads.py | 95 +++++++++++++++++++ .../framework/plugins/windows/thrdscan.py | 2 +- .../framework/plugins/windows/threads.py | 12 +-- 3 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 volatility3/framework/plugins/windows/orphan_kernel_threads.py diff --git a/volatility3/framework/plugins/windows/orphan_kernel_threads.py b/volatility3/framework/plugins/windows/orphan_kernel_threads.py new file mode 100644 index 000000000..013ba98ff --- /dev/null +++ b/volatility3/framework/plugins/windows/orphan_kernel_threads.py @@ -0,0 +1,95 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List, Generator + +from volatility3.framework import interfaces, symbols +from volatility3.framework.configuration import requirements +from volatility3.plugins.windows import thrdscan, ssdt + +vollog = logging.getLogger(__name__) + + +class Threads(thrdscan.ThrdScan): + """Lists process threads""" + + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.implementation = self.list_orphan_kernel_threads + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="thrdscan", plugin=thrdscan.ThrdScan, version=(1, 1, 0) + ), + requirements.PluginRequirement( + name="ssdt", plugin=ssdt.SSDT, version=(1, 0, 0) + ), + ] + + @classmethod + def list_orphan_kernel_threads( + cls, + context: interfaces.context.ContextInterface, + module_name: str, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Yields thread objects of kernel threads that do not map to a module + + Args: + kernel + + Returns: + A generator of thread objects of orphaned threads + """ + module = context.modules[module_name] + layer_name = module.layer_name + symbol_table = module.symbol_table_name + + collection = ssdt.SSDT.build_module_collection( + context, layer_name, symbol_table + ) + + # used to filter out smeared pointers + if symbols.symbol_table_is_64bit(context, symbol_table): + kernel_start = 0xFFFFF80000000000 + else: + kernel_start = 0x80000000 + + for thread in thrdscan.ThrdScan.scan_threads(context, module_name): + # we don't want smeared or terminated threads + try: + proc = thread.owning_process() + except AttributeError: + continue + + # we only care about kernel threads, 4 = System + # previous methods for determining if a thread was a kernel thread + # such as bit fields and flags are not stable in Win10+ + # so we check if the thread is from the kernel itself or one its child + # kernel processes (MemCompression, Regsitry, ...) + if proc.UniqueProcessId != 4 and proc.InheritedFromUniqueProcessId != 4: + continue + + if thread.StartAddress < kernel_start: + continue + + module_symbols = list( + collection.get_module_symbols_by_absolute_location(thread.StartAddress) + ) + + # alert on threads that do not map to a module + if not module_symbols: + yield thread + diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 6e664e052..ad885e8e9 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -22,8 +22,8 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) _version = (1, 1, 0) def __init__(self, *args, **kwargs): - self.implementation = self.scan_threads super().__init__(*args, **kwargs) + self.implementation = self.scan_threads @classmethod def get_requirements(cls): diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index ae70e717b..98a3169a5 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -18,9 +18,9 @@ class Threads(thrdscan.ThrdScan): _required_framework_version = (2, 4, 0) _version = (1, 0, 0) - def __init__(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.implementation = self.list_process_threads - super().__init__() @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -50,7 +50,6 @@ class Threads(thrdscan.ThrdScan): Args: proc: _EPROCESS object from which to list the VADs - filter_func: Function to take a virtual address descriptor value and return True if it should be filtered out Returns: A list of threads based on the process and filtered based on the filter function @@ -64,22 +63,19 @@ class Threads(thrdscan.ThrdScan): seen.add(thread.vol.offset) yield thread - @classmethod - def filter_func(cls, config: interfaces.configuration.HierarchicalDict) -> Callable: - return pslist.PsList.create_pid_filter(config.get("pid", None)) - @classmethod def list_process_threads( cls, context: interfaces.context.ContextInterface, module_name: str, - filter_func: Callable, ) -> Iterable[interfaces.objects.ObjectInterface]: """Runs through all processes and lists threads for each process""" module = context.modules[module_name] layer_name = module.layer_name symbol_table_name = module.symbol_table_name + filter_func = pslist.PsList.create_pid_filter(context.config.get("pid", None)) + for proc in pslist.PsList.list_processes( context=context, layer_name=layer_name, From 896b40bd22028f8e68d954a9f9080d564c6982e2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 29 Jul 2024 18:50:55 -0500 Subject: [PATCH 278/979] Fix bugs in thrdscan and threads. Add orphan kernel threads plugin --- volatility3/framework/plugins/windows/orphan_kernel_threads.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/orphan_kernel_threads.py b/volatility3/framework/plugins/windows/orphan_kernel_threads.py index 013ba98ff..1cdb1dcdf 100644 --- a/volatility3/framework/plugins/windows/orphan_kernel_threads.py +++ b/volatility3/framework/plugins/windows/orphan_kernel_threads.py @@ -92,4 +92,3 @@ class Threads(thrdscan.ThrdScan): # alert on threads that do not map to a module if not module_symbols: yield thread - From efc48d5831d00fa7db45e853d380165731736841 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 30 Jul 2024 17:18:14 +1000 Subject: [PATCH 279/979] Make timeliner able to sort aware datetimes. Otherwise, it will raise an exception when comparing the plugin output data with this naive datetime --- volatility3/framework/plugins/timeliner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 26a100f53..f657a2918 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -105,7 +105,9 @@ class Timeliner(interfaces.plugins.PluginInterface): data = item[1] def sortable(timestamp): - max_date = datetime.datetime(day=1, month=12, year=datetime.MAXYEAR) + max_date = datetime.datetime( + day=1, month=12, year=datetime.MAXYEAR, tzinfo=datetime.timezone.utc + ) if isinstance(timestamp, interfaces.renderers.BaseAbsentValue): return max_date return timestamp From 56b618f5c3345984c10af7e2868254e2d2bb7ea8 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 30 Jul 2024 13:00:38 -0500 Subject: [PATCH 280/979] Windows: Updates netscan with new symbol file `netscan` was missing coverage for Windows 10 Build 20348, causing owners and create times for `_TCP_ENDPOINTS` to be missing. This adds a symbol file and the necessary version check in the netscan plugin. Testing confirms that this returns the correct creation time and owner process. --- .../framework/plugins/windows/netscan.py | 1 + .../netscan/netscan-win10-20348-x64.json | 582 ++++++++++++++++++ 2 files changed, 583 insertions(+) create mode 100644 volatility3/framework/symbols/windows/netscan/netscan-win10-20348-x64.json diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 62ead3ab7..868bd8bcd 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -218,6 +218,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): (10, 0, 18362, 0): "netscan-win10-18362-x64", (10, 0, 18363, 0): "netscan-win10-18363-x64", (10, 0, 19041, 0): "netscan-win10-19041-x64", + (10, 0, 20348, 0): "netscan-win10-20348-x64", } # we do not need to check for tcpip's specific FileVersion in every case diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win10-20348-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-20348-x64.json new file mode 100644 index 000000000..bd574b2b7 --- /dev/null +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-20348-x64.json @@ -0,0 +1,582 @@ +{ + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned be short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "big" + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "symbols": {}, + "user_types": { + "_UDP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 88, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } + }, + "LocalAddr": { + "offset": 168, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS_WIN10_UDP" + } + } + }, + "InetAF": { + "offset": 32, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Port": { + "offset": 160, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 168 + }, + "_TCP_LISTENER": { + "fields": { + "Owner": { + "offset": 48, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 64, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 96, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + + } + }, + "InetAF": { + "offset": 40, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Next": { + "offset": 120, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } + }, + "Port": { + "offset": 114, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 128 + }, + "_TCP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 752, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + } + }, + "CreateTime": { + "offset": 776, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "AddrInfo": { + "offset": 24, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ADDRINFO" + } + } + }, + "ListEntry": { + "offset": 40, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "InetAF": { + "offset": 16, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + } + }, + "LocalPort": { + "offset": 112, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 114, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "State": { + "offset": 108, + "type": { + "kind": "enum", + "name": "TCPStateEnum" + } + } + }, + "kind": "struct", + "size": 632 + }, + "_LOCAL_ADDRESS": { + "fields": { + "pData": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + } + }, + "kind": "struct", + "size": 20 + }, + "_LOCAL_ADDRESS_WIN10_UDP": { + "fields": { + "pData": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 + }, + "_ADDRINFO": { + "fields": { + "Local": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "Remote": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 + }, + "_IN_ADDR": { + "fields": { + "addr4": { + "offset": 0, + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + }, + "addr6": { + "offset": 0, + "type": { + "count": 16, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + } + }, + "kind": "struct", + "size": 6 + }, + "_INETAF": { + "fields": { + "AddressFamily": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 26 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 6144 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 224, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 208, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 192 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 + } + }, + "enums": { + "TCPStateEnum": { + "base": "long", + "constants": { + "CLOSED": 0, + "LISTENING": 1, + "SYN_SENT": 2, + "SYN_RCVD": 3, + "ESTABLISHED": 4, + "FIN_WAIT1": 5, + "FIN_WAIT2": 6, + "CLOSE_WAIT": 7, + "CLOSING": 8, + "LAST_ACK": 9, + "TIME_WAIT": 12, + "DELETE_TCB": 13 + }, + "size": 4 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "dgmcdona-by-hand", + "datetime": "2024-07-30T13:00:00" + }, + "format": "6.0.0" + } +} From a84a3706c5acee2c93df650a167ab058fe657053 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 30 Jul 2024 14:00:41 -0500 Subject: [PATCH 281/979] Remove errant filter on ldrmodule checks --- volatility3/framework/plugins/windows/ldrmodules.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/ldrmodules.py b/volatility3/framework/plugins/windows/ldrmodules.py index ffd84ea4a..a888f22e1 100644 --- a/volatility3/framework/plugins/windows/ldrmodules.py +++ b/volatility3/framework/plugins/windows/ldrmodules.py @@ -47,14 +47,6 @@ class LdrModules(interfaces.plugins.PluginInterface): self.context, self.config_path, "windows", "pe", class_types=pe.class_types ) - def filter_function(x: interfaces.objects.ObjectInterface) -> bool: - try: - return not (x.get_private_memory() == 0 and x.ControlArea) - except AttributeError: - return False - - filter_func = filter_function - for proc in procs: proc_layer_name = proc.add_process_layer() @@ -69,7 +61,7 @@ class LdrModules(interfaces.plugins.PluginInterface): # Build dictionary of mapped files, where the VAD start address is the key and value is the file name of the mapped file mapped_files = {} - for vad in vadinfo.VadInfo.list_vads(proc, filter_func=filter_func): + for vad in vadinfo.VadInfo.list_vads(proc): dos_header = self.context.object( pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", offset=vad.get_start(), From 73bc10c2834d9a8fa2ab04c098a4735542226170 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 31 Jul 2024 21:46:57 +0100 Subject: [PATCH 282/979] Wire the argcomplete into volshell too --- volatility3/cli/volshell/__init__.py | 12 ++++++++++++ volshell.py | 1 + 2 files changed, 13 insertions(+) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 035ed9b2e..2bf1958e2 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -21,6 +21,14 @@ from volatility3.framework import ( plugins, ) +try: + import argcomplete + + HAS_ARGCOMPLETE = True +except ImportError: + HAS_ARGCOMPLETE = False + + # Make sure we log everything rootlog = logging.getLogger() @@ -276,6 +284,10 @@ class VolShell(cli.CommandLine): # Hand the plugin requirements over to the CLI (us) and let it construct the config tree # Run the argparser + if HAS_ARGCOMPLETE: + # The autocompletion line must be after the partial_arg handling, so that it doesn't trip it + # before all the plugins have been added + argcomplete.autocomplete(parser) args = parser.parse_args() vollog.log( diff --git a/volshell.py b/volshell.py index 71d35a47c..65b11885e 100755 --- a/volshell.py +++ b/volshell.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# PYTHON_ARGCOMPLETE_OK # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 From d576f8cb48d064ce3dc87936df642481aa1f3186 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 31 Jul 2024 21:49:07 +0100 Subject: [PATCH 283/979] Fix CodeQL error --- volatility3/cli/volargparse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index 3e7eb6751..2bd53077b 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -99,6 +99,7 @@ class HelpfulArgParser(argparse.ArgumentParser): # return the number of arguments matched return len(match.group(1)) - def _check_value(self, action, value): + def _check_value(self, action: argparse.Action, value: Any) -> None: if not isinstance(action, HelpfulSubparserAction): return super()._check_value(action, value) + return None From fd6e4bec5cab84035491f629d234522ac6d5a8af Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Wed, 31 Jul 2024 17:53:29 -0500 Subject: [PATCH 284/979] Updated with feedback from the PR --- .../framework/plugins/windows/psxview.py | 172 ++++++++---------- 1 file changed, 74 insertions(+), 98 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 1fde5b12d..dc5e77ec3 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -1,4 +1,4 @@ -import datetime, logging +import datetime, logging, string from volatility3.framework import constants, exceptions from volatility3.framework.interfaces import plugins @@ -22,7 +22,7 @@ class PsXView(plugins.PluginInterface): plugin's output in a terminal.""" # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality - # which the original plugin to do it. + # which the original plugin used to do it. # I don't think it's worth including the sessions method either because both the original psxview plugin # and Volatility3's sessions plugin begin with the list of processes found by PsList. @@ -35,6 +35,10 @@ class PsXView(plugins.PluginInterface): _required_framework_version = (2, 0, 0) _version = (1, 0, 0) + valid_proc_name_chars = set( + string.ascii_lowercase + string.ascii_uppercase + "." + " " + ) + @classmethod def get_requirements(cls): return [ @@ -58,17 +62,6 @@ class PsXView(plugins.PluginInterface): requirements.VersionRequirement( name="handles", component=handles.Handles, version=(1, 0, 0) ), - requirements.VersionRequirement( - name="sessions", component=sessions.Sessions, version=(0, 0, 0) - ), - requirements.BooleanRequirement( - name="identify-expected", - description='In the plugin\'s output, replace false with \ - normal where false is the expected result for a Windows machine running normally. \ - Keep in mind that this plugin uses simple checks to identify "normal" behavior, \ - so you may want to double-check the legitimacy of these processes yourself.', - optional=True, - ), requirements.BooleanRequirement( name="physical-offsets", description="List processes with phyiscall offsets instead of virtual offsets.", @@ -76,54 +69,56 @@ class PsXView(plugins.PluginInterface): ), ] - def proc_name_to_string(self, proc): + def _proc_name_to_string(self, proc): return proc.ImageFileName.cast( "string", max_length=proc.ImageFileName.vol.count, errors="replace" ) - def is_ascii(self, str): - return str.split(".")[0].isalnum() + def _is_valid_proc_name(self, str): + for c in str: + if not c in self.valid_proc_name_chars: + return False + return True - def filter_garbage_procs(self, proc_list): + def _filter_garbage_procs(self, proc_list): return [ p for p in proc_list - if p.is_valid() and self.is_ascii(self.proc_name_to_string(p)) + if p.is_valid() and self._is_valid_proc_name(self._proc_name_to_string(p)) ] - def translate_offset(self, offset): - if self.config["physical-offsets"]: + def _translate_offset(self, offset): + if not self.config["physical-offsets"]: return offset kernel = self.context.modules[self.config["kernel"]] layer_name = kernel.layer_name try: - offset = list( + _, _, offset, _, _ = list( self.context.layers[layer_name].mapping(offset=offset, length=0) - )[0][2] + )[0] except: # already have physical address pass return offset - def proc_list_to_dict(self, tasks): - return {self.translate_offset(proc.vol.offset): proc for proc in tasks} + def _proc_list_to_dict(self, tasks): + tasks = self._filter_garbage_procs(tasks) + return {self._translate_offset(proc.vol.offset): proc for proc in tasks} - def check_pslist(self, tasks): - res = self.filter_garbage_procs(tasks) - return self.proc_list_to_dict(tasks) + def _check_pslist(self, tasks): + return self._proc_list_to_dict(tasks) - def check_psscan(self, layer_name, symbol_table): + def _check_psscan(self, layer_name, symbol_table): res = psscan.PsScan.scan_processes( context=self.context, layer_name=layer_name, symbol_table=symbol_table ) - res = self.filter_garbage_procs(res) - return self.proc_list_to_dict(res) + return self._proc_list_to_dict(res) - def check_thrdscan(self): + def _check_thrdscan(self): ret = [] for ethread in thrdscan.ThrdScan.scan_threads( @@ -142,13 +137,13 @@ class PsXView(plugins.PluginInterface): "Unable to find the owning process of ethread", ) - return self.proc_list_to_dict(ret) + return self._proc_list_to_dict(ret) - def check_csrss_handles(self, tasks, layer_name, symbol_table): + def _check_csrss_handles(self, tasks, layer_name, symbol_table): ret = [] for p in tasks: - name = self.proc_name_to_string(p) + name = self._proc_name_to_string(p) if name == "csrss.exe": try: if p.has_member("ObjectTable"): @@ -172,13 +167,7 @@ class PsXView(plugins.PluginInterface): constants.LOGLEVEL_VVV, "Cannot access eprocess object table" ) - ret = self.filter_garbage_procs(ret) - return self.proc_list_to_dict(ret) - - def check_session(self, pslist_procs): - procs = [p for p in pslist_procs if p.get_session_id() != None] - - return self.proc_list_to_dict(procs) + return self._proc_list_to_dict(ret) def _generator(self): kernel = self.context.modules[self.config["kernel"]] @@ -192,72 +181,60 @@ class PsXView(plugins.PluginInterface): ) ) + # get processes from each source processes = {} - processes["pslist"] = self.check_pslist(kdbg_list_processes) - processes["psscan"] = self.check_psscan(layer_name, symbol_table) - processes["thrdscan"] = self.check_thrdscan() - processes["csrss"] = self.check_csrss_handles( + processes["pslist"] = self._check_pslist(kdbg_list_processes) + processes["psscan"] = self._check_psscan(layer_name, symbol_table) + processes["thrdscan"] = self._check_thrdscan() + processes["csrss"] = self._check_csrss_handles( kdbg_list_processes, layer_name, symbol_table ) - processes["sessions"] = self.check_session(kdbg_list_processes) - seen_offsets = set() - for source in processes: - for offset in processes[source]: - if offset not in seen_offsets: - seen_offsets.add(offset) - proc = processes[source][offset] + # print results - pid = proc.UniqueProcessId - name = self.proc_name_to_string(proc) + # list of lists of offsets + todo_offsets = [list(processes[source].keys()) for source in processes] - exit_time = proc.get_exit_time() - if type(exit_time) != datetime.datetime: - exit_time = "" - else: - exit_time = str(exit_time) + # flatten to one list + todo_offsets = sum(todo_offsets, []) - in_sources = { - src: str(offset in processes[src]) for src in processes - } + # remove duplicates + todo_offsets = set(todo_offsets) - if self.config["identify-expected"]: - f = "False" - n = "Normal" + for offset in todo_offsets: + proc = None - if in_sources["pslist"] == f: - if exit_time != "": - in_sources["pslist"] = n + in_sources = {src: False for src in processes} - if in_sources["thrdscan"] == f: - if exit_time != "": - in_sources["thrdscan"] = n + for source in processes: + if offset in processes[source]: + in_sources[source] = True + if not proc: + proc = processes[source][offset] - if in_sources["csrss"] == f: - if name.lower() in ["system", "smss.exe", "csrss.exe"]: - in_sources["csrss"] = n - elif exit_time != "": - in_sources["csrss"] = n + pid = proc.UniqueProcessId + name = self._proc_name_to_string(proc) - if in_sources["sessions"] == f: - if name.lower() in ["system", "smss.exe"]: - in_sources["sessions"] = n + exit_time = proc.get_exit_time() + if type(exit_time) != datetime.datetime: + exit_time = "" + else: + exit_time = str(exit_time) - yield ( - 0, - ( - format_hints.Hex(offset), - name, - pid, - in_sources["pslist"], - in_sources["psscan"], - in_sources["thrdscan"], - in_sources["csrss"], - in_sources["sessions"], - exit_time, - ), - ) + yield ( + 0, + ( + format_hints.Hex(offset), + name, + pid, + in_sources["pslist"], + in_sources["psscan"], + in_sources["thrdscan"], + in_sources["csrss"], + exit_time, + ), + ) def run(self): offset_type = "(Physical)" if self.config["physical-offsets"] else "(Virtual)" @@ -268,11 +245,10 @@ class PsXView(plugins.PluginInterface): (offset_str, format_hints.Hex), ("Name", str), ("PID", int), - ("pslist", str), - ("psscan", str), - ("thrdscan", str), - ("csrss", str), - ("sessions", str), + ("pslist", bool), + ("psscan", bool), + ("thrdscan", bool), + ("csrss", bool), ("Exit Time", str), ], self._generator(), From 9e8864521c3fb06e80536a8d0f249ac19fd9a9c0 Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:02:52 -0500 Subject: [PATCH 285/979] Added debug log for failed address translation --- volatility3/framework/plugins/windows/psxview.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index dc5e77ec3..918eb44ba 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -98,9 +98,8 @@ class PsXView(plugins.PluginInterface): _, _, offset, _, _ = list( self.context.layers[layer_name].mapping(offset=offset, length=0) )[0] - except: - # already have physical address - pass + except exceptions.PagedInvalidAddressException: + vollog.debug(f"Page fault: unable to translate {offset:0x}") return offset From 90b8b5340f9bc886e9b45ecadd80b7d86b8384f3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 1 Aug 2024 11:46:10 +1000 Subject: [PATCH 286/979] Linux: Add inode, timespec, and timespec64 object extensions to support different kernel versions, ensuring we will get aware datetimes when using them. --- .../framework/symbols/linux/__init__.py | 6 + .../symbols/linux/extensions/__init__.py | 132 ++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index c4e2587f4..03353135d 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -29,12 +29,18 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) self.set_type_class("cred", extensions.cred) + self.set_type_class("inode", extensions.inode) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) + # kernels >= 4.18 + self.optional_set_type_class("timespec64", extensions.timespec64) + # kernels < 4.18. Reuses timespec64 obj extension, since both has the same members + self.optional_set_type_class("timespec", extensions.timespec64) + # Mount self.set_type_class("vfsmount", extensions.vfsmount) # Might not exist in older kernels or the current symbols diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e7c6b66d7..be31e298c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -4,10 +4,13 @@ import collections.abc import logging +import stat +from datetime import datetime import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple, List from volatility3.framework import constants, exceptions, objects, interfaces, symbols +from volatility3.framework import renderers from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1761,3 +1764,132 @@ class kernel_cap_t(kernel_cap_struct): ) return cap_value & self.get_kernel_cap_full() + + +class timespec64(objects.StructType): + def to_datetime(self) -> datetime: + """Returns the respective aware datetime""" + + dt = renderers.conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) + return dt + + +class inode(objects.StructType): + def is_valid(self) -> bool: + # i_count is a 'signed' counter (atomic_t). Smear, or essentially a wrong inode + # pointer, will easily cause an integer overflow here. + return self.i_ino > 0 and self.i_count.counter >= 0 + + def is_dir(self) -> bool: + """Returns True if the inode is a directory""" + return stat.S_ISDIR(self.i_mode) != 0 + + def is_reg(self) -> bool: + """Returns True if the inode is a regular file""" + return stat.S_ISREG(self.i_mode) != 0 + + def is_link(self) -> bool: + """Returns True if the inode is a symlink""" + return stat.S_ISLNK(self.i_mode) != 0 + + def is_fifo(self) -> bool: + """Returns True if the inode is a FIFO""" + return stat.S_ISFIFO(self.i_mode) != 0 + + def is_sock(self) -> bool: + """Returns True if the inode is a socket""" + return stat.S_ISSOCK(self.i_mode) != 0 + + def is_block(self) -> bool: + """Returns True if the inode is a block device""" + return stat.S_ISBLK(self.i_mode) != 0 + + def is_char(self) -> bool: + """Returns True if the inode is a char device""" + return stat.S_ISCHR(self.i_mode) != 0 + + def is_sticky(self) -> bool: + """Returns True if the sticky bit is set""" + return (self.i_mode & stat.S_ISVTX) != 0 + + def get_inode_type(self) -> str: + """Returns inode type name + + Returns: + The inode type name + """ + if self.is_dir(): + return "DIR" + elif self.is_reg(): + return "REG" + elif self.is_link(): + return "LNK" + elif self.is_fifo(): + return "FIFO" + elif self.is_sock(): + return "SOCK" + elif self.is_char(): + return "CHR" + elif self.is_block(): + return "BLK" + else: + return renderers.UnparsableValue() + + def get_inode_number(self) -> int: + """Returns the inode number""" + return int(self.i_ino) + + def ___time_member_to_datetime(self, member) -> datetime: + if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): + # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 + # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 + return renderers.conversion.unixtime_to_datetime( + self.member(f"{member}_sec") + self.has_member(f"{member}_nsec") / 1e9 + ) + elif self.has_member(f"__{member}"): + # 6.6 <= kernels < 6.11 it's a timespec64 + # Ref Linux commit 13bc24457850583a2e7203ded05b7209ab4bc5ef / 12cd44023651666bd44baa36a5c999698890debb + return self.member(f"__{member}").to_datetime() + elif self.has_member(member): + # In kernels < 6.6 it's a timespec64 or timespec + return self.member(member).to_datetime() + else: + raise exceptions.VolatilityException( + "Unsupported kernel inode type implementation" + ) + + def get_access_time(self) -> datetime: + """Returns the inode's last access time + This is updated when inode contents are read + + Returns: + A datetime with the inode's last access time + """ + return self.___time_member_to_datetime("i_atime") + + def get_modification_time(self) -> datetime: + """Returns the inode's last modification time + This is updated when the inode contents change + + Returns: + A datetime with the inode's last data modification time + """ + + return self.___time_member_to_datetime("i_mtime") + + def get_change_time(self) -> datetime: + """Returns the inode's last change time + This is updated when the inode metadata changes + + Returns: + A datetime with the inode's last change time + """ + return self.___time_member_to_datetime("i_ctime") + + def get_file_mode(self) -> str: + """Returns the inode's file mode as string of the form '-rwxrwxrwx'. + + Returns: + The inode's file mode string + """ + return stat.filemode(self.i_mode) From 3017a7d00c3a9cfca3f94b9e7dfb8ba08d48fd0e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 1 Aug 2024 11:46:10 +1000 Subject: [PATCH 287/979] Linux: Add inode, timespec, and timespec64 object extensions to support different kernel versions, ensuring we will get aware datetimes when using them. --- .../framework/symbols/linux/__init__.py | 6 + .../symbols/linux/extensions/__init__.py | 132 ++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index c4e2587f4..03353135d 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -29,12 +29,18 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) self.set_type_class("cred", extensions.cred) + self.set_type_class("inode", extensions.inode) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) + # kernels >= 4.18 + self.optional_set_type_class("timespec64", extensions.timespec64) + # kernels < 4.18. Reuses timespec64 obj extension, since both has the same members + self.optional_set_type_class("timespec", extensions.timespec64) + # Mount self.set_type_class("vfsmount", extensions.vfsmount) # Might not exist in older kernels or the current symbols diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e7c6b66d7..be31e298c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -4,10 +4,13 @@ import collections.abc import logging +import stat +from datetime import datetime import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple, List from volatility3.framework import constants, exceptions, objects, interfaces, symbols +from volatility3.framework import renderers from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1761,3 +1764,132 @@ class kernel_cap_t(kernel_cap_struct): ) return cap_value & self.get_kernel_cap_full() + + +class timespec64(objects.StructType): + def to_datetime(self) -> datetime: + """Returns the respective aware datetime""" + + dt = renderers.conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) + return dt + + +class inode(objects.StructType): + def is_valid(self) -> bool: + # i_count is a 'signed' counter (atomic_t). Smear, or essentially a wrong inode + # pointer, will easily cause an integer overflow here. + return self.i_ino > 0 and self.i_count.counter >= 0 + + def is_dir(self) -> bool: + """Returns True if the inode is a directory""" + return stat.S_ISDIR(self.i_mode) != 0 + + def is_reg(self) -> bool: + """Returns True if the inode is a regular file""" + return stat.S_ISREG(self.i_mode) != 0 + + def is_link(self) -> bool: + """Returns True if the inode is a symlink""" + return stat.S_ISLNK(self.i_mode) != 0 + + def is_fifo(self) -> bool: + """Returns True if the inode is a FIFO""" + return stat.S_ISFIFO(self.i_mode) != 0 + + def is_sock(self) -> bool: + """Returns True if the inode is a socket""" + return stat.S_ISSOCK(self.i_mode) != 0 + + def is_block(self) -> bool: + """Returns True if the inode is a block device""" + return stat.S_ISBLK(self.i_mode) != 0 + + def is_char(self) -> bool: + """Returns True if the inode is a char device""" + return stat.S_ISCHR(self.i_mode) != 0 + + def is_sticky(self) -> bool: + """Returns True if the sticky bit is set""" + return (self.i_mode & stat.S_ISVTX) != 0 + + def get_inode_type(self) -> str: + """Returns inode type name + + Returns: + The inode type name + """ + if self.is_dir(): + return "DIR" + elif self.is_reg(): + return "REG" + elif self.is_link(): + return "LNK" + elif self.is_fifo(): + return "FIFO" + elif self.is_sock(): + return "SOCK" + elif self.is_char(): + return "CHR" + elif self.is_block(): + return "BLK" + else: + return renderers.UnparsableValue() + + def get_inode_number(self) -> int: + """Returns the inode number""" + return int(self.i_ino) + + def ___time_member_to_datetime(self, member) -> datetime: + if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): + # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 + # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 + return renderers.conversion.unixtime_to_datetime( + self.member(f"{member}_sec") + self.has_member(f"{member}_nsec") / 1e9 + ) + elif self.has_member(f"__{member}"): + # 6.6 <= kernels < 6.11 it's a timespec64 + # Ref Linux commit 13bc24457850583a2e7203ded05b7209ab4bc5ef / 12cd44023651666bd44baa36a5c999698890debb + return self.member(f"__{member}").to_datetime() + elif self.has_member(member): + # In kernels < 6.6 it's a timespec64 or timespec + return self.member(member).to_datetime() + else: + raise exceptions.VolatilityException( + "Unsupported kernel inode type implementation" + ) + + def get_access_time(self) -> datetime: + """Returns the inode's last access time + This is updated when inode contents are read + + Returns: + A datetime with the inode's last access time + """ + return self.___time_member_to_datetime("i_atime") + + def get_modification_time(self) -> datetime: + """Returns the inode's last modification time + This is updated when the inode contents change + + Returns: + A datetime with the inode's last data modification time + """ + + return self.___time_member_to_datetime("i_mtime") + + def get_change_time(self) -> datetime: + """Returns the inode's last change time + This is updated when the inode metadata changes + + Returns: + A datetime with the inode's last change time + """ + return self.___time_member_to_datetime("i_ctime") + + def get_file_mode(self) -> str: + """Returns the inode's file mode as string of the form '-rwxrwxrwx'. + + Returns: + The inode's file mode string + """ + return stat.filemode(self.i_mode) From dbc837474c9c1d6096abd7cd27496677f0a36e7b Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 1 Aug 2024 14:23:22 -0500 Subject: [PATCH 288/979] #816 - initial support for Windows Windows Server 2022 (10.0.20348) --- .../framework/plugins/windows/consoles.py | 786 ++++++++++++++++++ .../consoles-win10-20348-2461-x64.json | 595 +++++++++++++ .../consoles/consoles-win10-20348-x64.json | 595 +++++++++++++ .../symbols/windows/extensions/consoles.py | 303 +++++++ 4 files changed, 2279 insertions(+) create mode 100644 volatility3/framework/plugins/windows/consoles.py create mode 100644 volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json create mode 100644 volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json create mode 100644 volatility3/framework/symbols/windows/extensions/consoles.py diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py new file mode 100644 index 000000000..f6bbb3ae7 --- /dev/null +++ b/volatility3/framework/plugins/windows/consoles.py @@ -0,0 +1,786 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +# This module attempts to locate windows console histories. + +import logging +import os +import struct +from typing import Tuple, Generator, Set, Dict, Any, Type + +from volatility3.framework import interfaces, symbols, exceptions +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import scanners +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows import pdbutil, versions +from volatility3.framework.symbols.windows.extensions import pe, consoles +from volatility3.plugins.windows import pslist, vadinfo, info, verinfo +from volatility3.plugins.windows.registry import hivelist + + +try: + import capstone + + has_capstone = True +except ImportError: + has_capstone = False + +vollog = logging.getLogger(__name__) + + +class Consoles(interfaces.plugins.PluginInterface): + """Looks for Windows console buffers""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pdbutil", component=pdbutil.PDBUtility, version=(1, 0, 0) + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), + requirements.BooleanRequirement( + name="no_registry", + description="Don't search the registry for possible values of CommandHistorySize and HistoryBufferMax", + optional=True, + default=False, + ), + requirements.ListRequirement( + name="max_history", + element_type=int, + description="CommandHistorySize values to search for.", + optional=True, + default=[50], + ), + requirements.ListRequirement( + name="max_buffers", + element_type=int, + description="HistoryBufferMax values to search for.", + optional=True, + default=[4], + ), + ] + + @classmethod + def find_conhost_proc( + cls, proc_list: Generator[interfaces.objects.ObjectInterface, None, None] + ) -> Tuple[interfaces.context.ContextInterface, str]: + """ + Walks the process list and returns the conhost instances. + + Args: + proc_list: The process list generator + + Return: + The process object and layer name for conhost + """ + + for proc in proc_list: + try: + proc_id = proc.UniqueProcessId + proc_layer_name = proc.add_process_layer() + + yield proc, proc_layer_name + + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) + ) + + @classmethod + def find_conhostexe( + cls, conhost_proc: interfaces.context.ContextInterface + ) -> Tuple[int, int]: + """ + Finds the base address of conhost.exe + + Args: + conhost_proc: the process object for conhost.exe + + Returns: + A tuple of: + conhostexe_base: the base address of conhost.exe + conhostexe_size: the size of the VAD for conhost.exe + """ + for vad in conhost_proc.get_vad_root().traverse(): + filename = vad.get_file_name() + + if isinstance(filename, str) and filename.lower().endswith("conhost.exe"): + base = vad.get_start() + return base, vad.get_size() + + return None, None + + @classmethod + def determine_conhost_version( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + nt_symbol_table: str, + ) -> Tuple[str, Type]: + """Tries to determine which symbol filename to use for the image's console information. This is similar to the + netstat plugin. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + nt_symbol_table: The name of the table containing the kernel symbols + + Returns: + The filename of the symbol table to use and the associated class types. + """ + + is_64bit = symbols.symbol_table_is_64bit(context, nt_symbol_table) + + is_18363_or_later = versions.is_win10_18363_or_later( + context=context, symbol_table=nt_symbol_table + ) + + if is_64bit: + arch = "x64" + else: + arch = "x86" + + vers = info.Info.get_version_structure(context, layer_name, nt_symbol_table) + + kuser = info.Info.get_kuser_structure(context, layer_name, nt_symbol_table) + + try: + vers_minor_version = int(vers.MinorVersion) + nt_major_version = int(kuser.NtMajorVersion) + nt_minor_version = int(kuser.NtMinorVersion) + except ValueError: + # vers struct exists, but is not an int anymore? + raise NotImplementedError( + "Kernel Debug Structure version format not supported!" + ) + except: + # unsure what to raise here. Also, it might be useful to add some kind of fallback, + # either to a user-provided version or to another method to determine tcpip.sys's version + raise exceptions.VolatilityException( + "Kernel Debug Structure missing VERSION/KUSER structure, unable to determine Windows version!" + ) + + vollog.debug( + "Determined OS Version: {}.{} {}.{}".format( + kuser.NtMajorVersion, + kuser.NtMinorVersion, + vers.MajorVersion, + vers.MinorVersion, + ) + ) + + if nt_major_version == 10 and arch == "x64": + # win10 x64 has an additional class type we have to include. + class_types = consoles.win10_x64_class_types + else: + # default to general class types + class_types = consoles.class_types + + # these versions are listed explicitly because symbol files differ based on + # version *and* architecture. this is currently the clearest way to show + # the differences, even if it introduces a fair bit of redundancy. + # furthermore, it is easy to append new versions. + if arch == "x86": + version_dict = {} + else: + version_dict = { + (10, 0, 20348, 1): "consoles-win10-20348-x64", + (10, 0, 20348, 1970): "consoles-win10-20348-1970-x64", + (10, 0, 20348, 2461): "consoles-win10-20348-2461-x64", + (10, 0, 20348, 2520): "consoles-win10-20348-2461-x64", + } + + # we do not need to check for conhost's specific FileVersion in every case + conhost_mod_version = 0 # keep it 0 as a default + + # special use cases + + # Win10_18363 is not recognized by windows.info as 18363 + # because all kernel file headers and debug structures report 18363 as + # "10.0.18362.1198" with the last part being incremented. However, we can use + # os_distinguisher to differentiate between 18362 and 18363 + if vers_minor_version == 18362 and is_18363_or_later: + vollog.debug( + "Detected 18363 data structures: working with 18363 symbol table." + ) + vers_minor_version = 18363 + + # we need to define additional version numbers (which are then found via conhost.exe's FileVersion header) in case there is + # ambiguity _within_ an OS version. If such a version number (last number of the tuple) is defined for the current OS + # we need to inspect conhost.exe's headers to see if we can grab the precise version + if [ + (a, b, c, d) + for a, b, c, d in version_dict + if (a, b, c) == (nt_major_version, nt_minor_version, vers_minor_version) + and d != 0 + ]: + vollog.debug( + "Requiring further version inspection due to OS version by checking conhost.exe's FileVersion header" + ) + # the following is IntelLayer specific and might need to be adapted to other architectures. + physical_layer_name = context.layers[layer_name].config.get( + "memory_layer", None + ) + if physical_layer_name: + ver = verinfo.VerInfo.find_version_info( + context, physical_layer_name, "CONHOST.EXE" + ) + + if ver: + conhost_mod_version = ver[3] + vollog.debug( + "Determined conhost.exe's FileVersion: {}".format( + conhost_mod_version + ) + ) + else: + vollog.debug("Could not determine conhost.exe's FileVersion.") + else: + vollog.debug( + "Unable to retrieve physical memory layer, skipping FileVersion check." + ) + + # when determining the symbol file we have to consider the following cases: + # the determined version's symbol file is found by intermed.create -> proceed + # the determined version's symbol file is not found by intermed -> intermed will throw an exc and abort + # the determined version has no mapped symbol file -> if win10 use latest, otherwise throw exc + # windows version cannot be determined -> throw exc + + filename = version_dict.get( + ( + nt_major_version, + nt_minor_version, + vers_minor_version, + conhost_mod_version, + ) + ) + + if not filename: + # no match on filename means that we possibly have a version newer than those listed here. + # try to grab the latest supported version of the current image NT version. If that symbol + # version does not work, support has to be added manually. + current_versions = [ + (nt_maj, nt_min, vers_min, tcpip_ver) + for nt_maj, nt_min, vers_min, tcpip_ver in version_dict + if nt_maj == nt_major_version + and nt_min == nt_minor_version + and tcpip_ver <= conhost_mod_version + ] + current_versions.sort() + + if current_versions: + latest_version = current_versions[-1] + + filename = version_dict.get(latest_version) + + vollog.debug( + f"Unable to find exact matching symbol file, going with latest: {filename}" + ) + + else: + raise NotImplementedError( + "This version of Windows is not supported: {}.{} {}.{}!".format( + nt_major_version, + nt_minor_version, + vers.MajorVersion, + vers_minor_version, + ) + ) + + vollog.debug(f"Determined symbol filename: {filename}") + + return filename, class_types + + @classmethod + def create_conhost_symbol_table( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + nt_symbol_table: str, + config_path: str, + ) -> str: + """Creates a symbol table for TCP Listeners and TCP/UDP Endpoints. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + nt_symbol_table: The name of the table containing the kernel symbols + config_path: The config path where to find symbol files + + Returns: + The name of the constructed symbol table + """ + table_mapping = {"nt_symbols": nt_symbol_table} + + symbol_filename, class_types = cls.determine_conhost_version( + context, + layer_name, + nt_symbol_table, + ) + + vollog.debug(f"Using symbol file '{symbol_filename}' and types {class_types}") + + return intermed.IntermediateSymbolTable.create( + context, + config_path, + os.path.join("windows", "consoles"), + symbol_filename, + class_types=class_types, + table_mapping=table_mapping, + ) + + @classmethod + def get_console_info( + cls, + context: interfaces.context.ContextInterface, + kernel_layer_name: str, + kernel_table_name: str, + config_path: str, + procs: Generator[interfaces.objects.ObjectInterface, None, None], + max_history: Set[int], + max_buffers: Set[int], + ) -> Tuple[ + interfaces.context.ContextInterface, + interfaces.context.ContextInterface, + Dict[str, Any], + ]: + """Extracts the cmdline from PEB + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + kernel_layer_name: The name of the layer on which to operate + kernel_table_name: The name of the table containing the kernel symbols + config_path: The config path where to find symbol files + procs: list of process objects + max_history: an initial set of CommandHistorySize values + max_buffers: an initial list of HistoryBufferMax values + + Returns: + The conhost process object, the console information structure, a dictionary of properties for + that console information structure. + """ + + conhost_symbol_table = cls.create_conhost_symbol_table( + context, kernel_layer_name, kernel_table_name, config_path + ) + + for conhost_proc, proc_layer_name in cls.find_conhost_proc(procs): + if not conhost_proc: + vollog.info( + "Unable to find a valid conhost.exe process in the process list. Analysis cannot proceed." + ) + continue + vollog.debug( + f"Found conhost process {conhost_proc} with pid {conhost_proc.UniqueProcessId}" + ) + + conhostexe_base, conhostexe_size = cls.find_conhostexe(conhost_proc) + if not conhostexe_base: + vollog.info( + "Unable to find the location of conhost.exe. Analysis cannot proceed." + ) + continue + vollog.debug(f"Found conhost.exe base at {conhostexe_base:#x}") + + proc_layer = context.layers[proc_layer_name] + + conhost_module = context.module( + conhost_symbol_table, proc_layer_name, offset=conhostexe_base + ) + + # scan for potential _CONSOLE_INFORMATION structures by using the CommandHistorySize + for max_history_value in max_history: + max_history_bytes = struct.pack("H", max_history_value) + vollog.debug( + f"Scanning for CommandHistorySize value: {max_history_bytes}" + ) + for address in proc_layer.scan( + context, + scanners.BytesScanner(max_history_bytes), + sections=[(conhostexe_base, conhostexe_size)], + ): + + console_properties = [] + + try: + console_info = conhost_module.object( + "_CONSOLE_INFORMATION", + offset=address + - conhost_module.get_type( + "_CONSOLE_INFORMATION" + ).relative_child_offset("CommandHistorySize"), + absolute=True, + ) + + if not any( + [ + console_info.is_valid(max_buffer) + for max_buffer in max_buffers + ] + ): + continue + + vollog.debug( + f"Getting Console Information properties for {console_info}" + ) + console_properties.append( + { + "name": "_CONSOLE_INFORMATION.ScreenX", + "address": console_info.ScreenX.vol.offset, + "data": console_info.ScreenX, + } + ) + console_properties.append( + { + "name": "_CONSOLE_INFORMATION.ScreenY", + "address": console_info.ScreenY.vol.offset, + "data": console_info.ScreenY, + } + ) + console_properties.append( + { + "name": "_CONSOLE_INFORMATION.CommandHistorySize", + "address": console_info.CommandHistorySize.vol.offset, + "data": console_info.CommandHistorySize, + } + ) + console_properties.append( + { + "name": "_CONSOLE_INFORMATION.HistoryBufferCount", + "address": console_info.HistoryBufferCount.vol.offset, + "data": console_info.HistoryBufferCount, + } + ) + console_properties.append( + { + "name": "_CONSOLE_INFORMATION.HistoryBufferMax", + "address": console_info.HistoryBufferMax.vol.offset, + "data": console_info.HistoryBufferMax, + } + ) + console_properties.append( + { + "name": "_CONSOLE_INFORMATION.Title", + "address": console_info.Title.vol.offset, + "data": console_info.get_title(), + } + ) + console_properties.append( + { + "name": "_CONSOLE_INFORMATION.OriginalTitle", + "address": console_info.OriginalTitle.vol.offset, + "data": console_info.get_original_title(), + } + ) + + vollog.debug( + f"Getting ConsoleProcessList entries for {console_info.ConsoleProcessList}" + ) + console_properties.append( + { + "name": "_CONSOLE_INFORMATION.ProcessCount", + "address": console_info.ProcessCount.vol.offset, + "data": console_info.ProcessCount, + } + ) + console_properties.append( + { + "name": "_CONSOLE_INFORMATION.ConsoleProcessList", + "address": console_info.ConsoleProcessList.vol.offset, + "data": "", + } + ) + for index, attached_proc in enumerate( + console_info.get_processes() + ): + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.ConsoleProcessList.ConsoleProcess_{index}", + "address": attached_proc.ConsoleProcess.dereference().vol.offset, + "data": "", + } + ) + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.ConsoleProcessList.ConsoleProcess_{index}_ProcessId", + "address": attached_proc.ConsoleProcess.ProcessId.vol.offset, + "data": attached_proc.ConsoleProcess.ProcessId, + } + ) + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.ConsoleProcessList.ConsoleProcess_{index}_ProcessHandle", + "address": attached_proc.ConsoleProcess.ProcessHandle.vol.offset, + "data": hex( + attached_proc.ConsoleProcess.ProcessHandle + ), + } + ) + + vollog.debug( + f"Getting HistoryList entries for {console_info.HistoryList}" + ) + console_properties.append( + { + "name": "_CONSOLE_INFORMATION.HistoryList", + "address": console_info.HistoryList.vol.offset, + "data": "", + } + ) + for index, command_history in enumerate( + console_info.get_histories() + ): + try: + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}", + "address": command_history.vol.offset, + "data": "", + } + ) + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_Application", + "address": command_history.Application.vol.offset, + "data": command_history.get_application(), + } + ) + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_ProcessHandle", + "address": command_history.ConsoleProcessHandle.ProcessHandle.vol.offset, + "data": hex( + command_history.ConsoleProcessHandle.ProcessHandle + ), + } + ) + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_CommandCount", + "address": None, + "data": command_history.CommandCount, + } + ) + for ( + cmd_index, + bucket_cmd, + ) in command_history.get_commands(): + try: + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_Command_{cmd_index}", + "address": bucket_cmd.vol.offset, + "data": bucket_cmd.get_command(), + } + ) + except Exception as e: + vollog.debug( + f"reading {bucket_cmd} encountered exception {e}" + ) + except Exception as e: + vollog.debug( + f"reading {command_history} encountered exception {e}" + ) + + vollog.debug(f"Getting ScreenBuffer entries for {console_info}") + for screen_index, screen_info in enumerate( + console_info.get_screens() + ): + try: + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}", + "address": screen_info, + "data": "", + } + ) + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.ScreenX", + "address": None, + "data": screen_info.ScreenX, + } + ) + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.ScreenY", + "address": None, + "data": screen_info.ScreenY, + } + ) + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.Dump", + "address": None, + "data": "\n".join(screen_info.get_buffer()), + } + ) + except Exception as e: + vollog.debug( + f"reading {screen_info} encountered exception {e}" + ) + + except exceptions.PagedInvalidAddressException as exp: + vollog.debug( + f"Required memory at {exp.invalid_address:#x} is not valid" + ) + + yield conhost_proc, console_info, console_properties + + @classmethod + def get_console_settings_from_registry( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + kernel_layer_name: str, + kernel_symbol_table_name: str, + max_history: Set[int], + max_buffers: Set[int], + ) -> Tuple[Set[int], Set[int]]: + """ + Walks the Registry user hives and extracts any CommandHistorySize and HistoryBufferMax values + for scanning + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + config_path: The config path where to find symbol files + kernel_layer_name: The name of the layer on which to operate + kernel_symbol_table_name: The name of the table containing the kernel symbols + max_history: an initial set of CommandHistorySize values + max_buffers: an initial list of HistoryBufferMax values + + Returns: + The updated max_history and max_buffers sets. + """ + vollog.debug( + f"Possible CommandHistorySize values before checking Registry: {max_history}" + ) + vollog.debug( + f"Possible HistoryBufferMax values before checking Registry: {max_buffers}" + ) + + for hive in hivelist.HiveList.list_hives( + context=context, + base_config_path=config_path, + layer_name=kernel_layer_name, + symbol_table=kernel_symbol_table_name, + hive_offsets=None, + ): + try: + for value in hive.get_key("Console").get_values(): + val_name = str(value.get_name()) + if val_name == "HistoryBufferSize": + max_history.add(value.decode_data()) + elif val_name == "NumberOfHistoryBuffers": + max_buffers.add(value.decode_data()) + except: + continue + + return max_history, max_buffers + + def _generator( + self, procs: Generator[interfaces.objects.ObjectInterface, None, None] + ): + """ + Generates the console information to use in rendering + + Args: + procs: the process list filtered to conhost.exe instances + """ + + kernel = self.context.modules[self.config["kernel"]] + + max_history = set(self.config.get("max_history", [50])) + max_buffers = set(self.config.get("max_buffers", [4])) + no_registry = self.config.get("no_registry") + + if no_registry is False: + max_history, max_buffers = self.get_console_settings_from_registry( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + max_history, + max_buffers, + ) + + vollog.debug(f"Possible CommandHistorySize values: {max_history}") + vollog.debug(f"Possible HistoryBufferMax values: {max_buffers}") + + for proc, console_info, console_properties in self.get_console_info( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + self.config_path, + procs, + max_history, + max_buffers, + ): + process_name = utility.array_to_string(proc.ImageFileName) + + if console_info and console_properties: + for console_property in console_properties: + yield ( + 0, + ( + proc.UniqueProcessId, + process_name, + format_hints.Hex(console_info.vol.offset), + console_property["name"], + ( + renderers.NotApplicableValue() + if console_property["address"] is None + else format_hints.Hex(console_property["address"]) + ), + str(console_property["data"]), + ), + ) + + def _conhost_proc_filter(self, proc): + """ + Used to filter to only conhost.exe processes + """ + process_name = utility.array_to_string(proc.ImageFileName) + + return process_name != "conhost.exe" + + def run(self): + kernel = self.context.modules[self.config["kernel"]] + + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("ConsoleInfo", format_hints.Hex), + ("Property", str), + ("Address", format_hints.Hex), + ("Data", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=self._conhost_proc_filter, + ) + ), + ) diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json new file mode 100644 index 000000000..3ae41fdd7 --- /dev/null +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json @@ -0,0 +1,595 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "short": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_CONSOLE_INFORMATION": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 24 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 26 + }, + "CommandHistorySize": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 136 + }, + "HistoryBufferMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 140 + }, + "OriginalTitle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1616 + }, + "Title": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1552 + }, + "GetScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1680 + }, + "CurrentScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1296 + }, + "ConsoleProcessList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 1272 + }, + "ProcessCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1280 + }, + "HistoryList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 9176 + }, + "HistoryBufferCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 9184 + }, + "ExeAliasList": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 9232 + } + }, + "kind": "struct", + "size": 140 + }, + "_VECTOR": { + "fields": { + "Begin": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 0 + }, + "End": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 8 + }, + "EndCapacity": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 32 + }, + "_CONSOLE_PROCESS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ConsoleProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_CONSOLE_PROCESS_HANDLE": { + "fields": { + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 52 + }, + "_CONSOLE_PROCESS": { + "fields": { + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 28 + }, + "ThreadId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + }, + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND_HISTORY": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "CommandBucket": { + "type": { + "kind": "struct", + "name": "_VECTOR" + }, + "offset": 16 + }, + "CommandCountMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 40 + }, + "Application": { + "type": { + "kind": "struct", + "name": "_COMMAND" + }, + "offset": 48 + }, + "ConsoleProcessHandle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS_HANDLE" + } + }, + "offset": 80 + }, + "Flags": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 88 + }, + "LastDisplayed": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 92 + } + }, + "kind": "struct", + "size": 96 + }, + "_SCREEN_INFORMATION": { + "fields": { + "TextBufferInfo": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 56 + }, + "Next": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 64 + } + }, + "kind": "struct", + "size": 72 + }, + "_ROW_POINTER": { + "fields": { + "Row": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + + }, + "_ROWS_ARRAY": { + "fields": { + "Rows": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_ROW_POINTER" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_TEXT_BUFFER_INFO": { + "fields": { + "BufferDeque": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_DEQUE" + } + }, + "offset": 0 + }, + "BufferRows": { + "type": { + "kind": "pointer", + "name": "void", + "subtype": { + "kind": "struct", + "name": "_ROWS_ARRAY" + } + }, + "offset": 16 + }, + "BufferCapacity": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 40 + }, + "ThisBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 48 + }, + "BufferEnd": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 54 + }, + "BufferLastIndex": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 56 + }, + "BufferStart": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 72 + }, + "_CHAR_ROW_CELL": { + "fields": { + "Text": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "DbcsAttribute": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 2 + } + }, + "kind": "struct", + "size": 3 + }, + "_CHAR_ROW_CELL_ARRAY": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROW": { + "fields": { + "CharRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL_ARRAY" + } + }, + "offset": 8 + }, + "RowLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 64 + }, + "Index": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 72 + }, + "RowLength2": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 80 + }, + "TextBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 88 + } }, + "kind": "struct", + "size": 96 + }, + "_DEQUE": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle", + "datetime": "2024-07-31T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json new file mode 100644 index 000000000..b74862598 --- /dev/null +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json @@ -0,0 +1,595 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "short": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_CONSOLE_INFORMATION": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 24 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 26 + }, + "CommandHistorySize": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 136 + }, + "HistoryBufferMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 140 + }, + "OriginalTitle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1616 + }, + "Title": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1552 + }, + "GetScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1680 + }, + "CurrentScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1296 + }, + "ConsoleProcessList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 1272 + }, + "ProcessCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1280 + }, + "HistoryList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": -288 + }, + "HistoryBufferCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": -280 + }, + "ExeAliasList": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": -376 + } + }, + "kind": "struct", + "size": 140 + }, + "_VECTOR": { + "fields": { + "Begin": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 0 + }, + "End": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 8 + }, + "EndCapacity": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 32 + }, + "_CONSOLE_PROCESS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ConsoleProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_CONSOLE_PROCESS_HANDLE": { + "fields": { + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 52 + }, + "_CONSOLE_PROCESS": { + "fields": { + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 28 + }, + "ThreadId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + }, + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND_HISTORY": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "CommandBucket": { + "type": { + "kind": "struct", + "name": "_VECTOR" + }, + "offset": 16 + }, + "CommandCountMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 40 + }, + "Application": { + "type": { + "kind": "struct", + "name": "_COMMAND" + }, + "offset": 48 + }, + "ConsoleProcessHandle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS_HANDLE" + } + }, + "offset": 80 + }, + "Flags": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 88 + }, + "LastDisplayed": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 92 + } + }, + "kind": "struct", + "size": 96 + }, + "_SCREEN_INFORMATION": { + "fields": { + "TextBufferInfo": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 56 + }, + "Next": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 64 + } + }, + "kind": "struct", + "size": 72 + }, + "_ROW_POINTER": { + "fields": { + "Row": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + + }, + "_ROWS_ARRAY": { + "fields": { + "Rows": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_ROW_POINTER" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_TEXT_BUFFER_INFO": { + "fields": { + "BufferDeque": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_DEQUE" + } + }, + "offset": 0 + }, + "BufferRows": { + "type": { + "kind": "pointer", + "name": "void", + "subtype": { + "kind": "struct", + "name": "_ROWS_ARRAY" + } + }, + "offset": 16 + }, + "BufferCapacity": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 40 + }, + "ThisBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 48 + }, + "BufferEnd": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 54 + }, + "BufferLastIndex": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 56 + }, + "BufferStart": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 72 + }, + "_CHAR_ROW_CELL": { + "fields": { + "Text": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "DbcsAttribute": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 2 + } + }, + "kind": "struct", + "size": 3 + }, + "_CHAR_ROW_CELL_ARRAY": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROW": { + "fields": { + "CharRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL_ARRAY" + } + }, + "offset": 8 + }, + "RowLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 64 + }, + "Index": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 72 + }, + "RowLength2": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 80 + }, + "TextBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 88 + } }, + "kind": "struct", + "size": 96 + }, + "_DEQUE": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle", + "datetime": "2024-07-31T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py new file mode 100644 index 000000000..2db1f4cad --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -0,0 +1,303 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from volatility3.framework import objects +from volatility3.framework import constants + + +class ROW(objects.StructType): + """A Row Structure.""" + + def _valid_dbcs(self, c): + # TODO this need more research and testing + # https://github.com/search?q=repo%3Amicrosoft%2Fterminal+DbcsAttr&type=code + valid = c in ( + 0x0, + 0x1, + 0x2, + 0x20, + 0x28, + 0x30, + 0x48, + 0x60, + 0x80, + 0xF8, + 0xF0, + 0xA0, + ) + # if not valid: + # print("Bad Dbcs Attribute {}".format(hex(c))) + return valid + + def get_text(self, truncate=True): + """A convenience method to extract the text from the _ROW. The _ROW + contains a pointer CharRow to an array of CharRowCell objects. Each + CharRowCell contains the wide character and an attribute. Enumerating + self.CharRow.Chars and casting each character to unicode takes too long, + so this reads the whole row into a buffer, then extracts the text characters.""" + + layer = self._context.layers[self.vol.layer_name] + offset = self.CharRow.Chars.vol.offset + length = self.RowLength * 3 + char_row = layer.read(offset, length) + line = "" + try: + if char_row: + line = "".join( + # "{} {} =".format(char_row[i:i + 2].decode('utf-16le', errors='replace'), char_row[i+2]) if self._valid_dbcs(char_row[i+2]) else "" + ( + char_row[i : i + 2].decode("utf-16le", errors="replace") + if self._valid_dbcs(char_row[i + 2]) + else "" + ) + for i in range(0, len(char_row), 3) + ) + except Exception as e: + print(e) + line = "" + + if truncate: + return line.rstrip() + else: + return line + + +class SCREEN_INFORMATION(objects.StructType): + """A Screen Information Structure.""" + + @property + def ScreenX(self): + return self.TextBufferInfo.BufferRows.Rows[0].Row.RowLength2 + + @property + def ScreenY(self): + return self.TextBufferInfo.BufferCapacity + + def _truncate_rows(self, rows): + """To truncate empty rows at the end, walk the list + backwards and get the last non-empty row. Use that + row index to splice. Rows are created based on the + length given in the ROW structure, so empty rows will + be ''.""" + + non_empty_index = 0 + rows_traversed = False + + for index, row in enumerate(reversed(rows)): + # the string was created based on the length in the ROW structure so it shouldn't have any bad data + if len(row.rstrip()) > 0: + non_empty_index = index + break + rows_traversed = True + + if non_empty_index == 0 and rows_traversed: + rows = [] + else: + rows = rows[0 : len(rows) - non_empty_index] + + return rows + + def get_buffer(self, truncate_rows=True, truncate_lines=True): + """Get the screen buffer. + + The screen buffer is comprised of the screen's Y + coordinate which tells us the number of rows and + the X coordinate which tells us the width of each + row in characters. Windows 10 17763 changed from + a large text buffer to a grid of cells, with each + cell containing a single wide character in that + cell, stored in a CharRowCell object. + + @param truncate: True if the empty rows at the + end (i.e. bottom) of the screen buffer should be + supressed. + """ + rows = [] + + capacity = self.TextBufferInfo.BufferCapacity + start = self.TextBufferInfo.BufferStart + buffer_rows = self.TextBufferInfo.BufferRows.dereference() + buffer_rows.Rows.count = self.TextBufferInfo.BufferCapacity + + for i in range(capacity): + index = (start + i) % capacity + row = buffer_rows.Rows[index].Row + try: + text = row.get_text(truncate_lines) + rows.append(text) + except: + break + + if truncate_rows: + rows = self._truncate_rows(rows) + + if rows: + rows = ["=== Start of buffer ==="] + rows + ["=== End of buffer ==="] + else: + rows = ["=== No buffer data found ==="] + return rows + + +class CONSOLE_INFORMATION(objects.StructType): + """A Console Information Structure.""" + + @property + def ScreenBuffer(self): + return self.GetScreenBuffer + + def is_valid(self, max_buffers=4) -> bool: + """Determine if the structure is valid.""" + + # Last displayed must be between -1 and max + if self.HistoryBufferCount < 1 or self.HistoryBufferCount > max_buffers: + return False + + if not self.get_title() and not self.get_original_title(): + return False + + return True + + def get_screens(self): + """Generator for screens in the console. + + A console can have multiple screen buffers at a time, + but only the current/active one is displayed. + + Multiple screens are tracked using the singly-linked + list _SCREEN_INFORMATION.Next. + + See CreateConsoleScreenBuffer + """ + screens = [self.CurrentScreenBuffer] + + if self.ScreenBuffer not in screens: + screens.append(self.ScreenBuffer) + + seen = set() + + for screen in screens: + cur = screen + while cur and cur.vol.offset != 0 and cur.vol.offset not in seen: + cur.TextBufferInfo.BufferRows.Rows.count = ( + cur.TextBufferInfo.BufferCapacity + ) + yield cur + seen.add(cur.vol.offset) + cur = cur.Next + + def get_histories(self): + for cmd_hist in self.HistoryList.dereference().to_list( + f"{self.get_symbol_table_name()}{constants.BANG}_COMMAND_HISTORY", + "ListEntry", + ): + yield cmd_hist + + def get_processes(self): + for proc in self.ConsoleProcessList.dereference().to_list( + f"{self.get_symbol_table_name()}{constants.BANG}_CONSOLE_PROCESS_LIST", + "ListEntry", + ): + yield proc + + def get_title(self): + try: + return self.Title.dereference().cast( + "string", encoding="utf-16", errors="replace", max_length=512 + ) + except: + return "" + + def get_original_title(self): + try: + return self.OriginalTitle.dereference().cast( + "string", encoding="utf-16", errors="replace", max_length=512 + ) + except: + return "" + + +class COMMAND(objects.StructType): + """A Command Structure""" + + def get_command(self): + if self.Length < 8: + return self.Chars.cast( + "string", + encoding="utf-16", + errors="replace", + max_length=self.Length * 2, + ) + elif self.Length < 1024: + return self.Pointer.dereference().cast( + "string", encoding="utf-16", errors="replace", max_length=512 + ) + + +class COMMAND_HISTORY(objects.StructType): + """A Command History Structure.""" + + @property + def CommandCount(self): + command_type = self.get_symbol_table_name() + constants.BANG + "_COMMAND" + command_size = self._context.symbol_space.get_type(command_type).size + return int((self.CommandBucket.End - self.CommandBucket.Begin) / command_size) + + def get_application(self): + if self.Application.Length < 8: + return self.Application.Chars.cast( + "string", + encoding="utf-16", + errors="replace", + max_length=self.Application.Length * 2, + ) + elif self.Application.Length < 1024: + return self.Application.Pointer.dereference().cast( + "string", encoding="utf-16", errors="replace", max_length=512 + ) + + def scan_command_bucket(self, end=None): + """Brute force print all strings pointed to by the CommandBucket entries by + going to greater of EndCapacity or CommandCountMax*sizeof(_COMMAND)""" + + command_type = self.get_symbol_table_name() + constants.BANG + "_COMMAND" + command_history_size = self._context.symbol_space.get_type( + self.vol.type_name + ).size + command_size = self._context.symbol_space.get_type(command_type).size + if end is None: + end = max( + self.CommandBucket.EndCapacity, + self.CommandBucket.Begin + command_history_size * self.CommandCountMax, + ) + + for i, pointer in enumerate(range(self.CommandBucket.Begin, end, command_size)): + yield i, self._context.object(command_type, self.vol.layer_name, pointer) + + def get_commands(self): + """Generator for commands in the history buffer. + + The CommandBucket is an array of pointers to _COMMAND + structures. The array size is CommandCount. Once CommandCount + is reached, the oldest commands are cycled out and the + rest are coalesced. + """ + + for i, cmd in self.scan_command_bucket(self.CommandBucket.End): + yield i, cmd + + +win10_x64_class_types = { + "_ROW": ROW, + "_SCREEN_INFORMATION": SCREEN_INFORMATION, + "_CONSOLE_INFORMATION": CONSOLE_INFORMATION, + "_COMMAND_HISTORY": COMMAND_HISTORY, + "_COMMAND": COMMAND, +} +class_types = { + "_ROW": ROW, + "_SCREEN_INFORMATION": SCREEN_INFORMATION, + "_CONSOLE_INFORMATION": CONSOLE_INFORMATION, + "_COMMAND_HISTORY": COMMAND_HISTORY, + "_COMMAND": COMMAND, +} From e6308a6035156cab5abb6f0cdf537fb75e5881e8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 1 Aug 2024 21:04:24 +0100 Subject: [PATCH 289/979] Make suggested changes by gcmoreira --- volatility3/cli/volargparse.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index 2bd53077b..5ce2646ed 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -100,6 +100,11 @@ class HelpfulArgParser(argparse.ArgumentParser): return len(match.group(1)) def _check_value(self, action: argparse.Action, value: Any) -> None: + """This is called to ensure a value is correct/valid + This fails when we want to accept partial values for the plugin name, + so we disable the check (which will throw ArgumentErrors for failed checks) + but only for our plugin subparser, so all other arguments are checked correctly + """ if not isinstance(action, HelpfulSubparserAction): - return super()._check_value(action, value) + super()._check_value(action, value) return None From 0bda1543854d8f92e7fb2bf4c5eff3afdf117819 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 1 Aug 2024 21:08:18 +0100 Subject: [PATCH 290/979] Clarify the documentation a little --- volatility3/cli/volargparse.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index 5ce2646ed..fd61ddce0 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -101,9 +101,16 @@ class HelpfulArgParser(argparse.ArgumentParser): def _check_value(self, action: argparse.Action, value: Any) -> None: """This is called to ensure a value is correct/valid - This fails when we want to accept partial values for the plugin name, - so we disable the check (which will throw ArgumentErrors for failed checks) - but only for our plugin subparser, so all other arguments are checked correctly + + In normal operation, it would check that a value provided is valid and return None + If it was not valid, it would throw an ArgumentError + + When people provide a partial plugin name, we want to look for a matching plugin name + which happens in the HelpfulSubparserAction's __call_method + + To get there without tripping the check_value failure, we have to prevent the exception + being thrown when the value is a HelpfulSubparserAction. This therefore affects no other + checks for normal parameters. """ if not isinstance(action, HelpfulSubparserAction): super()._check_value(action, value) From d19013c85261ea48dd774dcda66dcb8d1b36782d Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:19:24 -0500 Subject: [PATCH 291/979] fixed typo, updated plugin docstring, and updated comment --- volatility3/framework/plugins/windows/psxview.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 918eb44ba..cab27f3e3 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -17,17 +17,14 @@ vollog = logging.getLogger(__name__) class PsXView(plugins.PluginInterface): - """Lists all processes found via 6 of the methods described in \"The Art of Memory Forensics,\" which may help + """Lists all processes found via four of the methods described in \"The Art of Memory Forensics,\" which may help identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this plugin's output in a terminal.""" # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality # which the original plugin used to do it. - # I don't think it's worth including the sessions method either because both the original psxview plugin - # and Volatility3's sessions plugin begin with the list of processes found by PsList. - # The original psxview plugin's session code essentially just filters the pslist for processes - # whose session ID is not None. I've matched this in my code, but again, it doesn't seem worth including. + # The sessions method is omitted because it begins with the list of processes found by Pslist anyway. # Lastly, I've omitted the pspcid method because I could not for the life of me get it to work. I saved the # code I do have from it, and will happily share it if anyone else wants to add it. @@ -64,7 +61,7 @@ class PsXView(plugins.PluginInterface): ), requirements.BooleanRequirement( name="physical-offsets", - description="List processes with phyiscall offsets instead of virtual offsets.", + description="List processes with physical offsets instead of virtual offsets.", optional=True, ), ] From fdb131f870f5a3ef97dfe5f33e3ee05cf17d83f2 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 1 Aug 2024 16:27:48 -0500 Subject: [PATCH 292/979] #816 - add cmdscan and bug fixes --- .../framework/plugins/windows/cmdscan.py | 327 ++++++++++++++++++ .../framework/plugins/windows/consoles.py | 2 +- .../consoles-win10-20348-2461-x64.json | 2 +- .../consoles/consoles-win10-20348-x64.json | 2 +- .../symbols/windows/extensions/consoles.py | 31 +- 5 files changed, 360 insertions(+), 4 deletions(-) create mode 100644 volatility3/framework/plugins/windows/cmdscan.py diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py new file mode 100644 index 000000000..86a34a4f8 --- /dev/null +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -0,0 +1,327 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +# This module attempts to locate windows console histories. + +import logging +import struct +from typing import Tuple, Generator, Set, Dict, Any, List, Optional + +from volatility3.framework import interfaces +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import scanners +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist, vadinfo, info, verinfo, consoles +from volatility3.plugins.windows.registry import hivelist + + +try: + import capstone + + has_capstone = True +except ImportError: + has_capstone = False + +vollog = logging.getLogger(__name__) + + +class CmdScan(interfaces.plugins.PluginInterface): + """Looks for Windows Command History lists""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), + requirements.BooleanRequirement( + name="no_registry", + description="Don't search the registry for possible values of CommandHistorySize", + optional=True, + default=False, + ), + requirements.ListRequirement( + name="max_history", + element_type=int, + description="CommandHistorySize values to search for.", + optional=True, + default=[50], + ), + ] + + @classmethod + def get_filtered_vads( + cls, conhost_proc: interfaces.context.ContextInterface, size_filter: Optional[int]=0x40000000 + ) -> List[Tuple[int, int]]: + """ + Returns vads of a process with smaller than size_filter + + Args: + conhost_proc: the process object for conhost.exe + + Returns: + A list of tuples of: + vad_base: the base address + vad_size: the size of the VAD + """ + vads = [] + for vad in conhost_proc.get_vad_root().traverse(): + base = vad.get_start() + if vad.get_size() < size_filter: + vads.append((base, vad.get_size())) + + return vads + + @classmethod + def get_command_history( + cls, + context: interfaces.context.ContextInterface, + kernel_layer_name: str, + kernel_table_name: str, + config_path: str, + procs: Generator[interfaces.objects.ObjectInterface, None, None], + max_history: Set[int], + ) -> Tuple[ + interfaces.context.ContextInterface, + interfaces.context.ContextInterface, + Dict[str, Any], + ]: + """Gets the list of commands from each Command History structure + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + kernel_layer_name: The name of the layer on which to operate + kernel_table_name: The name of the table containing the kernel symbols + config_path: The config path where to find symbol files + procs: list of process objects + max_history: an initial set of CommandHistorySize values + + Returns: + The conhost process object, the command history structure, a dictionary of properties for + that command historyn structure. + """ + + conhost_symbol_table = consoles.Consoles.create_conhost_symbol_table( + context, kernel_layer_name, kernel_table_name, config_path + ) + + for conhost_proc, proc_layer_name in consoles.Consoles.find_conhost_proc(procs): + if not conhost_proc: + vollog.info( + "Unable to find a valid conhost.exe process in the process list. Analysis cannot proceed." + ) + continue + vollog.debug( + f"Found conhost process {conhost_proc} with pid {conhost_proc.UniqueProcessId}" + ) + + conhostexe_base, conhostexe_size = consoles.Consoles.find_conhostexe(conhost_proc) + if not conhostexe_base: + vollog.info( + "Unable to find the location of conhost.exe. Analysis cannot proceed." + ) + continue + vollog.debug(f"Found conhost.exe base at {conhostexe_base:#x}") + + proc_layer = context.layers[proc_layer_name] + + conhost_module = context.module( + conhost_symbol_table, proc_layer_name, offset=conhostexe_base + ) + + sections = cls.get_filtered_vads(conhost_proc) + # scan for potential _COMMAND_HISTORY structures by using the CommandHistorySize + for max_history_value in max_history: + max_history_bytes = struct.pack("H", max_history_value) + vollog.debug( + f"Scanning for CommandHistorySize value: {max_history_bytes}" + ) + for address in proc_layer.scan( + context, + scanners.BytesScanner(max_history_bytes), + sections=sections, + ): + command_history_properties = [] + + try: + command_history = conhost_module.object( + "_COMMAND_HISTORY", + offset=address + - conhost_module.get_type( + "_COMMAND_HISTORY" + ).relative_child_offset("CommandCountMax"), + absolute=True, + ) + + if not command_history.is_valid(max_history_value): + continue + + vollog.debug( + f"Getting Command History properties for {command_history}" + ) + command_history_properties.append( + { + "name": f"_COMMAND_HISTORY.Application", + "address": command_history.Application.vol.offset, + "data": command_history.get_application(), + } + ) + command_history_properties.append( + { + "name": f"_COMMAND_HISTORY.ProcessHandle", + "address": command_history.ConsoleProcessHandle.ProcessHandle.vol.offset, + "data": hex( + command_history.ConsoleProcessHandle.ProcessHandle + ), + } + ) + command_history_properties.append( + { + "name": f"_COMMAND_HISTORY.CommandCount", + "address": None, + "data": command_history.CommandCount, + } + ) + command_history_properties.append( + { + "name": f"_COMMAND_HISTORY.LastDisplayed", + "address": command_history.LastDisplayed.vol.offset, + "data": command_history.LastDisplayed, + } + ) + command_history_properties.append( + { + "name": f"_COMMAND_HISTORY.CommandCountMax", + "address": command_history.CommandCountMax.vol.offset, + "data": command_history.CommandCountMax, + } + ) + + command_history_properties.append( + { + "name": f"_COMMAND_HISTORY.CommandBucket", + "address": command_history.CommandBucket.vol.offset, + "data": "", + } + ) + for ( + cmd_index, + bucket_cmd, + ) in command_history.scan_command_bucket(): + try: + command_history_properties.append( + { + "name": f"_COMMAND_HISTORY.CommandBucket_Command_{cmd_index}", + "address": bucket_cmd.vol.offset, + "data": bucket_cmd.get_command(), + } + ) + except Exception as e: + vollog.debug( + f"reading {bucket_cmd} encountered exception {e}" + ) + except Exception as e: + vollog.debug( + f"reading {command_history} encountered exception {e}" + ) + + yield conhost_proc, command_history, command_history_properties + + def _generator( + self, procs: Generator[interfaces.objects.ObjectInterface, None, None] + ): + """ + Generates the command history to use in rendering + + Args: + procs: the process list filtered to conhost.exe instances + """ + + kernel = self.context.modules[self.config["kernel"]] + + max_history = set(self.config.get("max_history", [50])) + no_registry = self.config.get("no_registry") + + if no_registry is False: + max_history, _max_buffers = consoles.Consoles.get_console_settings_from_registry( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + max_history, + [], + ) + + vollog.debug(f"Possible CommandHistorySize values: {max_history}") + + for proc, command_history, command_history_properties in self.get_command_history( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + self.config_path, + procs, + max_history, + ): + process_name = utility.array_to_string(proc.ImageFileName) + + if command_history and command_history_properties: + for command_history_property in command_history_properties: + yield ( + 0, + ( + proc.UniqueProcessId, + process_name, + format_hints.Hex(command_history.vol.offset), + command_history_property["name"], + ( + renderers.NotApplicableValue() + if command_history_property["address"] is None + else format_hints.Hex(command_history_property["address"]) + ), + str(command_history_property["data"]), + ), + ) + + def _conhost_proc_filter(self, proc): + """ + Used to filter to only conhost.exe processes + """ + process_name = utility.array_to_string(proc.ImageFileName) + + return process_name != "conhost.exe" + + def run(self): + kernel = self.context.modules[self.config["kernel"]] + + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("ConsoleInfo", format_hints.Hex), + ("Property", str), + ("Address", format_hints.Hex), + ("Data", str), + ], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=self._conhost_proc_filter, + ) + ), + ) diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index f6bbb3ae7..509b6ae9d 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -363,7 +363,7 @@ class Consoles(interfaces.plugins.PluginInterface): interfaces.context.ContextInterface, Dict[str, Any], ]: - """Extracts the cmdline from PEB + """Gets the Console Information structure and its related properties for each conhost process Args: context: The context to retrieve required elements (layers, symbol tables) from diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json index 3ae41fdd7..a4adf8028 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json @@ -232,7 +232,7 @@ "kind": "base", "name": "unsigned int" }, - "offset": 20 + "offset": 24 } }, "kind": "struct", diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json index b74862598..5cc7ab6f5 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json @@ -232,7 +232,7 @@ "kind": "base", "name": "unsigned int" }, - "offset": 20 + "offset": 24 } }, "kind": "struct", diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 2db1f4cad..279ee723e 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -220,6 +220,12 @@ class CONSOLE_INFORMATION(objects.StructType): class COMMAND(objects.StructType): """A Command Structure""" + def is_valid(self): + if self.Length < 1 or self.Allocated < 1 or self.Length > 1024 or self.Allocated > 1024: + return False + + return True + def get_command(self): if self.Length < 8: return self.Chars.cast( @@ -243,6 +249,26 @@ class COMMAND_HISTORY(objects.StructType): command_size = self._context.symbol_space.get_type(command_type).size return int((self.CommandBucket.End - self.CommandBucket.Begin) / command_size) + @property + def ProcessHandle(self): + """ Allow ProcessHandle to be referenced regardless of OS version """ + return self.ConsoleProcessHandle.ProcessHandle + + def is_valid(self, max_history=50): + # The count must be between zero and max + if self.CommandCount < 0 or self.CommandCount > max_history: + return False + + # Last displayed must be between -1 and max + if self.LastDisplayed < -1 or self.LastDisplayed > max_history: + return False + + # Process handle must be a valid pid + if self.ProcessHandle <= 0 or self.ProcessHandle > 0xFFFF or self.ProcessHandle % 4 != 0: + return False + + return True + def get_application(self): if self.Application.Length < 8: return self.Application.Chars.cast( @@ -265,6 +291,7 @@ class COMMAND_HISTORY(objects.StructType): self.vol.type_name ).size command_size = self._context.symbol_space.get_type(command_type).size + if end is None: end = max( self.CommandBucket.EndCapacity, @@ -272,7 +299,9 @@ class COMMAND_HISTORY(objects.StructType): ) for i, pointer in enumerate(range(self.CommandBucket.Begin, end, command_size)): - yield i, self._context.object(command_type, self.vol.layer_name, pointer) + cmd = self._context.object(command_type, self.vol.layer_name, pointer) + if cmd.is_valid(): + yield i, cmd def get_commands(self): """Generator for commands in the history buffer. From 98c0094da5cfb8a99c9e211004952c9104c619cd Mon Sep 17 00:00:00 2001 From: qpalzmz112 <68213464+qpalzmz112@users.noreply.github.com> Date: Thu, 1 Aug 2024 18:51:11 -0500 Subject: [PATCH 293/979] Updated unpacked variable names --- volatility3/framework/plugins/windows/psxview.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index cab27f3e3..71919c410 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -92,7 +92,7 @@ class PsXView(plugins.PluginInterface): layer_name = kernel.layer_name try: - _, _, offset, _, _ = list( + _original_offset, _original_length, offset, _length, _layer_name = list( self.context.layers[layer_name].mapping(offset=offset, length=0) )[0] except exceptions.PagedInvalidAddressException: @@ -190,15 +190,15 @@ class PsXView(plugins.PluginInterface): # print results # list of lists of offsets - todo_offsets = [list(processes[source].keys()) for source in processes] + offsets = [list(processes[source].keys()) for source in processes] # flatten to one list - todo_offsets = sum(todo_offsets, []) + offsets = sum(offsets, []) # remove duplicates - todo_offsets = set(todo_offsets) + offsets = set(offsets) - for offset in todo_offsets: + for offset in offsets: proc = None in_sources = {src: False for src in processes} From 1dcaf9c0b15dfd00f2b35846488f19f3982ae5b2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 13:50:51 +1000 Subject: [PATCH 294/979] PR review fixes: Rename method name from private to internal --- .../framework/symbols/linux/extensions/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index be31e298c..599fedb6f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1839,7 +1839,7 @@ class inode(objects.StructType): """Returns the inode number""" return int(self.i_ino) - def ___time_member_to_datetime(self, member) -> datetime: + def _time_member_to_datetime(self, member) -> datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 @@ -1865,7 +1865,7 @@ class inode(objects.StructType): Returns: A datetime with the inode's last access time """ - return self.___time_member_to_datetime("i_atime") + return self._time_member_to_datetime("i_atime") def get_modification_time(self) -> datetime: """Returns the inode's last modification time @@ -1875,7 +1875,7 @@ class inode(objects.StructType): A datetime with the inode's last data modification time """ - return self.___time_member_to_datetime("i_mtime") + return self._time_member_to_datetime("i_mtime") def get_change_time(self) -> datetime: """Returns the inode's last change time @@ -1884,7 +1884,7 @@ class inode(objects.StructType): Returns: A datetime with the inode's last change time """ - return self.___time_member_to_datetime("i_ctime") + return self._time_member_to_datetime("i_ctime") def get_file_mode(self) -> str: """Returns the inode's file mode as string of the form '-rwxrwxrwx'. From 79529fb8153b3a58b4a1d1c6192cb92cf107800f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 13:50:51 +1000 Subject: [PATCH 295/979] PR review fixes: Rename method name from private to internal --- .../framework/symbols/linux/extensions/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index be31e298c..599fedb6f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1839,7 +1839,7 @@ class inode(objects.StructType): """Returns the inode number""" return int(self.i_ino) - def ___time_member_to_datetime(self, member) -> datetime: + def _time_member_to_datetime(self, member) -> datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 @@ -1865,7 +1865,7 @@ class inode(objects.StructType): Returns: A datetime with the inode's last access time """ - return self.___time_member_to_datetime("i_atime") + return self._time_member_to_datetime("i_atime") def get_modification_time(self) -> datetime: """Returns the inode's last modification time @@ -1875,7 +1875,7 @@ class inode(objects.StructType): A datetime with the inode's last data modification time """ - return self.___time_member_to_datetime("i_mtime") + return self._time_member_to_datetime("i_mtime") def get_change_time(self) -> datetime: """Returns the inode's last change time @@ -1884,7 +1884,7 @@ class inode(objects.StructType): Returns: A datetime with the inode's last change time """ - return self.___time_member_to_datetime("i_ctime") + return self._time_member_to_datetime("i_ctime") def get_file_mode(self) -> str: """Returns the inode's file mode as string of the form '-rwxrwxrwx'. From 930d29046ad8c1cd47d6167e84118d78a3e549a6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 14:15:01 +1000 Subject: [PATCH 296/979] PR review fixes: Avoid using renderers in core functions. --- .../framework/symbols/linux/extensions/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 599fedb6f..1b5e1d286 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -7,10 +7,10 @@ import logging import stat from datetime import datetime import socket as socket_module -from typing import Generator, Iterable, Iterator, Optional, Tuple, List +from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union from volatility3.framework import constants, exceptions, objects, interfaces, symbols -from volatility3.framework import renderers +from volatility3.framework.renderers import conversion from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1770,7 +1770,7 @@ class timespec64(objects.StructType): def to_datetime(self) -> datetime: """Returns the respective aware datetime""" - dt = renderers.conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) + dt = conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) return dt @@ -1812,7 +1812,7 @@ class inode(objects.StructType): """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 - def get_inode_type(self) -> str: + def get_inode_type(self) -> Union[str, None]: """Returns inode type name Returns: @@ -1833,7 +1833,7 @@ class inode(objects.StructType): elif self.is_block(): return "BLK" else: - return renderers.UnparsableValue() + return None def get_inode_number(self) -> int: """Returns the inode number""" @@ -1843,7 +1843,7 @@ class inode(objects.StructType): if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 - return renderers.conversion.unixtime_to_datetime( + return conversion.unixtime_to_datetime( self.member(f"{member}_sec") + self.has_member(f"{member}_nsec") / 1e9 ) elif self.has_member(f"__{member}"): From efba3a1b7336d5b31f7ac5ee1d8e99d95bcd74f6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 14:17:57 +1000 Subject: [PATCH 297/979] PR review fixes: Convert inode's is_* functions to properties --- .../symbols/linux/extensions/__init__.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1b5e1d286..00f6730eb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1780,34 +1780,42 @@ class inode(objects.StructType): # pointer, will easily cause an integer overflow here. return self.i_ino > 0 and self.i_count.counter >= 0 + @property def is_dir(self) -> bool: """Returns True if the inode is a directory""" return stat.S_ISDIR(self.i_mode) != 0 + @property def is_reg(self) -> bool: """Returns True if the inode is a regular file""" return stat.S_ISREG(self.i_mode) != 0 + @property def is_link(self) -> bool: """Returns True if the inode is a symlink""" return stat.S_ISLNK(self.i_mode) != 0 + @property def is_fifo(self) -> bool: """Returns True if the inode is a FIFO""" return stat.S_ISFIFO(self.i_mode) != 0 + @property def is_sock(self) -> bool: """Returns True if the inode is a socket""" return stat.S_ISSOCK(self.i_mode) != 0 + @property def is_block(self) -> bool: """Returns True if the inode is a block device""" return stat.S_ISBLK(self.i_mode) != 0 + @property def is_char(self) -> bool: """Returns True if the inode is a char device""" return stat.S_ISCHR(self.i_mode) != 0 + @property def is_sticky(self) -> bool: """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 @@ -1818,19 +1826,19 @@ class inode(objects.StructType): Returns: The inode type name """ - if self.is_dir(): + if self.is_dir: return "DIR" - elif self.is_reg(): + elif self.is_reg: return "REG" - elif self.is_link(): + elif self.is_link: return "LNK" - elif self.is_fifo(): + elif self.is_fifo: return "FIFO" - elif self.is_sock(): + elif self.is_sock: return "SOCK" - elif self.is_char(): + elif self.is_char: return "CHR" - elif self.is_block(): + elif self.is_block: return "BLK" else: return None From b8d68b9a5ee0c643b68eaa9f33bd051279374eb3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 14:15:01 +1000 Subject: [PATCH 298/979] PR review fixes: Avoid using renderers in core functions. --- .../framework/symbols/linux/extensions/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 599fedb6f..1b5e1d286 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -7,10 +7,10 @@ import logging import stat from datetime import datetime import socket as socket_module -from typing import Generator, Iterable, Iterator, Optional, Tuple, List +from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union from volatility3.framework import constants, exceptions, objects, interfaces, symbols -from volatility3.framework import renderers +from volatility3.framework.renderers import conversion from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1770,7 +1770,7 @@ class timespec64(objects.StructType): def to_datetime(self) -> datetime: """Returns the respective aware datetime""" - dt = renderers.conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) + dt = conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) return dt @@ -1812,7 +1812,7 @@ class inode(objects.StructType): """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 - def get_inode_type(self) -> str: + def get_inode_type(self) -> Union[str, None]: """Returns inode type name Returns: @@ -1833,7 +1833,7 @@ class inode(objects.StructType): elif self.is_block(): return "BLK" else: - return renderers.UnparsableValue() + return None def get_inode_number(self) -> int: """Returns the inode number""" @@ -1843,7 +1843,7 @@ class inode(objects.StructType): if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 - return renderers.conversion.unixtime_to_datetime( + return conversion.unixtime_to_datetime( self.member(f"{member}_sec") + self.has_member(f"{member}_nsec") / 1e9 ) elif self.has_member(f"__{member}"): From 933a41fa3a60f3f02185b530a1a710b6dcae895c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 14:17:57 +1000 Subject: [PATCH 299/979] PR review fixes: Convert inode's is_* functions to properties --- .../symbols/linux/extensions/__init__.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1b5e1d286..00f6730eb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1780,34 +1780,42 @@ class inode(objects.StructType): # pointer, will easily cause an integer overflow here. return self.i_ino > 0 and self.i_count.counter >= 0 + @property def is_dir(self) -> bool: """Returns True if the inode is a directory""" return stat.S_ISDIR(self.i_mode) != 0 + @property def is_reg(self) -> bool: """Returns True if the inode is a regular file""" return stat.S_ISREG(self.i_mode) != 0 + @property def is_link(self) -> bool: """Returns True if the inode is a symlink""" return stat.S_ISLNK(self.i_mode) != 0 + @property def is_fifo(self) -> bool: """Returns True if the inode is a FIFO""" return stat.S_ISFIFO(self.i_mode) != 0 + @property def is_sock(self) -> bool: """Returns True if the inode is a socket""" return stat.S_ISSOCK(self.i_mode) != 0 + @property def is_block(self) -> bool: """Returns True if the inode is a block device""" return stat.S_ISBLK(self.i_mode) != 0 + @property def is_char(self) -> bool: """Returns True if the inode is a char device""" return stat.S_ISCHR(self.i_mode) != 0 + @property def is_sticky(self) -> bool: """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 @@ -1818,19 +1826,19 @@ class inode(objects.StructType): Returns: The inode type name """ - if self.is_dir(): + if self.is_dir: return "DIR" - elif self.is_reg(): + elif self.is_reg: return "REG" - elif self.is_link(): + elif self.is_link: return "LNK" - elif self.is_fifo(): + elif self.is_fifo: return "FIFO" - elif self.is_sock(): + elif self.is_sock: return "SOCK" - elif self.is_char(): + elif self.is_char: return "CHR" - elif self.is_block(): + elif self.is_block: return "BLK" else: return None From d34b1ded3e673e2d311d14c8e104668d7ebac78c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 19:46:23 +1000 Subject: [PATCH 300/979] PR review fixes: Remove get_inode_number. It's better to use the type's original member name and handle the casting on the consumer side. --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 00f6730eb..05679523f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1843,10 +1843,6 @@ class inode(objects.StructType): else: return None - def get_inode_number(self) -> int: - """Returns the inode number""" - return int(self.i_ino) - def _time_member_to_datetime(self, member) -> datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 From ed208347630428e21dec91f90eb431ed02595bc3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 2 Aug 2024 19:46:23 +1000 Subject: [PATCH 301/979] PR review fixes: Remove get_inode_number. It's better to use the type's original member name and handle the casting on the consumer side. --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 00f6730eb..05679523f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1843,10 +1843,6 @@ class inode(objects.StructType): else: return None - def get_inode_number(self) -> int: - """Returns the inode number""" - return int(self.i_ino) - def _time_member_to_datetime(self, member) -> datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 From 230ea09728dc9b756e27936544f03d43647cc0ba Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Aug 2024 17:52:51 +0200 Subject: [PATCH 302/979] Updating code following #1230 merge --- volatility3/framework/plugins/linux/lsof.py | 26 ++++++++--------- .../framework/symbols/linux/__init__.py | 28 ++++++++----------- .../symbols/linux/extensions/__init__.py | 14 ++++++++++ 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index c1de48c1a..fa9d2bf61 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -21,7 +21,6 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) @classmethod @@ -53,7 +52,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): symbol_table: str, filter_func: Callable[[int], bool] = lambda _: False, ): - linuxutils_symbol_table = None # type: ignore + linuxutils_symbol_table = None for task in pslist.PsList.list_tasks(context, symbol_table, filter_func): if linuxutils_symbol_table is None: if constants.BANG not in task.vol.type_name: @@ -71,21 +70,17 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): fd_num, filp, full_path = fd_fields inode_metadata = linux.LinuxUtilities.get_inode_metadata(context, filp) try: - inode_num, file_size, imode, ctime, mtime, atime = next( + inode_num, itype, file_size, imode, ctime, mtime, atime = next( inode_metadata ) except Exception as e: vollog.warning( f"Can't get inode metadata for file descriptor {fd_num}: {e}" ) - # Yield NotAvailableValue for each field in case of an exception - inode_num = renderers.NotAvailableValue() - file_size = renderers.NotAvailableValue() - imode = renderers.NotAvailableValue() - ctime = renderers.NotAvailableValue() - mtime = renderers.NotAvailableValue() - atime = renderers.NotAvailableValue() - yield pid, task_comm, task, fd_num, filp, full_path, inode_num, imode, ctime, mtime, atime, file_size + inode_num = itype = file_size = imode = ctime = mtime = atime = ( + renderers.NotAvailableValue() + ) + yield pid, task_comm, task, fd_num, filp, full_path, inode_num, itype, imode, ctime, mtime, atime, file_size def _generator(self, pids, symbol_table): filter_func = pslist.PsList.create_pid_filter(pids) @@ -100,6 +95,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): filp, full_path, inode_num, + itype, imode, ctime, mtime, @@ -112,6 +108,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): fd_num, full_path, inode_num, + itype, imode, ctime, mtime, @@ -130,6 +127,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ("FD", int), ("Path", str), ("Inode", int), + ("Type", str), ("Mode", str), ("Changed", datetime.datetime), ("Modified", datetime.datetime), @@ -144,6 +142,6 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): for row in self._generator(pids, symbol_table): _depth, row_data = row description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[3]}"' - yield description, timeliner.TimeLinerType.CHANGED, row_data[6] - yield description, timeliner.TimeLinerType.MODIFIED, row_data[7] - yield description, timeliner.TimeLinerType.ACCESSED, row_data[8] + yield description, timeliner.TimeLinerType.CHANGED, row_data[7] + yield description, timeliner.TimeLinerType.MODIFIED, row_data[8] + yield description, timeliner.TimeLinerType.ACCESSED, row_data[9] diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index a96fe9d2f..d52c43dae 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -280,23 +280,19 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): A helper function that gets the inodes metadata from a file descriptor """ dentry = filp.get_dentry() - if dentry != 0: + if dentry: inode_object = dentry.d_inode - inode_num = inode_object.i_ino - file_size = inode_object.i_size # file size in bytes - imode = stat.filemode(inode_object.i_mode) # file type & Permissions - - # Timestamps - ctime = datetime.datetime.fromtimestamp( - inode_object.i_ctime.tv_sec - ) # last change time - mtime = datetime.datetime.fromtimestamp( - inode_object.i_mtime.tv_sec - ) # last modify time - atime = datetime.datetime.fromtimestamp( - inode_object.i_atime.tv_sec - ) # last access time - yield inode_num, file_size, imode, ctime, mtime, atime + if inode_object and inode_object.is_valid(): + itype = inode_object.get_inode_type() or "?" + yield ( + inode_object.i_ino, + itype, + inode_object.i_size, + inode_object.get_file_mode(), + inode_object.get_change_time(), + inode_object.get_modification_time(), + inode_object.get_access_time(), + ) @classmethod def mask_mods_list( diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 05679523f..0ee6e7d95 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1820,6 +1820,16 @@ class inode(objects.StructType): """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 + @property + def is_whiteout(self) -> bool: + """Returns True if the inode is a whiteout""" + return (self.i_mode & 0o140000) == 0o140000 + + @property + def is_overlay(self) -> bool: + """Returns True if the inode is an overlay""" + return (self.i_mode & 0o40000) == 0o40000 + def get_inode_type(self) -> Union[str, None]: """Returns inode type name @@ -1840,6 +1850,10 @@ class inode(objects.StructType): return "CHR" elif self.is_block: return "BLK" + elif self.is_whiteout: + return "WHT" + elif self.is_overlay: + return "OVL" else: return None From 60b1c49e49864ce7cb5ae3a3491b6a7e9e40eef3 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Aug 2024 18:10:11 +0200 Subject: [PATCH 303/979] removing test code --- .../framework/symbols/linux/extensions/__init__.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0ee6e7d95..06d2e2bf4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1820,16 +1820,6 @@ class inode(objects.StructType): """Returns True if the sticky bit is set""" return (self.i_mode & stat.S_ISVTX) != 0 - @property - def is_whiteout(self) -> bool: - """Returns True if the inode is a whiteout""" - return (self.i_mode & 0o140000) == 0o140000 - - @property - def is_overlay(self) -> bool: - """Returns True if the inode is an overlay""" - return (self.i_mode & 0o40000) == 0o40000 - def get_inode_type(self) -> Union[str, None]: """Returns inode type name From 2e9b5b62faec7d8e7fa67dd1e9243013af7cbcb4 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Aug 2024 18:11:29 +0200 Subject: [PATCH 304/979] removing test code --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 06d2e2bf4..05679523f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1840,10 +1840,6 @@ class inode(objects.StructType): return "CHR" elif self.is_block: return "BLK" - elif self.is_whiteout: - return "WHT" - elif self.is_overlay: - return "OVL" else: return None From d4ed07f95b883183fcd28871e7fc7858e649347a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 20:51:32 +1000 Subject: [PATCH 305/979] PR review fixes: Add fixme to remember we should move wintime_to_datetime/unixtime_to_datetime out of renderers --- volatility3/framework/renderers/conversion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index 864794860..c8ddc19fd 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -11,6 +11,7 @@ from typing import Union from volatility3.framework import interfaces, renderers +# FIXME: Move wintime_to_datetime() and unixtime_to_datetime() out of renderers, possibly framework.objects.utility def wintime_to_datetime( wintime: int, ) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: From 0dfb9d8a0ff9080eef10b7505f7e1955e0f728e1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 20:57:23 +1000 Subject: [PATCH 306/979] Linux mountinfo: Add a method to yield all filesystem superblocks --- .../framework/plugins/linux/mountinfo.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index da743bb60..319c92cca 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -37,7 +37,7 @@ class MountInfo(plugins.PluginInterface): _required_framework_version = (2, 2, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -146,7 +146,7 @@ class MountInfo(plugins.PluginInterface): def _get_tasks_mountpoints( self, tasks: Iterable[interfaces.objects.ObjectInterface], - filtered_by_pids: bool, + filtered_by_pids: bool = False, ): seen_mountpoints = set() for task in tasks: @@ -184,8 +184,8 @@ class MountInfo(plugins.PluginInterface): self, tasks: Iterable[interfaces.objects.ObjectInterface], mnt_ns_ids: List[int], - mount_format: bool, - filtered_by_pids: bool, + mount_format: bool = False, + filtered_by_pids: bool = False, ) -> Iterable[Tuple[int, Tuple]]: show_filter_warning = False for task, mnt, mnt_ns_id in self._get_tasks_mountpoints( @@ -247,6 +247,31 @@ class MountInfo(plugins.PluginInterface): "Could not filter by mount namespace id. This field is not available in this kernel." ) + def get_superblocks(self): + """Yield file system superblocks based on the task's mounted filesystems. + + Yields: + super_block: Kernel's struct super_block object + """ + # No filter so that we get all the mount namespaces from all tasks + pid_filter = pslist.PsList.create_pid_filter() + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=pid_filter + ) + + seen_sb_ptr = set() + for task, mnt, _mnt_ns_id in self._get_tasks_mountpoints(tasks): + path_root = linux.LinuxUtilities.get_path_mnt(task, mnt) + if not path_root: + continue + + sb_ptr = mnt.get_mnt_sb() + if not sb_ptr or sb_ptr in seen_sb_ptr: + continue + seen_sb_ptr.add(sb_ptr) + + yield sb_ptr.dereference(), path_root + def run(self): pids = self.config.get("pids") mount_ns_ids = self.config.get("mntns") From 231f682b2769d7a5a6de0fc96008d44c06f0825c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:01:59 +1000 Subject: [PATCH 307/979] Linux: Improve mount's object extension method docstrings --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 05679523f..de5e432d3 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -47,7 +47,7 @@ class module(generic.GenericIntelProcess): ).choices except exceptions.SymbolError: vollog.debug( - f"Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" + "Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" ) # set to empty dict to show that the enum was not found, and so shouldn't be searched for again self._mod_mem_type = {} @@ -936,7 +936,8 @@ class mount(objects.StructType): MNT_RELATIME: "relatime", } - def get_mnt_sb(self): + def get_mnt_sb(self) -> int: + """Returns a pointer to the super_block""" if self.has_member("mnt"): return self.mnt.mnt_sb elif self.has_member("mnt_sb"): @@ -1251,6 +1252,7 @@ class vfsmount(objects.StructType): return self._get_real_mnt().has_parent() def get_mnt_sb(self): + """Returns a pointer to the super_block""" return self.mnt_sb def get_flags_access(self) -> str: From a369a9e23cea620a791129d687bd2bbe9c4d4442 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:05:23 +1000 Subject: [PATCH 308/979] Linux: dentry object extension: Add a method to walk dentries subdirectories --- .../symbols/linux/extensions/__init__.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index de5e432d3..3bfbe168a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -820,6 +820,26 @@ class dentry(objects.StructType): current_dentry = current_dentry.d_parent return None + def get_subdirs(self) -> interfaces.objects.ObjectInterface: + """Walks dentry subdirs + + Yields: + A dentry object + """ + if self.has_member("d_sib") and self.has_member("d_children"): + # kernels >= 6.8 + walk_member = "d_sib" + list_head_member = self.d_children.first + elif self.has_member("d_child") and self.has_member("d_subdirs"): + # 2.5.0 <= kernels < 6.8 + walk_member = "d_child" + list_head_member = self.d_subdirs + else: + raise exceptions.VolatilityException("Unsupported dentry type") + + dentry_type_name = self.get_symbol_table_name() + constants.BANG + "dentry" + yield from list_head_member.to_list(dentry_type_name, walk_member) + class struct_file(objects.StructType): def get_dentry(self) -> interfaces.objects.ObjectInterface: From 41684478ad7c12c2d862ac8612f44315bc6f855b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:21:49 +1000 Subject: [PATCH 309/979] Linux: Add page cache support, including abstractions like RadixTree, XArray, and IDR, to support both older and latest kernel versions --- .../framework/symbols/linux/__init__.py | 346 +++++++++++++++++- .../symbols/linux/extensions/__init__.py | 244 +++++++++++- 2 files changed, 588 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 03353135d..248cb8d75 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,6 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import math +from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework @@ -30,6 +32,9 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("kobject", extensions.kobject) self.set_type_class("cred", extensions.cred) self.set_type_class("inode", extensions.inode) + self.set_type_class("idr", extensions.IDR) + self.set_type_class("address_space", extensions.address_space) + self.set_type_class("page", extensions.page) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) @@ -67,7 +72,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 1, 0) + _version = (2, 2, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -425,3 +430,342 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): kernel = context.modules[kernel_module_name] return kernel + + @classmethod + def choose_kernel_tree(cls, vmlinux: interfaces.context.ModuleInterface) -> "Tree": + """Returns the appropriate tree data structure instance for the current kernel implementation. + This is used by the IDR and the PageCache to choose between the XArray and RadixTree. + + Args: + vmlinux: The kernel module object + + Returns: + The appropriate Tree instance for the current kernel + """ + address_space_type = vmlinux.get_type("address_space") + address_space_has_i_pages = address_space_type.has_member("i_pages") + i_pages_type_name = ( + address_space_type.child_template("i_pages").vol.type_name + if address_space_has_i_pages + else "" + ) + i_pages_is_xarray = i_pages_type_name.endswith(constants.BANG + "xarray") + i_pages_is_radix_tree_root = i_pages_type_name.endswith( + constants.BANG + "radix_tree_root" + ) and vmlinux.get_type("radix_tree_root").has_member("xa_head") + + if i_pages_is_xarray or i_pages_is_radix_tree_root: + return XArray(vmlinux) + else: + return RadixTree(vmlinux) + + +class Tree(ABC): + """Abstraction to support both XArray and RadixTree""" + + # Dynamic values, these will be initialized later + CHUNK_SHIFT = None + CHUNK_SIZE = None + CHUNK_MASK = None + + def __init__(self, vmlinux: interfaces.context.ModuleInterface): + self.vmlinux = vmlinux + self.vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + + self.pointer_size = self.vmlinux.get_type("pointer").size + # Dynamically work out the (XA_CHUNK|RADIX_TREE_MAP)_SHIFT values based on + # the node.slots[] array size + node_type = self.vmlinux.get_type(self.node_type_name) + slots_array_size = node_type.child_template("slots").count + + # Calculate the LSB index - 1 + self.CHUNK_SHIFT = slots_array_size.bit_length() - 1 + self.CHUNK_SIZE = 1 << self.CHUNK_SHIFT + self.CHUNK_MASK = self.CHUNK_SIZE - 1 + + @property + @abstractmethod + def node_type_name(self) -> str: + """Returns the Tree implementation node type name + + Returns: + A string with the node type name + """ + raise NotImplementedError() + + @property + def tag_internal_value(self) -> int: + """Returns the internal node flag for the tree""" + raise NotImplementedError() + + @abstractmethod + def node_is_internal(self, nodep) -> bool: + """Checks if the node is internal""" + raise NotImplementedError + + @abstractmethod + def is_node_tagged(self, nodep) -> bool: + """Checks if the node pointer is tagged""" + raise NotImplementedError + + @abstractmethod + def untag_node(self, nodep) -> int: + """Untags a node pointer""" + raise NotImplementedError + + @abstractmethod + def get_tree_height(self, treep) -> int: + """Returns the tree height""" + raise NotImplementedError + + @abstractmethod + def get_node_height(self, nodep) -> int: + """Returns the node height""" + raise NotImplementedError + + @abstractmethod + def get_head_node(self, tree) -> int: + """Returns a pointer to the tree's head""" + raise NotImplementedError + + @abstractmethod + def is_valid_node(self, nodep) -> bool: + """Validates a node pointer""" + raise NotImplementedError + + def nodep_to_node(self, nodep) -> interfaces.objects.ObjectInterface: + """Instanciates a tree node from its pointer + + Args: + nodep: Pointer to the XArray/RadixTree node + + Returns: + A XArray/RadixTree node instance + """ + node = self.vmlinux.object(self.node_type_name, offset=nodep, absolute=True) + return node + + def _slot_to_nodep(self, slot) -> int: + if self.node_is_internal(slot): + nodep = slot & ~self.tag_internal_value + else: + nodep = slot + + return nodep + + def _iter_node(self, nodep, height) -> int: + node = self.nodep_to_node(nodep) + node_slots = node.slots + for off in range(self.CHUNK_SIZE): + slot = node_slots[off] + if slot == 0: + continue + + nodep = self._slot_to_nodep(slot) + + if height == 1: + if self.is_valid_node(nodep): + yield nodep + else: + for child_node in self._iter_node(nodep, height - 1): + yield child_node + + def get_page_addresses(self, root: interfaces.objects.ObjectInterface) -> int: + """Walks the tree data structure + + Args: + root: The tree root object + + Yields: + A tree node pointer + """ + height = self.get_tree_height(root.vol.offset) + + nodep = self.get_head_node(root) + if not nodep: + return + + # Keep the internal flag before untagging it + is_internal = self.node_is_internal(nodep) + if self.is_node_tagged(nodep): + nodep = self.untag_node(nodep) + + if is_internal: + height = self.get_node_height(nodep) + + if height == 0: + if self.is_valid_node(nodep): + yield nodep + else: + for child_node in self._iter_node(nodep, height): + yield child_node + + +class XArray(Tree): + XARRAY_TAG_MASK = 3 + XARRAY_TAG_INTERNAL = 2 + + def get_tree_height(self, treep) -> int: + return 0 + + @property + def node_type_name(self) -> str: + return "xa_node" + + @property + def tag_internal_value(self) -> int: + return self.XARRAY_TAG_INTERNAL + + def get_node_height(self, nodep) -> int: + node = self.nodep_to_node(nodep) + return (node.shift / self.CHUNK_SHIFT) + 1 + + def get_head_node(self, tree) -> int: + return tree.xa_head + + def node_is_internal(self, nodep) -> bool: + return (nodep & self.XARRAY_TAG_MASK) == self.XARRAY_TAG_INTERNAL + + def is_node_tagged(self, nodep) -> bool: + return (nodep & self.XARRAY_TAG_MASK) != 0 + + def untag_node(self, nodep) -> int: + return nodep & (~self.XARRAY_TAG_MASK) + + def is_valid_node(self, nodep) -> bool: + # It should have the tag mask clear + return not self.is_node_tagged(nodep) + + +class RadixTree(Tree): + RADIX_TREE_INTERNAL_NODE = 1 + RADIX_TREE_EXCEPTIONAL_ENTRY = 2 + RADIX_TREE_ENTRY_MASK = 3 + + # Dynamic values. These will be initialized later + RADIX_TREE_INDEX_BITS = None + RADIX_TREE_MAX_PATH = None + RADIX_TREE_HEIGHT_SHIFT = None + RADIX_TREE_HEIGHT_MASK = None + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + char_bits = 8 + self.RADIX_TREE_INDEX_BITS = char_bits * self.pointer_size + self.RADIX_TREE_MAX_PATH = int( + math.ceil(self.RADIX_TREE_INDEX_BITS / float(self.CHUNK_SHIFT)) + ) + self.RADIX_TREE_HEIGHT_SHIFT = self.RADIX_TREE_MAX_PATH + 1 + self.RADIX_TREE_HEIGHT_MASK = (1 << self.RADIX_TREE_HEIGHT_SHIFT) - 1 + + if not self.vmlinux.has_type("radix_tree_root"): + # In kernels 4.20, RADIX_TREE_INTERNAL_NODE flag took RADIX_TREE_EXCEPTIONAL_ENTRY's + # value. RADIX_TREE_EXCEPTIONAL_ENTRY was removed but that's managed in is_valid_node() + # Note that the Radix Tree is still in use for IDR, even after kernels 4.20 when XArray + # mostly replace it + self.RADIX_TREE_INTERNAL_NODE = 2 + + @property + def node_type_name(self) -> str: + return "radix_tree_node" + + @property + def tag_internal_value(self) -> int: + return self.RADIX_TREE_INTERNAL_NODE + + def get_tree_height(self, treep) -> int: + try: + if self.vmlinux.get_type("radix_tree_root").has_member("height"): + # kernels < 4.7.10 + radix_tree_root = self.vmlinux.object( + "radix_tree_root", offset=treep, absolute=True + ) + return radix_tree_root.height + except exceptions.SymbolError: + pass + + # kernels >= 4.7.10 + return 0 + + def _radix_tree_maxindex(self, node, height) -> int: + """Return the maximum key which can be store into a radix tree with this height.""" + + if not self.vmlinux.has_symbol("height_to_maxindex"): + # Kernels >= 4.7 + return (self.CHUNK_SIZE << node.shift) - 1 + else: + # Kernels < 4.7 + height_to_maxindex_array = self.vmlinux.object_from_symbol( + "height_to_maxindex" + ) + maxindex = height_to_maxindex_array[height] + return maxindex + + def get_node_height(self, nodep) -> int: + node = self.nodep_to_node(nodep) + if hasattr(node, "shift"): + # 4.7 <= Kernels < 4.20 + return (node.shift / self.CHUNK_SHIFT) + 1 + elif hasattr(node, "path"): + # 3.15 <= Kernels < 4.7 + return node.path & self.RADIX_TREE_HEIGHT_MASK + elif hasattr(node, "height"): + # Kernels < 3.15 + return node.height + else: + raise exceptions.VolatilityException("Cannot find radix-tree node height") + + def get_head_node(self, tree) -> int: + return tree.rnode + + def node_is_internal(self, nodep) -> bool: + return (nodep & self.RADIX_TREE_INTERNAL_NODE) != 0 + + def is_node_tagged(self, nodep) -> bool: + return self.node_is_internal(nodep) + + def untag_node(self, nodep) -> int: + return nodep & (~self.RADIX_TREE_ENTRY_MASK) + + def is_valid_node(self, nodep) -> bool: + # In kernels 4.20, exceptional nodes were removed and internal entries took their bitmask + if self.vmlinux.has_type("radix_tree_root"): + return ( + nodep & self.RADIX_TREE_ENTRY_MASK + ) != self.RADIX_TREE_EXCEPTIONAL_ENTRY + + return True + + +class PageCache(object): + """Linux Page Cache abstraction""" + + def __init__( + self, + page_cache: interfaces.objects.ObjectInterface, + vmlinux: interfaces.context.ModuleInterface, + ): + """ + Args: + page_cache: Page cache address space + vmlinux: Kernel module object + """ + self.vmlinux = vmlinux + self._page_cache = page_cache + self._tree = LinuxUtilities.choose_kernel_tree(self.vmlinux) + + def get_cached_pages(self) -> interfaces.objects.ObjectInterface: + """Returns all page cache contents + + Yields: + Page objects + """ + + for page_addr in self._tree.get_page_addresses(self._page_cache.i_pages): + if not page_addr: + continue + + page = self.vmlinux.object("page", offset=page_addr, absolute=True) + if page: + yield page diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3bfbe168a..300ab2ed0 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -4,10 +4,11 @@ import collections.abc import logging +import functools import stat from datetime import datetime import socket as socket_module -from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union +from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, Dict from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion @@ -1919,3 +1920,244 @@ class inode(objects.StructType): The inode's file mode string """ return stat.filemode(self.i_mode) + + def get_pages(self) -> interfaces.objects.ObjectInterface: + """Gets the inode's cached pages + + Yields: + The inode's cached pages + """ + if not self.i_size: + return + elif not (self.i_mapping and self.i_mapping.nrpages > 0): + return + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + page_cache = linux.PageCache(self.i_mapping.dereference(), vmlinux) + yield from page_cache.get_cached_pages() + + def get_contents(self): + """Get the inode cached pages from the page cache + + Yields: + page_index (int): The page index in the Tree. File offset is page_index * PAGE_SIZE. + page_content (str): The page content + """ + for page_obj in self.get_pages(): + page_index = int(page_obj.index) + page_content = page_obj.get_content() + yield page_index, page_content + + +class address_space(objects.StructType): + @property + def i_pages(self): + """Returns the appropriate member containing the page cache tree""" + if self.has_member("i_pages"): + # Kernel >= 4.17 + return self.member("i_pages") + elif self.has_member("page_tree"): + # Kernel < 4.17 + return self.member("page_tree") + + raise exceptions.VolatilityException("Unsupported page cache tree") + + +class page(objects.StructType): + @property + @functools.cache + def pageflags_enum(self) -> Dict: + """Returns 'pageflags' enumeration key/values + + Returns: + A dictionary with the pageflags enumeration key/values + """ + # FIXME: It would be even better to use @functools.cached_property instead, + # however, this requires Python +3.8 + try: + pageflags_enum = self._context.symbol_space.get_enumeration( + self.get_symbol_table_name() + constants.BANG + "pageflags" + ).choices + except exceptions.SymbolError: + vollog.debug( + "Unable to find pageflags enum. This can happen in kernels < 2.6.26 or wrong ISF" + ) + # set to empty dict to show that the enum was not found, and so shouldn't be searched for again + pageflags_enum = {} + + return pageflags_enum + + def flags_list(self) -> List[str]: + """Returns a list of page flags + + Returns: + List of page flags + """ + flags = [] + for name, value in self.pageflags_enum.items(): + if self.flags & (1 << value) != 0: + flags.append(name) + + return flags + + def to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current physical memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + + vmemmap_start = None + if vmlinux.has_symbol("mem_section"): + # SPARSEMEM_VMEMMAP physical memory model: memmap is virtually contiguous + if vmlinux.has_symbol("vmemmap_base"): + # CONFIG_DYNAMIC_MEMORY_LAYOUT - KASLR kernels >= 4.9 + vmemmap_start = vmlinux.object_from_symbol("vmemmap_base") + else: + # !CONFIG_DYNAMIC_MEMORY_LAYOUT + if vmlinux_layer._maxvirtaddr < 57: + # 4-Level paging -> VMEMMAP_START = __VMEMMAP_BASE_L4 + vmemmap_base_l4 = 0xFFFFEA0000000000 + vmemmap_start = vmemmap_base_l4 + else: + # 5-Level paging -> VMEMMAP_START = __VMEMMAP_BASE_L5 + vmemmap_base_l5 = 0xFFD4000000000000 + vmemmap_start = vmemmap_base_l5 + + # FIXME: Remove this exception once 5-level paging is supported. + raise exceptions.VolatilityException( + "5-level paging is not yet supported" + ) + + elif vmlinux.has_symbol("mem_map"): + # FLATMEM physical memory model, typically 32bit + vmemmap_start = vmlinux.object_from_symbol("mem_map") + + elif vmlinux.has_symbol("node_data"): + raise exceptions.VolatilityException("NUMA systems are not yet supported") + else: + raise exceptions.VolatilityException("Unsupported Linux memory model") + + if not vmemmap_start: + raise exceptions.VolatilityException( + "Something went wrong, we shouldn't be here" + ) + + page_type_size = vmlinux.get_type("page").size + pagec = vmlinux_layer.canonicalize(self.vol.offset) + pfn = (pagec - vmemmap_start) // page_type_size + page_paddr = pfn * vmlinux_layer.page_size + + return page_paddr + + def get_content(self) -> Union[str, None]: + """Returns the page content + + Returns: + The page content + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + physical_layer = vmlinux.context.layers["memory_layer"] + page_paddr = self.to_paddr() + if not page_paddr: + return + + page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) + return page_data + + +class IDR(objects.StructType): + IDR_BITS = 8 + IDR_MASK = (1 << IDR_BITS) - 1 + INT_SIZE = 4 + MAX_IDR_SHIFT = INT_SIZE * 8 - 1 + MAX_IDR_BIT = 1 << MAX_IDR_SHIFT + + def idr_max(self, num_layers: int) -> int: + """Returns the maximum ID which can be allocated given idr::layers + + Args: + num_layers: Number of layers + + Returns: + Maximum ID for a given number of layers + """ + # Kernel < 4.17 + bits = min([self.INT_SIZE, num_layers * self.IDR_BITS, self.MAX_IDR_SHIFT]) + + return (1 << bits) - 1 + + def idr_find(self, idr_id: int) -> int: + """Finds an ID within the IDR data structure. + Based on idr_find_slowpath(), 3.9 <= Kernel < 4.11 + Args: + idr_id: The IDR element ID + + Returns: + A pointer to the given ID element + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + if not vmlinux.get_type("idr_layer").has_member("layer"): + vollog.info( + "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" + ) + return + + if idr_id < 0: + return + + cur_layer = self.top + if not cur_layer: + return + + n = (cur_layer.layer + 1) * self.IDR_BITS + + if idr_id > self.idr_max(cur_layer.layer + 1): + return + + assert n != 0 + + while n > 0 and cur_layer: + n -= self.IDR_BITS + assert n == cur_layer.layer * self.IDR_BITS + cur_layer = cur_layer.ary[(idr_id >> n) & self.IDR_MASK] + + return cur_layer.v() + + def _old_kernel_get_page_addresses(self, in_use) -> int: + # Kernels < 4.11 + total = next_id = 0 + while total < in_use: + page_addr = self.idr_find(next_id) + if page_addr: + yield page_addr + total += 1 + + next_id += 1 + + def _new_kernel_get_page_addresses(self, _in_use) -> int: + # Kernels >= 4.11 + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + tree = linux.LinuxUtilities.choose_kernel_tree(vmlinux) + for page_addr in tree.get_page_addresses(root=self.idr_rt): + yield page_addr + + def get_page_addresses(self, in_use=0) -> int: + """Walks the IDR and yield a pointer associated with each element. + + Args: + in_use (int, optional): _description_. Defaults to 0. + + Yields: + A pointer associated with each element. + """ + if self.has_member("idr_rt"): + get_page_addresses_func = self._new_kernel_get_page_addresses + else: + get_page_addresses_func = self._old_kernel_get_page_addresses + + for page_addr in get_page_addresses_func(in_use): + yield page_addr From ac27d6663a3733e002cb064afd155139103c4c43 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:26:29 +1000 Subject: [PATCH 310/979] Linux: Add two page cache plugins, linux.pagecache.Files and linux.pagecache.InodePages --- .../framework/plugins/linux/pagecache.py | 504 ++++++++++++++++++ 1 file changed, 504 insertions(+) create mode 100644 volatility3/framework/plugins/linux/pagecache.py diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py new file mode 100644 index 000000000..545c243e0 --- /dev/null +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -0,0 +1,504 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import math +import logging +import datetime +from dataclasses import dataclass, astuple +from typing import List + +from volatility3.framework import renderers, interfaces +from volatility3.framework.renderers import format_hints +from volatility3.framework.interfaces import plugins +from volatility3.framework.configuration import requirements +from volatility3.plugins import timeliner +from volatility3.plugins.linux import mountinfo + +vollog = logging.getLogger(__name__) + + +@dataclass +class InodeUser: + """Inode user representation, featuring augmented information and formatted fields. + This is the data the plugin will eventually display. + """ + + superblock_addr: int + mountpoint: str + device: str + inode_num: int + inode_addr: int + type: str + inode_pages: int + cached_pages: int + file_mode: str + access_time: str + modification_time: str + change_time: str + path: str + + +@dataclass +class InodeInternal: + """Inode internal representation containing only the core objects + + Fields: + superblock: 'super_block' struct + mountpoint: Superblock mountpoint path + inode: 'inode' struct + path: Dentry full path + """ + + superblock: interfaces.objects.ObjectInterface + mountpoint: str + inode: interfaces.objects.ObjectInterface + path: str + + def to_user( + self, kernel_layer: interfaces.layers.TranslationLayerInterface + ) -> InodeUser: + """Augment the inode information to be presented to the user + + Args: + kernel_layer: The kernel layer to obtain the page size + + Returns: + An InodeUser dataclass + """ + # Ensure all types are atomic immutable. Otherwise, astuple() will take a long + # time doing a deepcopy of the Volatility objects. + superblock_addr = self.superblock.vol.offset + device = f"{self.superblock.major}:{self.superblock.minor}" + inode_num = int(self.inode.i_ino) + inode_addr = self.inode.vol.offset + inode_type = renderers.UnparsableValue() + # Round up the number of pages to fit the inode's size + inode_pages = int(math.ceil(self.inode.i_size / float(kernel_layer.page_size))) + cached_pages = int(self.inode.i_mapping.nrpages) + file_mode = self.inode.get_file_mode() + access_time_dt = self.inode.get_access_time() + modification_time_str = self.inode.get_modification_time() + change_time_str = self.inode.get_change_time() + + inode_user = InodeUser( + superblock_addr=superblock_addr, + mountpoint=self.mountpoint, + device=device, + inode_num=inode_num, + inode_addr=inode_addr, + type=inode_type, + inode_pages=inode_pages, + cached_pages=cached_pages, + file_mode=file_mode, + access_time=access_time_dt, + modification_time=modification_time_str, + change_time=change_time_str, + path=self.path, + ) + return inode_user + + +class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): + """Lists files from memory""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 1, 0) + ), + requirements.ListRequirement( + name="type", + description="List of space-separated file type filters i.e. --type REG DIR", + element_type=str, + optional=True, + ), + requirements.StringRequirement( + name="find", + description="Filename (full path) to find", + optional=True, + ), + ] + + @staticmethod + def _follow_symlink(inode, symlink_path) -> str: + """Follows (fast) symlinks (kernels >= 4.2.x). + Fast symlinks are filesystem agnostic. + + Args: + inode: The inode (or pointer) to dump + symlink_path: The symlink name + + Returns: + If it can resolve the symlink, it returns a string "symlink_path -> target_path" + Otherwise, it returns the same symlink_path + """ + # i_link (fast symlinks) were introduced in 4.2 + if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: + i_link_str = inode.i_link.dereference().cast( + "string", max_length=255, encoding="utf-8", errors="replace" + ) + symlink_path = f"{symlink_path} -> {i_link_str}" + + return symlink_path + + @classmethod + def _walk_dentry(cls, seen_dentries, root_dentry, parent): + + for dentry in root_dentry.get_subdirs(): + dentry_addr = dentry.vol.offset + + # corruption + if dentry_addr == root_dentry.vol.offset: + continue + + if dentry_addr in seen_dentries: + continue + + seen_dentries.add(dentry_addr) + + inode = dentry.d_inode + if not (inode and inode.is_valid()): + continue + + # This allows us to have consistent paths + if dentry.d_name.name: + name = dentry.d_name.name_as_str() + # Do NOT use os.path.join() below + new_file = parent + "/" + name + else: + continue + + yield new_file, dentry, dentry.d_parent.vol.offset + + if inode.is_dir: + for new_file, dentry, parent_address in cls._walk_dentry( + seen_dentries, dentry, new_file + ): + yield new_file, dentry, parent_address + + @classmethod + def get_inodes( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + ): + """Retrieves the inodes from the superblocks + + Args: + context: The context that the plugin will operate within + config_path: The path to configuration data within the context configuration data + + Yields: + An InodeInternal object + """ + + superblocks_iter = mountinfo.MountInfo( + context=context, + config_path=config_path, + ).get_superblocks() + + seen_inodes = set() + seen_dentries = set() + for superblock, mountpoint in superblocks_iter: + parent = "" if mountpoint == "/" else mountpoint + + # Superblock root dentry + root_dentry = superblock.s_root + if not root_dentry: + continue + + # Dentry sanity check + if not root_dentry.is_root(): + continue + + # More dentry/inode sanity checks + root_inode_ptr = root_dentry.d_inode + if not root_inode_ptr: + continue + root_inode = root_inode_ptr.dereference() + if not root_inode.is_valid(): + continue + + # Inode already processed? + if root_inode_ptr in seen_inodes: + continue + seen_inodes.add(root_inode_ptr) + + root_path = mountpoint + + inode_in = InodeInternal( + superblock=superblock, + mountpoint=mountpoint, + inode=root_inode, + path=root_path, + ) + yield inode_in + + # Children + for file_path, file_dentry, _ in cls._walk_dentry( + seen_dentries, root_dentry, parent + ): + if not file_dentry: + continue + # Dentry/inode sanity checks + file_inode_ptr = file_dentry.d_inode + if not file_inode_ptr: + continue + file_inode = file_inode_ptr.dereference() + if not file_inode.is_valid(): + continue + + # Inode already processed? + if file_inode_ptr in seen_inodes: + continue + seen_inodes.add(file_inode_ptr) + + file_path = cls._follow_symlink(file_inode_ptr, file_path) + inode_in = InodeInternal( + superblock=superblock, + mountpoint=mountpoint, + inode=file_inode, + path=file_path, + ) + yield inode_in + + def _generator(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + inodes_iter = self.get_inodes( + context=self.context, config_path=self.config_path + ) + + types_filter = self.config["type"] + for inode_in in inodes_iter: + if types_filter and inode_in.inode.get_inode_type() not in types_filter: + continue + + if self.config["find"]: + if inode_in.path == self.config["find"]: + inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) + break # Only the first match + else: + inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) + + def generate_timeline(self): + """Generates tuples of (description, timestamp_type, timestamp) + + These need not be generated in any particular order, sorting + will be done later + """ + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + inodes_iter = self.get_inodes( + context=self.context, config_path=self.config_path + ) + for inode_in in inodes_iter: + inode_out = inode_in.to_user(vmlinux_layer) + description = f"Cached Inode for {inode_out.path}" + yield description, timeliner.TimeLinerType.ACCESSED, inode_out.access_time + yield description, timeliner.TimeLinerType.MODIFIED, inode_out.modification_time + yield description, timeliner.TimeLinerType.CHANGE, inode_out.change_time + + @staticmethod + def format_fields_with_headers(headers, generator): + """Uses the headers type to cast the fields obtained from the generator""" + for level, fields in generator: + formatted_fields = [] + for header, field in zip(headers, fields): + header_type = header[1] + + if isinstance( + field, (header_type, interfaces.renderers.BaseAbsentValue) + ): + formatted_field = field + else: + formatted_field = header_type(field) + + formatted_fields.append(formatted_field) + yield level, formatted_fields + + def run(self): + headers = [ + ("SuperblockAddr", format_hints.Hex), + ("MountPoint", str), + ("Device", str), + ("InodeNum", int), + ("InodeAddr", format_hints.Hex), + ("FileType", str), + ("InodePages", int), + ("CachedPages", int), + ("FileMode", str), + ("AccessTime", datetime.datetime), + ("ModificationTime", datetime.datetime), + ("ChangeTime", datetime.datetime), + ("FilePath", str), + ] + + return renderers.TreeGrid( + headers, self.format_fields_with_headers(headers, self._generator()) + ) + + +class InodePages(plugins.PluginInterface): + """Lists and recovers cached inode pages""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="files", plugin=Files, version=(1, 0, 0) + ), + requirements.StringRequirement( + name="find", + description="Filename (full path) to find ", + optional=True, + ), + requirements.IntRequirement( + name="inode", + description="Inode address", + optional=True, + ), + requirements.StringRequirement( + name="dump", + description="Output file path", + optional=True, + ), + ] + + @staticmethod + def write_inode_content_to_file( + inode: interfaces.objects.ObjectInterface, + filename: str, + vmlinux_layer: interfaces.layers.TranslationLayerInterface, + ) -> None: + """Extracts the inode's contents from the page cache and saves them to a file + + Args: + inode: The inode to dump + filename: Filename for writing the inode content + vmlinux_layer: The kernel layer to obtain the page size + """ + if not inode.is_reg: + vollog.error("The inode is not a regular file") + return + + # By using truncate/seek, provided the filesystem supports it, a sparse file will be + # created, saving both disk space and I/O time. + # Additionally, using the page index will guarantee that each page is written at the + # appropriate file position. + try: + with open(filename, "wb") as f: + inode_size = inode.i_size + f.truncate(inode_size) + + for page_idx, page_content in inode.get_contents(): + current_fp = page_idx * vmlinux_layer.page_size + max_length = inode_size - current_fp + page_bytes = page_content[:max_length] + if current_fp + len(page_bytes) > inode_size: + vollog.error( + "Page out of file bounds: inode 0x%x, inode size %d, page index %d", + inode.vol.object, + inode_size, + page_idx, + ) + f.seek(current_fp) + f.write(page_bytes) + + except IOError as e: + vollog.error("Unable to write to file (%s): %s", filename, e) + + def _generator(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + if self.config["inode"] and self.config["find"]: + vollog.error("Cannot use --inode and --find simultaneously") + return + + if self.config["find"]: + inodes_iter = Files.get_inodes( + context=self.context, config_path=self.config_path + ) + for inode_in in inodes_iter: + if inode_in.path == self.config["find"]: + inode = inode_in.inode + break # Only the first match + + elif self.config["inode"]: + inode = vmlinux.object("inode", self.config["inode"], absolute=True) + else: + vollog.error("You must use either --inode or --find") + return + + if not inode.is_reg: + vollog.error("The inode is not a regular file") + return + + inode_size = inode.i_size + if not inode.is_valid(): + vollog.error("Invalid inode at 0x%x", self.config["inode"]) + return + + for page_obj in inode.get_pages(): + page_vaddr = page_obj.vol.offset + page_paddr = page_obj.to_paddr() + page_mapping_addr = page_obj.mapping + page_index = int(page_obj.index) + page_file_offset = page_index * vmlinux_layer.page_size + dump_safe = page_file_offset < inode_size + page_flags_list = page_obj.get_flags() + page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) + fields = ( + page_vaddr, + page_paddr, + page_mapping_addr, + page_index, + dump_safe, + page_flags, + ) + + yield 0, fields + + if self.config["dump"]: + filename = self.config["dump"] + vollog.info("[*] Writing inode at 0x%x to '%s'", inode.vol.offset, filename) + self.write_inode_content_to_file(inode, filename, vmlinux_layer) + + def run(self): + headers = [ + ("PageVAddr", format_hints.Hex), + ("PagePAddr", format_hints.Hex), + ("MappingAddr", format_hints.Hex), + ("Index", int), + ("DumpSafe", bool), + ("Flags", str), + ] + + return renderers.TreeGrid( + headers, Files.format_fields_with_headers(headers, self._generator()) + ) From 55212008f805abdd47f2f3d7d6211c198097f8d4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:39:51 +1000 Subject: [PATCH 311/979] Linux: Add pidhashtable plugin. This is based on the vol2 plugin, removing ancient kernel support, curating code and enhancing comments, while using the new IDR abstraction included also in this effort. --- .../framework/plugins/linux/pidhashtable.py | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 volatility3/framework/plugins/linux/pidhashtable.py diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py new file mode 100644 index 000000000..b24c73e77 --- /dev/null +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -0,0 +1,249 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List + +from volatility3.framework import renderers, interfaces, constants +from volatility3.framework.symbols import linux +from volatility3.framework.renderers import format_hints +from volatility3.framework.interfaces import plugins +from volatility3.framework.configuration import requirements +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + + +class PIDHashTable(plugins.PluginInterface): + """Enumerates processes through the PID hash table""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 2, 0) + ), + requirements.BooleanRequirement( + name="decorate_comm", + description="Show `user threads` comm in curly brackets, and `kernel threads` comm in square brackets", + optional=True, + default=False, + ), + ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.vmlinux = None + self.vmlinux_layer = None + + def _is_valid_task(self, task): + return task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent) + + def _get_pidtype_pid(self): + # The pid_type enumeration is present since 2.5.37, just in case + pid_type_enum = self.vmlinux.get_enumeration("pid_type") + if not pid_type_enum: + vollog.error("Cannot find pid_type enum. Unsupported kernel") + return + + pidtype_pid = pid_type_enum.choices.get("PIDTYPE_PID") + if pidtype_pid is None: + vollog.error("Cannot find PIDTYPE_PID. Unsupported kernel") + return + + # Typically PIDTYPE_PID = 0 + return pidtype_pid + + def _get_pidhash_array(self): + pidhash_shift = self.vmlinux.object_from_symbol("pidhash_shift") + pidhash_size = 1 << pidhash_shift + + array_type_name = self.vmlinux.symbol_table_name + constants.BANG + "array" + + pidhash_ptr = self.vmlinux.object_from_symbol("pid_hash") + # pidhash is an array of hlist_heads + pidhash = self._context.object( + array_type_name, + offset=pidhash_ptr, + subtype=self.vmlinux.get_type("hlist_head"), + count=pidhash_size, + layer_name=self.vmlinux.layer_name, + ) + + return pidhash + + def _walk_upid(self, seen_upids, upid): + while upid and self.vmlinux_layer.is_valid(upid.vol.offset): + if upid.vol.offset in seen_upids: + break + seen_upids.add(upid.vol.offset) + + pid_chain = upid.pid_chain + if not (pid_chain and self.vmlinux_layer.is_valid(pid_chain.vol.offset)): + break + + upid = linux.LinuxUtilities.container_of( + pid_chain.next, "upid", "pid_chain", self.vmlinux + ) + + def _get_upids(self): + # 2.6.24 <= kernels < 4.15 + pidhash = self._get_pidhash_array() + + seen_upids = set() + for hlist in pidhash: + # each entry in the hlist is a upid which is wrapped in a pid + ent = hlist.first + + while ent and self.vmlinux_layer.is_valid(ent.vol.offset): + # upid->pid_chain exists 2.6.24 <= kernel < 4.15 + upid = linux.LinuxUtilities.container_of( + ent.vol.offset, "upid", "pid_chain", self.vmlinux + ) + + if upid.vol.offset in seen_upids: + break + + self._walk_upid(seen_upids, upid) + + ent = ent.next + + return seen_upids + + def _pid_hash_implementation(self): + # 2.6.24 <= kernels < 4.15 + task_pids_off = self.vmlinux.get_type("task_struct").relative_child_offset( + "pids" + ) + pidtype_pid = self._get_pidtype_pid() + + for upid in self._get_upids(): + pid = linux.LinuxUtilities.container_of( + upid, "pid", "numbers", self.vmlinux + ) + if not pid: + continue + + pid_tasks_0 = pid.tasks[pidtype_pid].first + if not pid_tasks_0: + continue + + task = self.vmlinux.object( + "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True + ) + if self._is_valid_task(task): + yield task + + def _task_for_radix_pid_node(self, nodep): + # kernels >= 4.15 + pid = self.vmlinux.object("pid", offset=nodep, absolute=True) + pidtype_pid = self._get_pidtype_pid() + + pid_tasks_0 = pid.tasks[pidtype_pid].first + if not pid_tasks_0: + return + + task_struct_type = self.vmlinux.get_type("task_struct") + if task_struct_type.has_member("pids"): + member = "pids" + elif task_struct_type.has_member("pid_links"): + member = "pid_links" + else: + return None + + task_pids_off = task_struct_type.relative_child_offset(member) + task = self.vmlinux.object( + "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True + ) + return task + + def _pid_namespace_idr(self): + # kernels >= 4.15 + ns_addr = self.vmlinux.get_symbol("init_pid_ns").address + ns = self.vmlinux.object("pid_namespace", offset=ns_addr) + + for page_addr in ns.idr.get_page_addresses(): + task = self._task_for_radix_pid_node(page_addr) + if self._is_valid_task(task): + yield task + + def _determine_pid_func(self): + pid_hash = self.vmlinux.has_symbol("pid_hash") and self.vmlinux.has_symbol( + "pidhash_shift" + ) # 2.5.55 <= kernels < 4.15 + + has_pid_numbers = self.vmlinux.has_type("pid") and self.vmlinux.get_type( + "pid" + ).has_member( + "numbers" + ) # kernels >= 2.6.24 + + has_pid_numbers = self.vmlinux.has_type("upid") and self.vmlinux.get_type( + "upid" + ).has_member( + "pid_chain" + ) # 2.6.24 <= kernels < 4.15 + + # kernels >= 4.15 + pid_idr = self.vmlinux.has_type("pid_namespace") and self.vmlinux.get_type( + "pid_namespace" + ).has_member("idr") + + if pid_idr: + # kernels >= 4.15 + return self._pid_namespace_idr + elif pid_hash and has_pid_numbers and has_pid_numbers: + # 2.6.24 <= kernels < 4.15 + return self._pid_hash_implementation + + return None + + def get_tasks(self) -> interfaces.objects.ObjectInterface: + """Enumerates processes through the PID hash table + + Yields: + task_struct objects + """ + self.vmlinux = self.context.modules[self.config["kernel"]] + self.vmlinux_layer = self.context.layers[self.vmlinux.layer_name] + pid_func = self._determine_pid_func() + if not pid_func: + vollog.error("Cannot determine which PID hash table this kernel is using") + return + + yield from sorted(pid_func(), key=lambda t: (t.tgid, t.pid)) + + def _generator( + self, decorate_comm: bool = False + ) -> interfaces.objects.ObjectInterface: + for task in self.get_tasks(): + offset, pid, tid, ppid, name = pslist.PsList.get_task_fields( + task, decorate_comm + ) + fields = format_hints.Hex(offset), pid, tid, ppid, name + yield 0, fields + + def run(self): + decorate_comm = self.config.get("decorate_comm") + + headers = [ + ("OFFSET", format_hints.Hex), + ("PID", int), + ("TID", int), + ("PPID", int), + ("COMM", str), + ] + return renderers.TreeGrid(headers, self._generator(decorate_comm=decorate_comm)) From 103537801ee0b49ca3475be11e1fe62670938e91 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 21:44:38 +1000 Subject: [PATCH 312/979] Linux: Add a basic eBPF program enumeration plugin to test and demonstrate using the IDR abstraction --- volatility3/framework/plugins/linux/ebpf.py | 78 +++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 volatility3/framework/plugins/linux/ebpf.py diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py new file mode 100644 index 000000000..33ba71faf --- /dev/null +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -0,0 +1,78 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import binascii +import logging +from typing import List + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.interfaces import plugins +from volatility3.framework.configuration import requirements + +vollog = logging.getLogger(__name__) + + +class EBPF(plugins.PluginInterface): + """Enumerate eBPF programs""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + ] + + def get_ebpf_programs(self, vmlinux) -> interfaces.objects.ObjectInterface: + """Enumerate eBPF programs walking its IDR. + + Args: + vmlinux: The kernel symbols object + + Yields: + eBPF program objects + """ + if not vmlinux.has_symbol("prog_idr"): + raise exceptions.VolatilityException( + "Cannot find the eBPF prog idr. Unsupported kernel" + ) + + prog_idr_addr = vmlinux.get_symbol("prog_idr").address + prog_idr = vmlinux.object("idr", offset=prog_idr_addr) + for page_addr in prog_idr.get_page_addresses(): + bpf_prog = vmlinux.object("bpf_prog", offset=page_addr, absolute=True) + yield bpf_prog + + def _generator(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + bpf_prog_types = vmlinux.get_enumeration("bpf_prog_type") + for prog in self.get_ebpf_programs(vmlinux): + prog_addr = prog.vol.offset + prog_type = bpf_prog_types.lookup(prog.type) + prog_tag_addr = prog.tag.vol.offset + prog_tag_size = prog.tag.count + prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) + prog_tag = binascii.hexlify(prog_tag_bytes).decode() + prog_name = ( + utility.array_to_string(prog.aux.name) or renderers.NotAvailableValue() + ) + fields = (format_hints.Hex(prog_addr), prog_name, prog_tag, prog_type) + yield (0, fields) + + def run(self): + headers = [ + ("Address", format_hints.Hex), + ("Name", str), + ("Tag", str), + ("Type", str), + ] + return renderers.TreeGrid(headers, self._generator()) From cc04f665e35989fea4c108de2bdd1b31016145ed Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 3 Aug 2024 23:47:28 +1000 Subject: [PATCH 313/979] Fix inode type --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 545c243e0..c36f8a339 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -72,7 +72,7 @@ class InodeInternal: device = f"{self.superblock.major}:{self.superblock.minor}" inode_num = int(self.inode.i_ino) inode_addr = self.inode.vol.offset - inode_type = renderers.UnparsableValue() + inode_type = self.inode.get_inode_type() or renderers.UnparsableValue() # Round up the number of pages to fit the inode's size inode_pages = int(math.ceil(self.inode.i_size / float(kernel_layer.page_size))) cached_pages = int(self.inode.i_mapping.nrpages) From a515e571dcb9eef2fc268b707cc175951f0831b8 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Sat, 3 Aug 2024 12:03:20 -0700 Subject: [PATCH 314/979] add Win10x64 17763 --- .../framework/plugins/windows/consoles.py | 8 + .../consoles/consoles-win10-17763-x64.json | 595 ++++++++++++++++++ .../symbols/windows/extensions/consoles.py | 5 + 3 files changed, 608 insertions(+) create mode 100644 volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index 509b6ae9d..0ef9e990a 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -203,6 +203,7 @@ class Consoles(interfaces.plugins.PluginInterface): version_dict = {} else: version_dict = { + (10, 0, 17763, 0): "consoles-win10-17763-x64", (10, 0, 20348, 1): "consoles-win10-20348-x64", (10, 0, 20348, 1970): "consoles-win10-20348-1970-x64", (10, 0, 20348, 2461): "consoles-win10-20348-2461-x64", @@ -580,6 +581,13 @@ class Consoles(interfaces.plugins.PluginInterface): "data": command_history.CommandCount, } ) + console_properties.append( + { + "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_LastDisplayed", + "address": command_history.LastDisplayed.vol.offset, + "data": command_history.LastDisplayed, + } + ) for ( cmd_index, bucket_cmd, diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json new file mode 100644 index 000000000..f11fb39b1 --- /dev/null +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json @@ -0,0 +1,595 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "short": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_CONSOLE_INFORMATION": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 32 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 34 + }, + "CommandHistorySize": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 144 + }, + "HistoryBufferMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 148 + }, + "OriginalTitle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1736 + }, + "Title": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1672 + }, + "GetScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1800 + }, + "CurrentScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1384 + }, + "ConsoleProcessList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 1360 + }, + "ProcessCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1368 + }, + "HistoryList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": -760 + }, + "HistoryBufferCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": -752 + }, + "ExeAliasList": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1392 + } + }, + "kind": "struct", + "size": 140 + }, + "_VECTOR": { + "fields": { + "Begin": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 0 + }, + "End": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 8 + }, + "EndCapacity": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + }, + "_CONSOLE_PROCESS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ConsoleProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_CONSOLE_PROCESS_HANDLE": { + "fields": { + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 52 + }, + "_CONSOLE_PROCESS": { + "fields": { + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 28 + }, + "ThreadId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + }, + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND_HISTORY": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "CommandBucket": { + "type": { + "kind": "struct", + "name": "_VECTOR" + }, + "offset": 16 + }, + "CommandCountMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 40 + }, + "Application": { + "type": { + "kind": "struct", + "name": "_COMMAND" + }, + "offset": 48 + }, + "ConsoleProcessHandle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS_HANDLE" + } + }, + "offset": 80 + }, + "Flags": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 88 + }, + "LastDisplayed": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 92 + } + }, + "kind": "struct", + "size": 96 + }, + "_SCREEN_INFORMATION": { + "fields": { + "TextBufferInfo": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 32 + }, + "Next": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 36 + } + }, + "kind": "struct", + "size": 72 + }, + "_ROW_POINTER": { + "fields": { + "Row": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + + }, + "_ROWS_ARRAY": { + "fields": { + "Rows": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_ROW_POINTER" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_TEXT_BUFFER_INFO": { + "fields": { + "BufferDeque": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_DEQUE" + } + }, + "offset": 0 + }, + "BufferRows": { + "type": { + "kind": "pointer", + "name": "void", + "subtype": { + "kind": "struct", + "name": "_ROWS_ARRAY" + } + }, + "offset": 8 + }, + "BufferCapacity": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 32 + }, + "ThisBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 48 + }, + "BufferEnd": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 56 + }, + "BufferLastIndex": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 58 + }, + "BufferStart": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 112 + } + }, + "kind": "struct", + "size": 72 + }, + "_CHAR_ROW_CELL": { + "fields": { + "Text": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "DbcsAttribute": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 2 + } + }, + "kind": "struct", + "size": 3 + }, + "_CHAR_ROW_CELL_ARRAY": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROW": { + "fields": { + "CharRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL_ARRAY" + } + }, + "offset": 8 + }, + "RowLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 64 + }, + "Index": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 72 + }, + "RowLength2": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 80 + }, + "TextBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 88 + } }, + "kind": "struct", + "size": 96 + }, + "_DEQUE": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle", + "datetime": "2024-07-31T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 279ee723e..74ea80e84 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -20,8 +20,13 @@ class ROW(objects.StructType): 0x28, 0x30, 0x48, + 0x50, 0x60, 0x80, + 0xa8, + 0xc0, + 0xc8, + 0x98, 0xF8, 0xF0, 0xA0, From 583cffe960a65a136d80bab194765368220bd9bf Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Sat, 3 Aug 2024 14:24:17 -0700 Subject: [PATCH 315/979] #816 - next console properties for better readability in treegrid --- .../framework/plugins/windows/cmdscan.py | 17 ++++++- .../framework/plugins/windows/consoles.py | 44 +++++++++++++++++-- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 86a34a4f8..1e435c56f 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -175,6 +175,15 @@ class CmdScan(interfaces.plugins.PluginInterface): ) command_history_properties.append( { + "level": 0, + "name": "_COMMAND_HISTORY", + "address": command_history.vol.offset, + "data": "", + } + ) + command_history_properties.append( + { + "level": 1, "name": f"_COMMAND_HISTORY.Application", "address": command_history.Application.vol.offset, "data": command_history.get_application(), @@ -182,6 +191,7 @@ class CmdScan(interfaces.plugins.PluginInterface): ) command_history_properties.append( { + "level": 1, "name": f"_COMMAND_HISTORY.ProcessHandle", "address": command_history.ConsoleProcessHandle.ProcessHandle.vol.offset, "data": hex( @@ -191,6 +201,7 @@ class CmdScan(interfaces.plugins.PluginInterface): ) command_history_properties.append( { + "level": 1, "name": f"_COMMAND_HISTORY.CommandCount", "address": None, "data": command_history.CommandCount, @@ -198,6 +209,7 @@ class CmdScan(interfaces.plugins.PluginInterface): ) command_history_properties.append( { + "level": 1, "name": f"_COMMAND_HISTORY.LastDisplayed", "address": command_history.LastDisplayed.vol.offset, "data": command_history.LastDisplayed, @@ -205,6 +217,7 @@ class CmdScan(interfaces.plugins.PluginInterface): ) command_history_properties.append( { + "level": 1, "name": f"_COMMAND_HISTORY.CommandCountMax", "address": command_history.CommandCountMax.vol.offset, "data": command_history.CommandCountMax, @@ -213,6 +226,7 @@ class CmdScan(interfaces.plugins.PluginInterface): command_history_properties.append( { + "level": 1, "name": f"_COMMAND_HISTORY.CommandBucket", "address": command_history.CommandBucket.vol.offset, "data": "", @@ -225,6 +239,7 @@ class CmdScan(interfaces.plugins.PluginInterface): try: command_history_properties.append( { + "level": 2, "name": f"_COMMAND_HISTORY.CommandBucket_Command_{cmd_index}", "address": bucket_cmd.vol.offset, "data": bucket_cmd.get_command(), @@ -281,7 +296,7 @@ class CmdScan(interfaces.plugins.PluginInterface): if command_history and command_history_properties: for command_history_property in command_history_properties: yield ( - 0, + command_history_property["level"], ( proc.UniqueProcessId, process_name, diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index 0ef9e990a..3b87734e7 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -445,6 +445,15 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 0, + "name": "_CONSOLE_INFORMATION", + "address": console_info.vol.offset, + "data": "", + } + ) + console_properties.append( + { + "level": 1, "name": "_CONSOLE_INFORMATION.ScreenX", "address": console_info.ScreenX.vol.offset, "data": console_info.ScreenX, @@ -452,6 +461,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 1, "name": "_CONSOLE_INFORMATION.ScreenY", "address": console_info.ScreenY.vol.offset, "data": console_info.ScreenY, @@ -459,6 +469,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 1, "name": "_CONSOLE_INFORMATION.CommandHistorySize", "address": console_info.CommandHistorySize.vol.offset, "data": console_info.CommandHistorySize, @@ -466,6 +477,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 1, "name": "_CONSOLE_INFORMATION.HistoryBufferCount", "address": console_info.HistoryBufferCount.vol.offset, "data": console_info.HistoryBufferCount, @@ -473,6 +485,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 1, "name": "_CONSOLE_INFORMATION.HistoryBufferMax", "address": console_info.HistoryBufferMax.vol.offset, "data": console_info.HistoryBufferMax, @@ -480,6 +493,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 1, "name": "_CONSOLE_INFORMATION.Title", "address": console_info.Title.vol.offset, "data": console_info.get_title(), @@ -487,6 +501,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 1, "name": "_CONSOLE_INFORMATION.OriginalTitle", "address": console_info.OriginalTitle.vol.offset, "data": console_info.get_original_title(), @@ -498,6 +513,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 1, "name": "_CONSOLE_INFORMATION.ProcessCount", "address": console_info.ProcessCount.vol.offset, "data": console_info.ProcessCount, @@ -505,6 +521,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 1, "name": "_CONSOLE_INFORMATION.ConsoleProcessList", "address": console_info.ConsoleProcessList.vol.offset, "data": "", @@ -515,6 +532,7 @@ class Consoles(interfaces.plugins.PluginInterface): ): console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.ConsoleProcessList.ConsoleProcess_{index}", "address": attached_proc.ConsoleProcess.dereference().vol.offset, "data": "", @@ -522,6 +540,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.ConsoleProcessList.ConsoleProcess_{index}_ProcessId", "address": attached_proc.ConsoleProcess.ProcessId.vol.offset, "data": attached_proc.ConsoleProcess.ProcessId, @@ -529,6 +548,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.ConsoleProcessList.ConsoleProcess_{index}_ProcessHandle", "address": attached_proc.ConsoleProcess.ProcessHandle.vol.offset, "data": hex( @@ -542,6 +562,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 1, "name": "_CONSOLE_INFORMATION.HistoryList", "address": console_info.HistoryList.vol.offset, "data": "", @@ -553,6 +574,7 @@ class Consoles(interfaces.plugins.PluginInterface): try: console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}", "address": command_history.vol.offset, "data": "", @@ -560,6 +582,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_Application", "address": command_history.Application.vol.offset, "data": command_history.get_application(), @@ -567,6 +590,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_ProcessHandle", "address": command_history.ConsoleProcessHandle.ProcessHandle.vol.offset, "data": hex( @@ -576,6 +600,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_CommandCount", "address": None, "data": command_history.CommandCount, @@ -583,6 +608,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_LastDisplayed", "address": command_history.LastDisplayed.vol.offset, "data": command_history.LastDisplayed, @@ -593,8 +619,8 @@ class Consoles(interfaces.plugins.PluginInterface): bucket_cmd, ) in command_history.get_commands(): try: - console_properties.append( - { + console_properties.append({ + "level": 3, "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_Command_{cmd_index}", "address": bucket_cmd.vol.offset, "data": bucket_cmd.get_command(), @@ -610,12 +636,21 @@ class Consoles(interfaces.plugins.PluginInterface): ) vollog.debug(f"Getting ScreenBuffer entries for {console_info}") + console_properties.append( + { + "level": 1, + "name": "_CONSOLE_INFORMATION.CurrentScreenBuffer", + "address": console_info.CurrentScreenBuffer.vol.offset, + "data": "", + } + ) for screen_index, screen_info in enumerate( console_info.get_screens() ): try: console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}", "address": screen_info, "data": "", @@ -623,6 +658,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.ScreenX", "address": None, "data": screen_info.ScreenX, @@ -630,6 +666,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.ScreenY", "address": None, "data": screen_info.ScreenY, @@ -637,6 +674,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) console_properties.append( { + "level": 2, "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.Dump", "address": None, "data": "\n".join(screen_info.get_buffer()), @@ -748,7 +786,7 @@ class Consoles(interfaces.plugins.PluginInterface): if console_info and console_properties: for console_property in console_properties: yield ( - 0, + console_property["level"], ( proc.UniqueProcessId, process_name, From a3c50e8221ed39e399b500376cb448019f9403db Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Sat, 3 Aug 2024 14:27:42 -0700 Subject: [PATCH 316/979] #816 - black fixes --- .../framework/plugins/windows/consoles.py | 3 ++- .../symbols/windows/extensions/consoles.py | 21 +++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index 3b87734e7..9b41b2890 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -619,7 +619,8 @@ class Consoles(interfaces.plugins.PluginInterface): bucket_cmd, ) in command_history.get_commands(): try: - console_properties.append({ + console_properties.append( + { "level": 3, "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_Command_{cmd_index}", "address": bucket_cmd.vol.offset, diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 74ea80e84..23670bb9b 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -23,9 +23,9 @@ class ROW(objects.StructType): 0x50, 0x60, 0x80, - 0xa8, - 0xc0, - 0xc8, + 0xA8, + 0xC0, + 0xC8, 0x98, 0xF8, 0xF0, @@ -226,7 +226,12 @@ class COMMAND(objects.StructType): """A Command Structure""" def is_valid(self): - if self.Length < 1 or self.Allocated < 1 or self.Length > 1024 or self.Allocated > 1024: + if ( + self.Length < 1 + or self.Allocated < 1 + or self.Length > 1024 + or self.Allocated > 1024 + ): return False return True @@ -256,7 +261,7 @@ class COMMAND_HISTORY(objects.StructType): @property def ProcessHandle(self): - """ Allow ProcessHandle to be referenced regardless of OS version """ + """Allow ProcessHandle to be referenced regardless of OS version""" return self.ConsoleProcessHandle.ProcessHandle def is_valid(self, max_history=50): @@ -269,7 +274,11 @@ class COMMAND_HISTORY(objects.StructType): return False # Process handle must be a valid pid - if self.ProcessHandle <= 0 or self.ProcessHandle > 0xFFFF or self.ProcessHandle % 4 != 0: + if ( + self.ProcessHandle <= 0 + or self.ProcessHandle > 0xFFFF + or self.ProcessHandle % 4 != 0 + ): return False return True From 60f14478e2860cb97ef15954f80ee82202f5935f Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Sat, 3 Aug 2024 14:30:27 -0700 Subject: [PATCH 317/979] #816 - black fixes --- .../framework/plugins/windows/cmdscan.py | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 1e435c56f..b92d7efab 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -65,7 +65,9 @@ class CmdScan(interfaces.plugins.PluginInterface): @classmethod def get_filtered_vads( - cls, conhost_proc: interfaces.context.ContextInterface, size_filter: Optional[int]=0x40000000 + cls, + conhost_proc: interfaces.context.ContextInterface, + size_filter: Optional[int] = 0x40000000, ) -> List[Tuple[int, int]]: """ Returns vads of a process with smaller than size_filter @@ -129,7 +131,9 @@ class CmdScan(interfaces.plugins.PluginInterface): f"Found conhost process {conhost_proc} with pid {conhost_proc.UniqueProcessId}" ) - conhostexe_base, conhostexe_size = consoles.Consoles.find_conhostexe(conhost_proc) + conhostexe_base, conhostexe_size = consoles.Consoles.find_conhostexe( + conhost_proc + ) if not conhostexe_base: vollog.info( "Unable to find the location of conhost.exe. Analysis cannot proceed." @@ -154,7 +158,7 @@ class CmdScan(interfaces.plugins.PluginInterface): context, scanners.BytesScanner(max_history_bytes), sections=sections, - ): + ): command_history_properties = [] try: @@ -272,18 +276,24 @@ class CmdScan(interfaces.plugins.PluginInterface): no_registry = self.config.get("no_registry") if no_registry is False: - max_history, _max_buffers = consoles.Consoles.get_console_settings_from_registry( - self.context, - self.config_path, - kernel.layer_name, - kernel.symbol_table_name, - max_history, - [], + max_history, _max_buffers = ( + consoles.Consoles.get_console_settings_from_registry( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + max_history, + [], + ) ) vollog.debug(f"Possible CommandHistorySize values: {max_history}") - for proc, command_history, command_history_properties in self.get_command_history( + for ( + proc, + command_history, + command_history_properties, + ) in self.get_command_history( self.context, kernel.layer_name, kernel.symbol_table_name, @@ -305,7 +315,9 @@ class CmdScan(interfaces.plugins.PluginInterface): ( renderers.NotApplicableValue() if command_history_property["address"] is None - else format_hints.Hex(command_history_property["address"]) + else format_hints.Hex( + command_history_property["address"] + ) ), str(command_history_property["data"]), ), From 3e75c2ae9d29084485f6d2803c64d2c42e064ee0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 4 Aug 2024 14:40:45 +1000 Subject: [PATCH 318/979] Fix @functools.cache . It's available since Python 3.9 --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 300ab2ed0..2d48f677b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1965,7 +1965,7 @@ class address_space(objects.StructType): class page(objects.StructType): @property - @functools.cache + @functools.lru_cache() def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values From f8c25176f3cc07e2c0a3688b26166a92f386fb16 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Sun, 4 Aug 2024 09:30:37 -0700 Subject: [PATCH 319/979] #816 - add 17763, 18362, 19041 x64 support --- .../framework/plugins/windows/consoles.py | 15 +- .../consoles-win10-17763-3232-x64.json | 595 ++++++++++++++++++ .../consoles/consoles-win10-18362-x64.json | 585 +++++++++++++++++ .../consoles/consoles-win10-19041-x64.json | 585 +++++++++++++++++ 4 files changed, 1774 insertions(+), 6 deletions(-) create mode 100644 volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json create mode 100644 volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json create mode 100644 volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index 9b41b2890..c9e8522a4 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -174,7 +174,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) except: # unsure what to raise here. Also, it might be useful to add some kind of fallback, - # either to a user-provided version or to another method to determine tcpip.sys's version + # either to a user-provided version or to another method to determine conhost.exe's version raise exceptions.VolatilityException( "Kernel Debug Structure missing VERSION/KUSER structure, unable to determine Windows version!" ) @@ -203,7 +203,10 @@ class Consoles(interfaces.plugins.PluginInterface): version_dict = {} else: version_dict = { - (10, 0, 17763, 0): "consoles-win10-17763-x64", + (10, 0, 17763, 1): "consoles-win10-17763-x64", + (10, 0, 17763, 3232): "consoles-win10-17763-3232-x64", + (10, 0, 18362, 0): "consoles-win10-18362-x64", + (10, 0, 19041, 0): "consoles-win10-19041-x64", (10, 0, 20348, 1): "consoles-win10-20348-x64", (10, 0, 20348, 1970): "consoles-win10-20348-1970-x64", (10, 0, 20348, 2461): "consoles-win10-20348-2461-x64", @@ -280,11 +283,11 @@ class Consoles(interfaces.plugins.PluginInterface): # try to grab the latest supported version of the current image NT version. If that symbol # version does not work, support has to be added manually. current_versions = [ - (nt_maj, nt_min, vers_min, tcpip_ver) - for nt_maj, nt_min, vers_min, tcpip_ver in version_dict + (nt_maj, nt_min, vers_min, conhost_ver) + for nt_maj, nt_min, vers_min, conhost_ver in version_dict if nt_maj == nt_major_version and nt_min == nt_minor_version - and tcpip_ver <= conhost_mod_version + and conhost_ver <= conhost_mod_version ] current_versions.sort() @@ -319,7 +322,7 @@ class Consoles(interfaces.plugins.PluginInterface): nt_symbol_table: str, config_path: str, ) -> str: - """Creates a symbol table for TCP Listeners and TCP/UDP Endpoints. + """Creates a symbol table for conhost structures. Args: context: The context to retrieve required elements (layers, symbol tables) from diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json new file mode 100644 index 000000000..ba17645c4 --- /dev/null +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json @@ -0,0 +1,595 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "short": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_CONSOLE_INFORMATION": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 32 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 34 + }, + "CommandHistorySize": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 144 + }, + "HistoryBufferMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 148 + }, + "OriginalTitle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1736 + }, + "Title": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1672 + }, + "GetScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1800 + }, + "CurrentScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1384 + }, + "ConsoleProcessList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 1360 + }, + "ProcessCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1368 + }, + "HistoryList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": -760 + }, + "HistoryBufferCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": -752 + }, + "ExeAliasList": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1392 + } + }, + "kind": "struct", + "size": 140 + }, + "_VECTOR": { + "fields": { + "Begin": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 0 + }, + "End": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 8 + }, + "EndCapacity": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + }, + "_CONSOLE_PROCESS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ConsoleProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_CONSOLE_PROCESS_HANDLE": { + "fields": { + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 52 + }, + "_CONSOLE_PROCESS": { + "fields": { + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 28 + }, + "ThreadId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + }, + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND_HISTORY": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "CommandBucket": { + "type": { + "kind": "struct", + "name": "_VECTOR" + }, + "offset": 16 + }, + "CommandCountMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 40 + }, + "Application": { + "type": { + "kind": "struct", + "name": "_COMMAND" + }, + "offset": 48 + }, + "ConsoleProcessHandle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS_HANDLE" + } + }, + "offset": 80 + }, + "Flags": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 88 + }, + "LastDisplayed": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 92 + } + }, + "kind": "struct", + "size": 96 + }, + "_SCREEN_INFORMATION": { + "fields": { + "TextBufferInfo": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 56 + }, + "Next": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 64 + } + }, + "kind": "struct", + "size": 72 + }, + "_ROW_POINTER": { + "fields": { + "Row": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + + }, + "_ROWS_ARRAY": { + "fields": { + "Rows": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_ROW_POINTER" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_TEXT_BUFFER_INFO": { + "fields": { + "BufferDeque": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_DEQUE" + } + }, + "offset": 0 + }, + "BufferRows": { + "type": { + "kind": "pointer", + "name": "void", + "subtype": { + "kind": "struct", + "name": "_ROWS_ARRAY" + } + }, + "offset": 8 + }, + "BufferCapacity": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 32 + }, + "ThisBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 48 + }, + "BufferEnd": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 56 + }, + "BufferLastIndex": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 58 + }, + "BufferStart": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 112 + } + }, + "kind": "struct", + "size": 72 + }, + "_CHAR_ROW_CELL": { + "fields": { + "Text": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "DbcsAttribute": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 2 + } + }, + "kind": "struct", + "size": 3 + }, + "_CHAR_ROW_CELL_ARRAY": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROW": { + "fields": { + "CharRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL_ARRAY" + } + }, + "offset": 8 + }, + "RowLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 64 + }, + "Index": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 72 + }, + "RowLength2": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 80 + }, + "TextBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 88 + } }, + "kind": "struct", + "size": 96 + }, + "_DEQUE": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle", + "datetime": "2024-07-31T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json new file mode 100644 index 000000000..3c1fde26e --- /dev/null +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json @@ -0,0 +1,585 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "short": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_CONSOLE_INFORMATION": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 32 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 34 + }, + "CommandHistorySize": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 144 + }, + "HistoryBufferMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 148 + }, + "OriginalTitle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1744 + }, + "Title": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1680 + }, + "GetScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1808 + }, + "CurrentScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1392 + }, + "ConsoleProcessList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 1368 + }, + "ProcessCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1376 + }, + "HistoryList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": -768 + }, + "HistoryBufferCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": -760 + }, + "ExeAliasList": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1436 + } + }, + "kind": "struct", + "size": 140 + }, + "_VECTOR": { + "fields": { + "Begin": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 0 + }, + "End": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 8 + }, + "EndCapacity": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + }, + "_CONSOLE_PROCESS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ConsoleProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_CONSOLE_PROCESS_HANDLE": { + "fields": { + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 52 + }, + "_CONSOLE_PROCESS": { + "fields": { + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 28 + }, + "ThreadId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + }, + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND_HISTORY": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "CommandBucket": { + "type": { + "kind": "struct", + "name": "_VECTOR" + }, + "offset": 16 + }, + "CommandCountMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 40 + }, + "Application": { + "type": { + "kind": "struct", + "name": "_COMMAND" + }, + "offset": 48 + }, + "ConsoleProcessHandle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS_HANDLE" + } + }, + "offset": 80 + }, + "Flags": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 88 + }, + "LastDisplayed": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 92 + } + }, + "kind": "struct", + "size": 96 + }, + "_SCREEN_INFORMATION": { + "fields": { + "TextBufferInfo": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 56 + }, + "Next": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 64 + } + }, + "kind": "struct", + "size": 72 + }, + "_ROW_POINTER": { + "fields": { + "Row": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + + }, + "_ROWS_ARRAY": { + "fields": { + "Rows": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_ROW_POINTER" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_TEXT_BUFFER_INFO": { + "fields": { + "BufferDeque": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_DEQUE" + } + }, + "offset": 0 + }, + "BufferRows": { + "type": { + "kind": "pointer", + "name": "void", + "subtype": { + "kind": "struct", + "name": "_ROWS_ARRAY" + } + }, + "offset": 8 + }, + "BufferCapacity": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 32 + }, + "BufferEnd": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 56 + }, + "BufferLastIndex": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 58 + }, + "BufferStart": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 88 + } + }, + "kind": "struct", + "size": 72 + }, + "_CHAR_ROW_CELL": { + "fields": { + "Text": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "DbcsAttribute": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 2 + } + }, + "kind": "struct", + "size": 3 + }, + "_CHAR_ROW_CELL_ARRAY": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROW": { + "fields": { + "CharRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL_ARRAY" + } + }, + "offset": 8 + }, + "RowLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 64 + }, + "Index": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 72 + }, + "RowLength2": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 80 + }, + "TextBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 88 + } }, + "kind": "struct", + "size": 96 + }, + "_DEQUE": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle", + "datetime": "2024-07-31T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json new file mode 100644 index 000000000..39ca5f0eb --- /dev/null +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json @@ -0,0 +1,585 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "short": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_CONSOLE_INFORMATION": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 32 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 34 + }, + "CommandHistorySize": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 144 + }, + "HistoryBufferMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 148 + }, + "OriginalTitle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1752 + }, + "Title": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1688 + }, + "GetScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1816 + }, + "CurrentScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1392 + }, + "ConsoleProcessList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 1368 + }, + "ProcessCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1376 + }, + "HistoryList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": -352 + }, + "HistoryBufferCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": -344 + }, + "ExeAliasList": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1436 + } + }, + "kind": "struct", + "size": 140 + }, + "_VECTOR": { + "fields": { + "Begin": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 0 + }, + "End": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 8 + }, + "EndCapacity": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + }, + "_CONSOLE_PROCESS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ConsoleProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_CONSOLE_PROCESS_HANDLE": { + "fields": { + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 52 + }, + "_CONSOLE_PROCESS": { + "fields": { + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 28 + }, + "ThreadId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + }, + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND_HISTORY": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "CommandBucket": { + "type": { + "kind": "struct", + "name": "_VECTOR" + }, + "offset": 16 + }, + "CommandCountMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 40 + }, + "Application": { + "type": { + "kind": "struct", + "name": "_COMMAND" + }, + "offset": 48 + }, + "ConsoleProcessHandle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS_HANDLE" + } + }, + "offset": 80 + }, + "Flags": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 88 + }, + "LastDisplayed": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 92 + } + }, + "kind": "struct", + "size": 96 + }, + "_SCREEN_INFORMATION": { + "fields": { + "TextBufferInfo": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 56 + }, + "Next": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 64 + } + }, + "kind": "struct", + "size": 72 + }, + "_ROW_POINTER": { + "fields": { + "Row": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + + }, + "_ROWS_ARRAY": { + "fields": { + "Rows": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_ROW_POINTER" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_TEXT_BUFFER_INFO": { + "fields": { + "BufferDeque": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_DEQUE" + } + }, + "offset": 0 + }, + "BufferRows": { + "type": { + "kind": "pointer", + "name": "void", + "subtype": { + "kind": "struct", + "name": "_ROWS_ARRAY" + } + }, + "offset": 8 + }, + "BufferCapacity": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 32 + }, + "BufferEnd": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 56 + }, + "BufferLastIndex": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 58 + }, + "BufferStart": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 88 + } + }, + "kind": "struct", + "size": 72 + }, + "_CHAR_ROW_CELL": { + "fields": { + "Text": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "DbcsAttribute": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 2 + } + }, + "kind": "struct", + "size": 3 + }, + "_CHAR_ROW_CELL_ARRAY": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROW": { + "fields": { + "CharRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL_ARRAY" + } + }, + "offset": 8 + }, + "RowLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 64 + }, + "Index": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 72 + }, + "RowLength2": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 80 + }, + "TextBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 88 + } }, + "kind": "struct", + "size": 96 + }, + "_DEQUE": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle", + "datetime": "2024-07-31T15:05:35-06:00" + }, + "format": "4.1.0" + } +} From 9b9535d8d716599630ab4c09b92b7e331a832d8b Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Sun, 4 Aug 2024 10:14:36 -0700 Subject: [PATCH 320/979] #816 - other 20348 symbols --- .../consoles-win10-20348-1970-x64.json | 595 ++++++++++++++++++ 1 file changed, 595 insertions(+) create mode 100644 volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json new file mode 100644 index 000000000..e0935b4be --- /dev/null +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json @@ -0,0 +1,595 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "short": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_CONSOLE_INFORMATION": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 24 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 26 + }, + "CommandHistorySize": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 136 + }, + "HistoryBufferMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 140 + }, + "OriginalTitle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1616 + }, + "Title": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1552 + }, + "GetScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1680 + }, + "CurrentScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1296 + }, + "ConsoleProcessList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 1272 + }, + "ProcessCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1280 + }, + "HistoryList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 9320 + }, + "HistoryBufferCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 9328 + }, + "ExeAliasList": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 2410 + } + }, + "kind": "struct", + "size": 140 + }, + "_VECTOR": { + "fields": { + "Begin": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 0 + }, + "End": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 8 + }, + "EndCapacity": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + }, + "_CONSOLE_PROCESS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ConsoleProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_CONSOLE_PROCESS_HANDLE": { + "fields": { + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 52 + }, + "_CONSOLE_PROCESS": { + "fields": { + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 28 + }, + "ThreadId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + }, + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND_HISTORY": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "CommandBucket": { + "type": { + "kind": "struct", + "name": "_VECTOR" + }, + "offset": 16 + }, + "CommandCountMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 40 + }, + "Application": { + "type": { + "kind": "struct", + "name": "_COMMAND" + }, + "offset": 48 + }, + "ConsoleProcessHandle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS_HANDLE" + } + }, + "offset": 80 + }, + "Flags": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 88 + }, + "LastDisplayed": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 92 + } + }, + "kind": "struct", + "size": 96 + }, + "_SCREEN_INFORMATION": { + "fields": { + "TextBufferInfo": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 56 + }, + "Next": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 64 + } + }, + "kind": "struct", + "size": 72 + }, + "_ROW_POINTER": { + "fields": { + "Row": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + + }, + "_ROWS_ARRAY": { + "fields": { + "Rows": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_ROW_POINTER" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_TEXT_BUFFER_INFO": { + "fields": { + "BufferDeque": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_DEQUE" + } + }, + "offset": 0 + }, + "BufferRows": { + "type": { + "kind": "pointer", + "name": "void", + "subtype": { + "kind": "struct", + "name": "_ROWS_ARRAY" + } + }, + "offset": 16 + }, + "BufferCapacity": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 40 + }, + "ThisBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 48 + }, + "BufferEnd": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 54 + }, + "BufferLastIndex": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 56 + }, + "BufferStart": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 96 + } + }, + "kind": "struct", + "size": 72 + }, + "_CHAR_ROW_CELL": { + "fields": { + "Text": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "DbcsAttribute": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 2 + } + }, + "kind": "struct", + "size": 3 + }, + "_CHAR_ROW_CELL_ARRAY": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROW": { + "fields": { + "CharRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL_ARRAY" + } + }, + "offset": 8 + }, + "RowLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 64 + }, + "Index": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 72 + }, + "RowLength2": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 80 + }, + "TextBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 88 + } }, + "kind": "struct", + "size": 96 + }, + "_DEQUE": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle", + "datetime": "2024-07-31T15:05:35-06:00" + }, + "format": "4.1.0" + } +} From 8d6fd3cd78f0fadd223ae93a70a268048b4ccfe9 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Mon, 5 Aug 2024 14:28:43 +0200 Subject: [PATCH 321/979] Moved get_inode_metadata, separated inode and FD processing, error handling precision --- volatility3/framework/plugins/linux/lsof.py | 65 ++++++++++++------- .../framework/symbols/linux/__init__.py | 26 +------- 2 files changed, 45 insertions(+), 46 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index fa9d2bf61..167556e7d 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -6,7 +6,7 @@ found in Linux's /proc file system.""" import logging, datetime from typing import List, Callable -from volatility3.framework import renderers, interfaces, constants +from volatility3.framework import renderers, interfaces, constants, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -46,7 +46,30 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ] @classmethod - def list_fds_and_inodes( + def get_inode_metadata(cls, filp: interfaces.objects.ObjectInterface): + try: + dentry = filp.get_dentry() + if dentry: + inode_object = dentry.d_inode + if inode_object and inode_object.is_valid(): + itype = ( + inode_object.get_inode_type() or renderers.NotAvailableValue() + ) + return ( + inode_object.i_ino, + itype, + inode_object.i_size, + inode_object.get_file_mode(), + inode_object.get_change_time(), + inode_object.get_modification_time(), + inode_object.get_access_time(), + ) + except (exceptions.InvalidAddressException, AttributeError) as e: + vollog.warning(f"Can't get inode metadata: {e}") + return tuple(renderers.NotAvailableValue() for _ in range(7)) + + @classmethod + def list_fds( cls, context: interfaces.context.ContextInterface, symbol_table: str, @@ -67,26 +90,27 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ) for fd_fields in fd_generator: - fd_num, filp, full_path = fd_fields - inode_metadata = linux.LinuxUtilities.get_inode_metadata(context, filp) - try: - inode_num, itype, file_size, imode, ctime, mtime, atime = next( - inode_metadata - ) - except Exception as e: - vollog.warning( - f"Can't get inode metadata for file descriptor {fd_num}: {e}" - ) - inode_num = itype = file_size = imode = ctime = mtime = atime = ( - renderers.NotAvailableValue() - ) - yield pid, task_comm, task, fd_num, filp, full_path, inode_num, itype, imode, ctime, mtime, atime, file_size + yield pid, task_comm, task, fd_fields + + @classmethod + def list_fds_and_inodes( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + filter_func: Callable[[int], bool] = lambda _: False, + ): + for pid, task_comm, task, (fd_num, filp, full_path) in cls.list_fds( + context, symbol_table, filter_func + ): + inode_metadata = cls.get_inode_metadata(filp) + yield pid, task_comm, task, fd_num, filp, full_path, inode_metadata def _generator(self, pids, symbol_table): filter_func = pslist.PsList.create_pid_filter(pids) fds_generator = self.list_fds_and_inodes( self.context, symbol_table, filter_func=filter_func ) + for ( pid, task_comm, @@ -94,14 +118,9 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): fd_num, filp, full_path, - inode_num, - itype, - imode, - ctime, - mtime, - atime, - file_size, + inode_metadata, ) in fds_generator: + inode_num, itype, file_size, imode, ctime, mtime, atime = inode_metadata fields = ( pid, task_comm, diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index d52c43dae..03353135d 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -1,8 +1,8 @@ -# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # from typing import Iterator, List, Tuple, Optional, Union -import datetime, stat + from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility @@ -67,7 +67,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 2, 0) + _version = (2, 1, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -274,26 +274,6 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): yield fd_num, filp, full_path - @classmethod - def get_inode_metadata(cls, context: interfaces.context.ContextInterface, filp): - """ - A helper function that gets the inodes metadata from a file descriptor - """ - dentry = filp.get_dentry() - if dentry: - inode_object = dentry.d_inode - if inode_object and inode_object.is_valid(): - itype = inode_object.get_inode_type() or "?" - yield ( - inode_object.i_ino, - itype, - inode_object.i_size, - inode_object.get_file_mode(), - inode_object.get_change_time(), - inode_object.get_modification_time(), - inode_object.get_access_time(), - ) - @classmethod def mask_mods_list( cls, From c9eb81c95fa530c58e493d77cde506f001e9e4f3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 04:40:48 -0700 Subject: [PATCH 322/979] PR review fixes: Improve eBPF extension objects: bpf_prog and added bpf_prog_aux. Apply changes to the EBPF and Sockstat plugins. --- volatility3/framework/plugins/linux/ebpf.py | 15 ++---- .../framework/plugins/linux/sockstat.py | 14 +++--- .../framework/symbols/linux/__init__.py | 1 + .../symbols/linux/extensions/__init__.py | 48 +++++++++++++++++-- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 33ba71faf..8df506b06 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -1,12 +1,10 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import binascii import logging from typing import List from volatility3.framework import renderers, interfaces, exceptions -from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins from volatility3.framework.configuration import requirements @@ -53,18 +51,11 @@ class EBPF(plugins.PluginInterface): def _generator(self): vmlinux = self.context.modules[self.config["kernel"]] - vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] - bpf_prog_types = vmlinux.get_enumeration("bpf_prog_type") for prog in self.get_ebpf_programs(vmlinux): prog_addr = prog.vol.offset - prog_type = bpf_prog_types.lookup(prog.type) - prog_tag_addr = prog.tag.vol.offset - prog_tag_size = prog.tag.count - prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) - prog_tag = binascii.hexlify(prog_tag_bytes).decode() - prog_name = ( - utility.array_to_string(prog.aux.name) or renderers.NotAvailableValue() - ) + prog_type = prog.get_type() or renderers.NotAvailableValue() + prog_tag = prog.get_tag() or renderers.NotAvailableValue() + prog_name = prog.get_name() or renderers.NotAvailableValue() fields = (format_hints.Hex(prog_addr), prog_name, prog_tag, prog_type) yield (0, fields) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 78217fbec..b0503b105 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -151,17 +151,15 @@ class SockHandlers(interfaces.configuration.VersionableInterface): bpfprog = sock_filter.prog - BPF_PROG_TYPE_UNSPEC = 0 # cBPF filter - try: - bpfprog_type = bpfprog.get_type() - if bpfprog_type == BPF_PROG_TYPE_UNSPEC: - return # cBPF filter - except AttributeError: + bpfprog_type = bpfprog.get_type() + if not bpfprog_type: # kernel < 3.18.140, it's a cBPF filter return None - BPF_PROG_TYPE_SOCKET_FILTER = 1 # eBPF filter - if bpfprog_type != BPF_PROG_TYPE_SOCKET_FILTER: + if bpfprog_type == "BPF_PROG_TYPE_UNSPEC": + return None # cBPF filter + + if bpfprog_type != "BPF_PROG_TYPE_SOCKET_FILTER": socket_filter["bpf_filter_type"] = f"UNK({bpfprog_type})" vollog.warning(f"Unexpected BPF type {bpfprog_type} for a socket") return None diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 248cb8d75..7a87135ff 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -38,6 +38,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) + self.optional_set_type_class("bpf_prog_aux", extensions.bpf_prog_aux) self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 2d48f677b..2b971fb79 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -5,6 +5,7 @@ import collections.abc import logging import functools +import binascii import stat from datetime import datetime import socket as socket_module @@ -1607,20 +1608,59 @@ class xdp_sock(objects.StructType): class bpf_prog(objects.StructType): - def get_type(self): + def get_type(self) -> Union[str, None]: + """Returns a string with the eBPF program type""" + # The program type was in `bpf_prog_aux::prog_type` from 3.18.140 to # 4.1.52 before it was moved to `bpf_prog::type` if self.has_member("type"): # kernel >= 4.1.52 - return self.type + return self.type.description if self.has_member("aux") and self.aux: if self.aux.has_member("prog_type"): # 3.18.140 <= kernel < 4.1.52 - return self.aux.prog_type + return self.aux.prog_type.description # kernel < 3.18.140 - raise AttributeError("Unable to find the BPF type") + return None + + def get_tag(self) -> Union[str, None]: + """Returns a string with the eBPF program tag""" + # 'tag' was added in kernels 4.10 + if not self.has_member("tag"): + return None + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + + prog_tag_addr = self.tag.vol.offset + prog_tag_size = self.tag.count + prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) + + prog_tag = binascii.hexlify(prog_tag_bytes).decode() + return prog_tag + + def get_name(self) -> Union[str, None]: + """Returns a string with the eBPF program name""" + if not self.has_member("aux"): + # 'prog_aux' was added in kernels 3.18 + return None + + return self.aux.get_name() + + +class bpf_prog_aux(objects.StructType): + def get_name(self) -> Union[str, None]: + """Returns a string with the eBPF program name""" + if not self.has_member("name"): + # 'name' was added in kernels 4.15 + return None + + if not self.name: + return None + + return utility.array_to_string(self.name) class cred(objects.StructType): From bd37aa3930c056fc8511969592f731a29e253c84 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 04:48:49 -0700 Subject: [PATCH 323/979] PR review fixes: Fix pidhashtable plugin --- volatility3/framework/plugins/linux/pidhashtable.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index b24c73e77..ef110bdf1 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -191,7 +191,7 @@ class PIDHashTable(plugins.PluginInterface): "numbers" ) # kernels >= 2.6.24 - has_pid_numbers = self.vmlinux.has_type("upid") and self.vmlinux.get_type( + has_pid_chain = self.vmlinux.has_type("upid") and self.vmlinux.get_type( "upid" ).has_member( "pid_chain" @@ -205,7 +205,7 @@ class PIDHashTable(plugins.PluginInterface): if pid_idr: # kernels >= 4.15 return self._pid_namespace_idr - elif pid_hash and has_pid_numbers and has_pid_numbers: + elif pid_hash and has_pid_numbers and has_pid_numbers and has_pid_chain: # 2.6.24 <= kernels < 4.15 return self._pid_hash_implementation From 7df0636f30d47f1d5b9373671aac410549474766 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 04:57:33 -0700 Subject: [PATCH 324/979] PR review fixes: Fix pidhashtable plugin explicit returns mixed with implicit returns --- volatility3/framework/plugins/linux/pidhashtable.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index ef110bdf1..73cdff452 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -50,19 +50,19 @@ class PIDHashTable(plugins.PluginInterface): self.vmlinux_layer = None def _is_valid_task(self, task): - return task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent) + return bool(task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent)) def _get_pidtype_pid(self): # The pid_type enumeration is present since 2.5.37, just in case pid_type_enum = self.vmlinux.get_enumeration("pid_type") if not pid_type_enum: vollog.error("Cannot find pid_type enum. Unsupported kernel") - return + return None pidtype_pid = pid_type_enum.choices.get("PIDTYPE_PID") if pidtype_pid is None: vollog.error("Cannot find PIDTYPE_PID. Unsupported kernel") - return + return None # Typically PIDTYPE_PID = 0 return pidtype_pid @@ -154,7 +154,7 @@ class PIDHashTable(plugins.PluginInterface): pid_tasks_0 = pid.tasks[pidtype_pid].first if not pid_tasks_0: - return + return None task_struct_type = self.vmlinux.get_type("task_struct") if task_struct_type.has_member("pids"): From 1e3e9e2c78cbfc4e24362c323432660c98373516 Mon Sep 17 00:00:00 2001 From: Arcuri Davide Date: Tue, 6 Aug 2024 16:28:59 +0200 Subject: [PATCH 325/979] add args and kwargs to threads.py init Without args and kwargs there were an issue with timeliner plugin that tried to pass additional parameters like progress_callback raising TypeError --- volatility3/framework/plugins/windows/threads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index ae70e717b..39f3b7d77 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -18,9 +18,9 @@ class Threads(thrdscan.ThrdScan): _required_framework_version = (2, 4, 0) _version = (1, 0, 0) - def __init__(self): + def __init__(self, *args, **kwargs): self.implementation = self.list_process_threads - super().__init__() + super().__init__(*args, **kwargs) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 1d43eb305d993a1306cc3971333d8a28bfee03af Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Tue, 6 Aug 2024 15:18:49 -0700 Subject: [PATCH 326/979] fix invalid json on all files --- .../symbols/windows/consoles/consoles-win10-17763-3232-x64.json | 1 - .../symbols/windows/consoles/consoles-win10-17763-x64.json | 1 - .../symbols/windows/consoles/consoles-win10-18362-x64.json | 1 - .../symbols/windows/consoles/consoles-win10-19041-x64.json | 1 - .../symbols/windows/consoles/consoles-win10-20348-1970-x64.json | 1 - .../symbols/windows/consoles/consoles-win10-20348-2461-x64.json | 1 - 6 files changed, 6 deletions(-) diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json index ba17645c4..0a8cb5782 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json @@ -434,7 +434,6 @@ "BufferRows": { "type": { "kind": "pointer", - "name": "void", "subtype": { "kind": "struct", "name": "_ROWS_ARRAY" diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json index f11fb39b1..2b98945c9 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json @@ -434,7 +434,6 @@ "BufferRows": { "type": { "kind": "pointer", - "name": "void", "subtype": { "kind": "struct", "name": "_ROWS_ARRAY" diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json index 3c1fde26e..3d76d1cb4 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json @@ -434,7 +434,6 @@ "BufferRows": { "type": { "kind": "pointer", - "name": "void", "subtype": { "kind": "struct", "name": "_ROWS_ARRAY" diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json index 39ca5f0eb..681113b0f 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json @@ -434,7 +434,6 @@ "BufferRows": { "type": { "kind": "pointer", - "name": "void", "subtype": { "kind": "struct", "name": "_ROWS_ARRAY" diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json index e0935b4be..7bc6ae9d1 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json @@ -434,7 +434,6 @@ "BufferRows": { "type": { "kind": "pointer", - "name": "void", "subtype": { "kind": "struct", "name": "_ROWS_ARRAY" diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json index a4adf8028..f612c527a 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json @@ -434,7 +434,6 @@ "BufferRows": { "type": { "kind": "pointer", - "name": "void", "subtype": { "kind": "struct", "name": "_ROWS_ARRAY" From ac08f42cfd9b9f75067c7f0bd9162d15be05c8a2 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Sun, 4 Aug 2024 12:09:59 -0700 Subject: [PATCH 327/979] #816 - fix invalid json --- .../symbols/windows/consoles/consoles-win10-20348-x64.json | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json index 5cc7ab6f5..083a966a0 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json @@ -434,7 +434,6 @@ "BufferRows": { "type": { "kind": "pointer", - "name": "void", "subtype": { "kind": "struct", "name": "_ROWS_ARRAY" From 339a9a94f57adb2039984facf0cd22df0ba4bf90 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 23:00:09 -0700 Subject: [PATCH 328/979] PR review fixes: pidhashtable plugin add missing typing. --- volatility3/framework/plugins/linux/pidhashtable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 73cdff452..3c429dc2f 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -49,7 +49,7 @@ class PIDHashTable(plugins.PluginInterface): self.vmlinux = None self.vmlinux_layer = None - def _is_valid_task(self, task): + def _is_valid_task(self, task) -> bool: return bool(task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent)) def _get_pidtype_pid(self): From c8cb4465da3d71a879b47d981fdd4871afd9d521 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 23:08:21 -0700 Subject: [PATCH 329/979] PR review fixes: Remove filter function, it isn't needed --- volatility3/framework/plugins/linux/mountinfo.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 319c92cca..dfb2e2f52 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -254,10 +254,7 @@ class MountInfo(plugins.PluginInterface): super_block: Kernel's struct super_block object """ # No filter so that we get all the mount namespaces from all tasks - pid_filter = pslist.PsList.create_pid_filter() - tasks = pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=pid_filter - ) + tasks = pslist.PsList.list_tasks(self.context, self.config["kernel"]) seen_sb_ptr = set() for task, mnt, _mnt_ns_id in self._get_tasks_mountpoints(tasks): From f737b88d03d9be9f4f0a9b43a83e02a88a620068 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 6 Aug 2024 23:12:01 -0700 Subject: [PATCH 330/979] PR review fixes: Improve _walk_dentry() and get_inodes() variable names, arguments and return values --- .../framework/plugins/linux/pagecache.py | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index c36f8a339..cf09f23d8 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List +from typing import List, Set from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints @@ -153,7 +153,23 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): return symlink_path @classmethod - def _walk_dentry(cls, seen_dentries, root_dentry, parent): + def _walk_dentry( + cls, + seen_dentries: Set[int], + root_dentry: interfaces.objects.ObjectInterface, + parent_dir: str, + ): + """Walk dentries recursively + + Args: + seen_dentries: A set to ensure each dentry is processed only once + root_dentry: Root dentry object + parent_dir: Parent directory path + + Yields: + file_path: Filename including path + dentry: Dentry object + """ for dentry in root_dentry.get_subdirs(): dentry_addr = dentry.vol.offset @@ -173,19 +189,16 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): # This allows us to have consistent paths if dentry.d_name.name: - name = dentry.d_name.name_as_str() + basename = dentry.d_name.name_as_str() # Do NOT use os.path.join() below - new_file = parent + "/" + name + file_path = parent_dir + "/" + basename else: continue - yield new_file, dentry, dentry.d_parent.vol.offset + yield file_path, dentry if inode.is_dir: - for new_file, dentry, parent_address in cls._walk_dentry( - seen_dentries, dentry, new_file - ): - yield new_file, dentry, parent_address + yield from cls._walk_dentry(seen_dentries, dentry, parent_dir=file_path) @classmethod def get_inodes( @@ -211,13 +224,15 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): seen_inodes = set() seen_dentries = set() for superblock, mountpoint in superblocks_iter: - parent = "" if mountpoint == "/" else mountpoint + parent_dir = "" if mountpoint == "/" else mountpoint # Superblock root dentry - root_dentry = superblock.s_root - if not root_dentry: + root_dentry_ptr = superblock.s_root + if not root_dentry_ptr: continue + root_dentry = root_dentry_ptr.dereference() + # Dentry sanity check if not root_dentry.is_root(): continue @@ -246,8 +261,8 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): yield inode_in # Children - for file_path, file_dentry, _ in cls._walk_dentry( - seen_dentries, root_dentry, parent + for file_path, file_dentry in cls._walk_dentry( + seen_dentries, root_dentry, parent_dir ): if not file_dentry: continue From 805b3514c3b14c53724cd38d722540d84949d51f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 7 Aug 2024 00:23:19 -0700 Subject: [PATCH 331/979] PR review fixes: Fix page flags list method name, this was introduced earlier in another commit of this PR. --- volatility3/framework/plugins/linux/pagecache.py | 2 +- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index cf09f23d8..fd62cc1e4 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -486,7 +486,7 @@ class InodePages(plugins.PluginInterface): page_index = int(page_obj.index) page_file_offset = page_index * vmlinux_layer.page_size dump_safe = page_file_offset < inode_size - page_flags_list = page_obj.get_flags() + page_flags_list = page_obj.get_flags_list() page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) fields = ( page_vaddr, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 2b971fb79..900cc9b6c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2027,7 +2027,7 @@ class page(objects.StructType): return pageflags_enum - def flags_list(self) -> List[str]: + def get_flags_list(self) -> List[str]: """Returns a list of page flags Returns: From 46842981fb25aa6f1b242ef5990642fdeeb0cf00 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 7 Aug 2024 00:24:58 -0700 Subject: [PATCH 332/979] PR review fixes: Use contextlib.suppress() instead of an empty exception handler --- volatility3/framework/symbols/linux/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 7a87135ff..96bc56a18 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import math +import contextlib from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union @@ -676,15 +677,13 @@ class RadixTree(Tree): return self.RADIX_TREE_INTERNAL_NODE def get_tree_height(self, treep) -> int: - try: + with contextlib.suppress(exceptions.SymbolError): if self.vmlinux.get_type("radix_tree_root").has_member("height"): # kernels < 4.7.10 radix_tree_root = self.vmlinux.object( "radix_tree_root", offset=treep, absolute=True ) return radix_tree_root.height - except exceptions.SymbolError: - pass # kernels >= 4.7.10 return 0 From 10376a2687b8546df88bf6b3a3e11126f8816479 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Wed, 7 Aug 2024 10:38:29 +0200 Subject: [PATCH 333/979] use one class with flag --- README.md | 2 +- pyproject.toml | 2 +- volatility3/framework/__init__.py | 2 +- .../framework/plugins/windows/vadyarascan.py | 7 +- volatility3/framework/plugins/yarascan.py | 81 ++++++++----------- 5 files changed, 38 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 886790df8..1463c2bde 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.7.3 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.8.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/pyproject.toml b/pyproject.toml index 207f762cc..2e1636a43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, ] -requires-python = ">=3.7.3" +requires-python = ">=3.8.0" license = { text = "VSL" } dynamic = ["dependencies", "optional-dependencies", "version"] diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 74db773cf..51310bfa2 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 7, 3) +required_python_version = (3, 8, 0) if ( sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 4a84a1285..7bc3377c3 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -73,8 +73,9 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ) continue + data = layer.read(start, size, True) if not yarascan.YaraScan._yara_x: - for match in rules.match(data=layer.read(start, size, True)): + for match in rules.match(data=data): if yarascan.YaraScan.yara_returns_instances(): for match_string in match.strings: for instance in match_string.instances: @@ -95,9 +96,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): value, ) else: - data = layer.read(start, size, True) - results = rules.scan(data) - for match in results.matching_rules: + for match in rules.scan(data).matching_rules: for match_string in match.patterns: for instance in match_string.matches: yield 0, ( diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 6a4dd9251..310bbd072 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -36,7 +36,7 @@ except ImportError: raise -class BaseYaraScanner(interfaces.layers.ScannerInterface): +class YaraScanner(interfaces.layers.ScannerInterface): _version = (2, 1, 0) # yara.Rules isn't exposed, so we can't type this properly @@ -45,32 +45,44 @@ class BaseYaraScanner(interfaces.layers.ScannerInterface): if rules is None: raise ValueError("No rules provided to YaraScanner") self._rules = rules - - -class YaraPythonScanner(BaseYaraScanner): - def __init__(self, rules) -> None: - super().__init__(rules) - self.st_object = not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) + self.st_object = ( + None + if USE_YARA_X + else not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3) + ) def __call__( self, data: bytes, data_offset: int ) -> Iterable[Tuple[int, str, str, bytes]]: - for match in self._rules.match(data=data): - if YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: + if USE_YARA_X: + for match in self._rules.scan(data).matching_rules: + for match_string in match.patterns: + for instance in match_string.matches: yield ( instance.offset + data_offset, - match.rule, + f"{match.namespace}.{match.identifier}", match_string.identifier, - instance.matched_data, + data[instance.offset : instance.offset + instance.length], ) - else: - for offset, name, value in match.strings: - yield (offset + data_offset, match.rule, name, value) + else: + for match in self._rules.match(data=data): + if YaraScan.yara_returns_instances(): + for match_string in match.strings: + for instance in match_string.instances: + yield ( + instance.offset + data_offset, + match.rule, + match_string.identifier, + instance.matched_data, + ) + else: + for offset, name, value in match.strings: + yield (offset + data_offset, match.rule, name, value) @staticmethod def get_rule(rule): + if USE_YARA_X: + return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") return yara.compile( sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} ) @@ -78,47 +90,18 @@ class YaraPythonScanner(BaseYaraScanner): @staticmethod def from_compiled_file(filepath): with resources.ResourceAccessor().open(filepath, "rb") as fp: + if USE_YARA_X: + return yara_x.Rules.deserialize_from(file=fp) return yara.load(file=fp) @staticmethod def from_file(filepath): with resources.ResourceAccessor().open(filepath, "rb") as fp: + if USE_YARA_X: + return yara_x.compile(fp.read().decode()) return yara.compile(file=fp) -class YaraXScanner(BaseYaraScanner): - def __call__( - self, data: bytes, data_offset: int - ) -> Iterable[Tuple[int, str, str, bytes]]: - results = self._rules.scan(data) - for match in results.matching_rules: - for match_string in match.patterns: - for instance in match_string.matches: - yield ( - instance.offset + data_offset, - f"{match.namespace}.{match.identifier}", - match_string.identifier, - data[instance.offset : instance.offset + instance.length], - ) - - @staticmethod - def get_rule(rule): - return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") - - @staticmethod - def from_compiled_file(filepath): - with resources.ResourceAccessor().open(filepath, "rb") as fp: - return yara_x.Rules.deserialize_from(file=fp) - - @staticmethod - def from_file(filepath): - with resources.ResourceAccessor().open(filepath, "rb") as fp: - return yara_x.compile(fp.read().decode()) - - -YaraScanner = YaraXScanner if USE_YARA_X else YaraPythonScanner - - class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" From de6637c9871ac16f1a5278c4831dfe60778852b9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 00:52:34 -0700 Subject: [PATCH 334/979] PR review fixes: ebpf plugin code improvement. Use the object_from_symbol() instead --- volatility3/framework/plugins/linux/ebpf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 8df506b06..9d41c0ffb 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -43,8 +43,7 @@ class EBPF(plugins.PluginInterface): "Cannot find the eBPF prog idr. Unsupported kernel" ) - prog_idr_addr = vmlinux.get_symbol("prog_idr").address - prog_idr = vmlinux.object("idr", offset=prog_idr_addr) + prog_idr = vmlinux.object_from_symbol("prog_idr") for page_addr in prog_idr.get_page_addresses(): bpf_prog = vmlinux.object("bpf_prog", offset=page_addr, absolute=True) yield bpf_prog From ee10ba8abb7b8c932d9af85a37edefc3fd03fa63 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 01:46:53 -0700 Subject: [PATCH 335/979] PR review fixes: Fix IDR explicit returns mixed with implicit returns and improve and fix code. --- .../symbols/linux/extensions/__init__.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 900cc9b6c..6eafd4173 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2134,7 +2134,7 @@ class IDR(objects.StructType): """Finds an ID within the IDR data structure. Based on idr_find_slowpath(), 3.9 <= Kernel < 4.11 Args: - idr_id: The IDR element ID + idr_id: The IDR lookup ID Returns: A pointer to the given ID element @@ -2144,28 +2144,28 @@ class IDR(objects.StructType): vollog.info( "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" ) - return + return None if idr_id < 0: - return + return None - cur_layer = self.top - if not cur_layer: - return + idr_layer = self.top + if not idr_layer: + return None - n = (cur_layer.layer + 1) * self.IDR_BITS + n = (idr_layer.layer + 1) * self.IDR_BITS - if idr_id > self.idr_max(cur_layer.layer + 1): - return + if idr_id > self.idr_max(idr_layer.layer + 1): + return None assert n != 0 - while n > 0 and cur_layer: + while n > 0 and idr_layer: n -= self.IDR_BITS - assert n == cur_layer.layer * self.IDR_BITS - cur_layer = cur_layer.ary[(idr_id >> n) & self.IDR_MASK] + assert n == idr_layer.layer * self.IDR_BITS + idr_layer = idr_layer.ary[(idr_id >> n) & self.IDR_MASK] - return cur_layer.v() + return idr_layer def _old_kernel_get_page_addresses(self, in_use) -> int: # Kernels < 4.11 From 0f3f33863370f68b7da3069db5b29282dabab932 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 01:52:57 -0700 Subject: [PATCH 336/979] PR review fixes: Since the IDR, XArray and RadixTree can store any value, it renames the function names to a more generic name --- volatility3/framework/plugins/linux/ebpf.py | 2 +- .../framework/plugins/linux/pidhashtable.py | 2 +- volatility3/framework/symbols/linux/__init__.py | 4 ++-- .../symbols/linux/extensions/__init__.py | 16 +++++++++------- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 9d41c0ffb..70082daf5 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -44,7 +44,7 @@ class EBPF(plugins.PluginInterface): ) prog_idr = vmlinux.object_from_symbol("prog_idr") - for page_addr in prog_idr.get_page_addresses(): + for page_addr in prog_idr.get_entries(): bpf_prog = vmlinux.object("bpf_prog", offset=page_addr, absolute=True) yield bpf_prog diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 3c429dc2f..c384cb5cd 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -175,7 +175,7 @@ class PIDHashTable(plugins.PluginInterface): ns_addr = self.vmlinux.get_symbol("init_pid_ns").address ns = self.vmlinux.object("pid_namespace", offset=ns_addr) - for page_addr in ns.idr.get_page_addresses(): + for page_addr in ns.idr.get_entries(): task = self._task_for_radix_pid_node(page_addr) if self._is_valid_task(task): yield task diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 96bc56a18..89f970275 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -572,7 +572,7 @@ class Tree(ABC): for child_node in self._iter_node(nodep, height - 1): yield child_node - def get_page_addresses(self, root: interfaces.objects.ObjectInterface) -> int: + def get_entries(self, root: interfaces.objects.ObjectInterface) -> int: """Walks the tree data structure Args: @@ -762,7 +762,7 @@ class PageCache(object): Page objects """ - for page_addr in self._tree.get_page_addresses(self._page_cache.i_pages): + for page_addr in self._tree.get_entries(self._page_cache.i_pages): if not page_addr: continue diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 6eafd4173..29af5c510 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2167,7 +2167,7 @@ class IDR(objects.StructType): return idr_layer - def _old_kernel_get_page_addresses(self, in_use) -> int: + def _old_kernel_get_entries(self) -> int: # Kernels < 4.11 total = next_id = 0 while total < in_use: @@ -2178,14 +2178,14 @@ class IDR(objects.StructType): next_id += 1 - def _new_kernel_get_page_addresses(self, _in_use) -> int: + def _new_kernel_get_entries(self) -> int: # Kernels >= 4.11 vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) tree = linux.LinuxUtilities.choose_kernel_tree(vmlinux) - for page_addr in tree.get_page_addresses(root=self.idr_rt): + for page_addr in tree.get_entries(root=self.idr_rt): yield page_addr - def get_page_addresses(self, in_use=0) -> int: + def get_entries(self) -> int: """Walks the IDR and yield a pointer associated with each element. Args: @@ -2195,9 +2195,11 @@ class IDR(objects.StructType): A pointer associated with each element. """ if self.has_member("idr_rt"): - get_page_addresses_func = self._new_kernel_get_page_addresses + # Kernels >= 4.11 + get_entries_func = self._new_kernel_get_entries else: - get_page_addresses_func = self._old_kernel_get_page_addresses + # Kernels < 4.11 + get_entries_func = self._old_kernel_get_entries - for page_addr in get_page_addresses_func(in_use): + for page_addr in get_entries_func(): yield page_addr From 7a8dea3356c709cda2bdd20b7cb3ff97f1f7987d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 01:57:35 -0700 Subject: [PATCH 337/979] PR review fixes: Code scanning complains about these unused variables. Let's comment them and adapt the FIXME message --- volatility3/framework/symbols/linux/extensions/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 29af5c510..9b03235cb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2063,10 +2063,9 @@ class page(objects.StructType): vmemmap_start = vmemmap_base_l4 else: # 5-Level paging -> VMEMMAP_START = __VMEMMAP_BASE_L5 - vmemmap_base_l5 = 0xFFD4000000000000 - vmemmap_start = vmemmap_base_l5 - - # FIXME: Remove this exception once 5-level paging is supported. + # FIXME: Once 5-level paging is supported, uncomment the following lines and remove the exception + # vmemmap_base_l5 = 0xFFD4000000000000 + # vmemmap_start = vmemmap_base_l5 raise exceptions.VolatilityException( "5-level paging is not yet supported" ) From 06508a4afba235812e7d1b7bfb79467adf2ed6fc Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 02:00:27 -0700 Subject: [PATCH 338/979] PR review fixes: Fix the IDR's old kernel get_entries --- .../framework/symbols/linux/extensions/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9b03235cb..f7b0df6be 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2168,11 +2168,12 @@ class IDR(objects.StructType): def _old_kernel_get_entries(self) -> int: # Kernels < 4.11 + cur = self.cur total = next_id = 0 - while total < in_use: - page_addr = self.idr_find(next_id) - if page_addr: - yield page_addr + while next_id < cur: + entry = self.idr_find(next_id) + if entry: + yield entry total += 1 next_id += 1 From 2c85ea525e15c0cb745f3b806836f22ea44a4b4c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 8 Aug 2024 02:03:28 -0700 Subject: [PATCH 339/979] PR review fixes: Fix page extension object get_content() explicit returns mixed with implicit returns. --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f7b0df6be..be4df6a13 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2102,7 +2102,7 @@ class page(objects.StructType): physical_layer = vmlinux.context.layers["memory_layer"] page_paddr = self.to_paddr() if not page_paddr: - return + return None page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) return page_data From f22575669a6ccd9afaeef126e81a6adad8b880f6 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 9 Aug 2024 10:20:28 +0200 Subject: [PATCH 340/979] Modifications following the review --- volatility3/framework/plugins/linux/lsof.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 167556e7d..9a0fd7417 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -66,7 +66,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ) except (exceptions.InvalidAddressException, AttributeError) as e: vollog.warning(f"Can't get inode metadata: {e}") - return tuple(renderers.NotAvailableValue() for _ in range(7)) + return None @classmethod def list_fds( @@ -103,6 +103,10 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): context, symbol_table, filter_func ): inode_metadata = cls.get_inode_metadata(filp) + if inode_metadata is None: + inode_metadata = tuple( + interfaces.renderers.BaseAbsentValue() for _ in range(7) + ) yield pid, task_comm, task, fd_num, filp, full_path, inode_metadata def _generator(self, pids, symbol_table): From 53f3d12341e722f1058c42436adfea600af94bab Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 9 Aug 2024 23:35:41 -0700 Subject: [PATCH 341/979] linuxutilities code improvement. Remove code duplication --- volatility3/framework/symbols/linux/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 89f970275..90f5cc8a2 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -419,9 +419,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): Returns: A kernel object (vmlinux) """ - symbol_table_arr = volobj.vol.type_name.split("!", 1) - symbol_table = symbol_table_arr[0] if len(symbol_table_arr) == 2 else None - + symbol_table = volobj.get_symbol_table_name() module_names = context.modules.get_modules_by_symbol_tables(symbol_table) module_names = list(module_names) From 17861618df3744d572955b50b2b1b1ad1d0961e5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 10 Aug 2024 00:13:38 -0700 Subject: [PATCH 342/979] PR review fixes: Rename Tree to IDStorage. Move choose_id_storage() form LinuxUtilities to IDStorage. Use context and kernel_module_name instead of vmlinux --- .../framework/symbols/linux/__init__.py | 87 +++++++++++-------- .../symbols/linux/extensions/__init__.py | 14 +-- 2 files changed, 59 insertions(+), 42 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 90f5cc8a2..632ac2f6b 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -431,17 +431,51 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return kernel + +class IDStorage(ABC): + """Abstraction to support both XArray and RadixTree""" + + # Dynamic values, these will be initialized later + CHUNK_SHIFT = None + CHUNK_SIZE = None + CHUNK_MASK = None + + def __init__( + self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ): + self.vmlinux = context.modules[kernel_module_name] + self.vmlinux_layer = self.vmlinux.context.layers[self.vmlinux.layer_name] + + self.pointer_size = self.vmlinux.get_type("pointer").size + # Dynamically work out the (XA_CHUNK|RADIX_TREE_MAP)_SHIFT values based on + # the node.slots[] array size + node_type = self.vmlinux.get_type(self.node_type_name) + slots_array_size = node_type.child_template("slots").count + + # Calculate the LSB index - 1 + self.CHUNK_SHIFT = slots_array_size.bit_length() - 1 + self.CHUNK_SIZE = 1 << self.CHUNK_SHIFT + self.CHUNK_MASK = self.CHUNK_SIZE - 1 + @classmethod - def choose_kernel_tree(cls, vmlinux: interfaces.context.ModuleInterface) -> "Tree": - """Returns the appropriate tree data structure instance for the current kernel implementation. + def choose_id_storage( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ) -> "IDStorage": + """Returns the appropriate ID storage data structure instance for the current kernel implementation. This is used by the IDR and the PageCache to choose between the XArray and RadixTree. Args: - vmlinux: The kernel module object + context: The context to retrieve required elements (layers, symbol tables) from + kernel_module_name: The name of the kernel module on which to operate Returns: - The appropriate Tree instance for the current kernel + The appropriate ID storage instance for the current kernel """ + vmlinux = context.modules[kernel_module_name] address_space_type = vmlinux.get_type("address_space") address_space_has_i_pages = address_space_type.has_member("i_pages") i_pages_type_name = ( @@ -455,33 +489,9 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ) and vmlinux.get_type("radix_tree_root").has_member("xa_head") if i_pages_is_xarray or i_pages_is_radix_tree_root: - return XArray(vmlinux) + return XArray(context, kernel_module_name) else: - return RadixTree(vmlinux) - - -class Tree(ABC): - """Abstraction to support both XArray and RadixTree""" - - # Dynamic values, these will be initialized later - CHUNK_SHIFT = None - CHUNK_SIZE = None - CHUNK_MASK = None - - def __init__(self, vmlinux: interfaces.context.ModuleInterface): - self.vmlinux = vmlinux - self.vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] - - self.pointer_size = self.vmlinux.get_type("pointer").size - # Dynamically work out the (XA_CHUNK|RADIX_TREE_MAP)_SHIFT values based on - # the node.slots[] array size - node_type = self.vmlinux.get_type(self.node_type_name) - slots_array_size = node_type.child_template("slots").count - - # Calculate the LSB index - 1 - self.CHUNK_SHIFT = slots_array_size.bit_length() - 1 - self.CHUNK_SIZE = 1 << self.CHUNK_SHIFT - self.CHUNK_MASK = self.CHUNK_SIZE - 1 + return RadixTree(context, kernel_module_name) @property @abstractmethod @@ -601,7 +611,7 @@ class Tree(ABC): yield child_node -class XArray(Tree): +class XArray(IDStorage): XARRAY_TAG_MASK = 3 XARRAY_TAG_INTERNAL = 2 @@ -637,7 +647,7 @@ class XArray(Tree): return not self.is_node_tagged(nodep) -class RadixTree(Tree): +class RadixTree(IDStorage): RADIX_TREE_INTERNAL_NODE = 1 RADIX_TREE_EXCEPTIONAL_ENTRY = 2 RADIX_TREE_ENTRY_MASK = 3 @@ -741,17 +751,20 @@ class PageCache(object): def __init__( self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, page_cache: interfaces.objects.ObjectInterface, - vmlinux: interfaces.context.ModuleInterface, ): """ Args: + context: interfaces.context.ContextInterface, + kernel_module_name: The name of the kernel module on which to operate page_cache: Page cache address space - vmlinux: Kernel module object """ - self.vmlinux = vmlinux + self.vmlinux = context.modules[kernel_module_name] + self._page_cache = page_cache - self._tree = LinuxUtilities.choose_kernel_tree(self.vmlinux) + self._idstorage = IDStorage.choose_id_storage(context, kernel_module_name) def get_cached_pages(self) -> interfaces.objects.ObjectInterface: """Returns all page cache contents @@ -760,7 +773,7 @@ class PageCache(object): Page objects """ - for page_addr in self._tree.get_entries(self._page_cache.i_pages): + for page_addr in self._idstorage.get_entries(self._page_cache.i_pages): if not page_addr: continue diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index be4df6a13..d00af7a3f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1972,8 +1972,11 @@ class inode(objects.StructType): elif not (self.i_mapping and self.i_mapping.nrpages > 0): return - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - page_cache = linux.PageCache(self.i_mapping.dereference(), vmlinux) + page_cache = linux.PageCache( + context=self._context, + kernel_module_name="kernel", + page_cache=self.i_mapping.dereference(), + ) yield from page_cache.get_cached_pages() def get_contents(self): @@ -2180,9 +2183,10 @@ class IDR(objects.StructType): def _new_kernel_get_entries(self) -> int: # Kernels >= 4.11 - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - tree = linux.LinuxUtilities.choose_kernel_tree(vmlinux) - for page_addr in tree.get_entries(root=self.idr_rt): + id_storage = linux.IDStorage.choose_id_storage( + self._context, kernel_module_name="kernel" + ) + for page_addr in id_storage.get_entries(root=self.idr_rt): yield page_addr def get_entries(self) -> int: From 8f9d565f6300750ad30c0aaca0aa01cfbcb1a417 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 10 Aug 2024 00:49:18 -0700 Subject: [PATCH 343/979] PR review fixes: Fix minor typo to match verb form from other docstrings --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index fd62cc1e4..f062655a6 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -159,7 +159,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): root_dentry: interfaces.objects.ObjectInterface, parent_dir: str, ): - """Walk dentries recursively + """Walks dentries recursively Args: seen_dentries: A set to ensure each dentry is processed only once From f804b44ff6f8451e8f1091383b8d9c2300f7bfbb Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Aug 2024 01:00:57 -0700 Subject: [PATCH 344/979] Fix test.yaml, it should remove *.bin and not *.lime. There is no *.lime atm. --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 55b2e4b60..668b814ce 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,7 +46,7 @@ jobs: - name: Clean up post-test run: | - rm -rf *.lime + rm -rf *.bin rm -rf *.img cd volatility3/symbols rm -rf linux From 7efa2210a550d55f356b6ebb35cafcdfd4a58d1f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Aug 2024 01:05:44 -0700 Subject: [PATCH 345/979] PR review fixes: Adjust LinuxUtilities version since we moved choose_id_storage() back to the IDStorage class. --- volatility3/framework/plugins/linux/pidhashtable.py | 2 +- volatility3/framework/symbols/linux/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index c384cb5cd..9a2528543 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -34,7 +34,7 @@ class PIDHashTable(plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.VersionRequirement( - name="linuxutils", component=linux.LinuxUtilities, version=(2, 2, 0) + name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), requirements.BooleanRequirement( name="decorate_comm", diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 632ac2f6b..91abf7db4 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -74,7 +74,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 2, 0) + _version = (2, 1, 1) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) From 918584537fafd67b2367e283871889bf291b2951 Mon Sep 17 00:00:00 2001 From: Steven Luke <97394870+sluke-nuix@users.noreply.github.com> Date: Tue, 13 Aug 2024 08:34:35 -0400 Subject: [PATCH 346/979] Update the modules.Modules version requirement. This is a response to PR [#1173](https://github.com/volatilityfoundation/volatility3/pull/1173) --- volatility3/framework/plugins/windows/truecrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py index 81250a749..7fd26cb4e 100644 --- a/volatility3/framework/plugins/windows/truecrypt.py +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -33,7 +33,7 @@ class Passphrase(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name="modules", component=modules.Modules, version=(1, 1, 0) + name="modules", component=modules.Modules, version=(2, 0, 0) ), requirements.IntRequirement( name="min-length", From 87fe3444ebb0d3bf7595c6cefc24f04145ff8dae Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 14 Aug 2024 23:50:39 -0700 Subject: [PATCH 347/979] Linux: Add support for mount namespace in kernels 6.8+. This fixes volatilityfoundation/volatility3#1187 --- .../framework/symbols/linux/__init__.py | 33 +++++++++++++++ .../symbols/linux/extensions/__init__.py | 42 +++++++++++++++---- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 03353135d..525492141 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -425,3 +425,36 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): kernel = context.modules[kernel_module_name] return kernel + + +class RBTree(object): + """Simple Red-Black tree abstraction""" + + def __init__(self, root): + self.root = root + + def _walk_nodes(self, root_node) -> Iterator[int]: + """Traverses the Red-Black tree from the root node and yields a pointer to each + node in this tree. + + Args: + root_node: A Red-Black tree node from which to start descending + + Yields: + A pointer to every node descending from the specified root node + """ + if not root_node: + return + + yield root_node + yield from self._walk_nodes(root_node.rb_left) + yield from self._walk_nodes(root_node.rb_right) + + def get_nodes(self) -> Iterator[int]: + """Yields a pointer to each node in the Red-Black tree + + Yields: + A pointer to every node in the Red-Black tree + """ + + yield from self._walk_nodes(root_node=self.root.rb_node) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 05679523f..645f493cb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1299,14 +1299,42 @@ class mnt_namespace(objects.StructType): else: raise AttributeError("Unable to find mnt_namespace inode") - def get_mount_points(self): + def get_mount_points( + self, + ) -> Iterator[interfaces.objects.ObjectInterface]: + """Yields the mount points for this mount namespace. + + Yields: + mount struct instances + """ table_name = self.vol.type_name.split(constants.BANG)[0] - mnt_type = table_name + constants.BANG + "mount" - if not self._context.symbol_space.has_type(mnt_type): - # Old kernels ~ 2.6 - mnt_type = table_name + constants.BANG + "vfsmount" - for mount in self.list.to_list(mnt_type, "mnt_list"): - yield mount + + if self.has_member("list"): + # kernels < 6.8 + mnt_type = table_name + constants.BANG + "mount" + if not self._context.symbol_space.has_type(mnt_type): + # In kernels < 3.3, the 'mount' struct didn't exist, and the 'mnt_list' + # member was part of the 'vfsmount' struct. + mnt_type = table_name + constants.BANG + "vfsmount" + + yield from self.list.to_list(mnt_type, "mnt_list") + elif ( + self.has_member("mounts") + and self.mounts.vol.type_name == table_name + constants.BANG + "rb_root" + ): + # kernels >= 6.8 + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type( + self._context, self + ) + for node in linux.RBTree(self.mounts).get_nodes(): + mnt = linux.LinuxUtilities.container_of( + node, "mount", "mnt_list", vmlinux + ) + yield mnt + else: + raise exceptions.VolatilityException( + "Unsupported kernel mount namespace implementation" + ) class net(objects.StructType): From 4513806a7ec7f8c4da746a7ffde952d9f170a07f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 15 Aug 2024 07:55:45 -0700 Subject: [PATCH 348/979] Move RBTree to the rb_root extension object for better integration --- .../framework/symbols/linux/__init__.py | 34 +------------------ .../symbols/linux/extensions/__init__.py | 30 +++++++++++++++- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 525492141..c2eb3d791 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -46,6 +46,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): # Might not exist in older kernels or the current symbols self.optional_set_type_class("mount", extensions.mount) self.optional_set_type_class("mnt_namespace", extensions.mnt_namespace) + self.optional_set_type_class("rb_root", extensions.rb_root) # Network self.set_type_class("net", extensions.net) @@ -425,36 +426,3 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): kernel = context.modules[kernel_module_name] return kernel - - -class RBTree(object): - """Simple Red-Black tree abstraction""" - - def __init__(self, root): - self.root = root - - def _walk_nodes(self, root_node) -> Iterator[int]: - """Traverses the Red-Black tree from the root node and yields a pointer to each - node in this tree. - - Args: - root_node: A Red-Black tree node from which to start descending - - Yields: - A pointer to every node descending from the specified root node - """ - if not root_node: - return - - yield root_node - yield from self._walk_nodes(root_node.rb_left) - yield from self._walk_nodes(root_node.rb_right) - - def get_nodes(self) -> Iterator[int]: - """Yields a pointer to each node in the Red-Black tree - - Yields: - A pointer to every node in the Red-Black tree - """ - - yield from self._walk_nodes(root_node=self.root.rb_node) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 645f493cb..c18cd093d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1326,7 +1326,7 @@ class mnt_namespace(objects.StructType): vmlinux = linux.LinuxUtilities.get_module_from_volobj_type( self._context, self ) - for node in linux.RBTree(self.mounts).get_nodes(): + for node in self.mounts.get_nodes(): mnt = linux.LinuxUtilities.container_of( node, "mount", "mnt_list", vmlinux ) @@ -1925,3 +1925,31 @@ class inode(objects.StructType): The inode's file mode string """ return stat.filemode(self.i_mode) + + +class rb_root(objects.StructType): + def _walk_nodes(self, root_node) -> Iterator[int]: + """Traverses the Red-Black tree from the root node and yields a pointer to each + node in this tree. + + Args: + root_node: A Red-Black tree node from which to start descending + + Yields: + A pointer to every node descending from the specified root node + """ + if not root_node: + return + + yield root_node + yield from self._walk_nodes(root_node.rb_left) + yield from self._walk_nodes(root_node.rb_right) + + def get_nodes(self) -> Iterator[int]: + """Yields a pointer to each node in the Red-Black tree + + Yields: + A pointer to every node in the Red-Black tree + """ + + yield from self._walk_nodes(root_node=self.rb_node) From 8d7edfdca6fa84983b4ee734e3f45a9d07c28ff1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 21 Aug 2024 20:36:09 +0100 Subject: [PATCH 349/979] Bump as the release branch has been cut --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index f219fb0af..4df0b9041 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 8 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From 71cdca5883b234680773e000a663750964d4860e Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Thu, 22 Aug 2024 16:45:04 +0200 Subject: [PATCH 350/979] Updating version + docstring --- volatility3/framework/plugins/linux/lsof.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 9a0fd7417..360f89749 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -18,10 +18,10 @@ vollog = logging.getLogger(__name__) class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): - """Lists all memory maps for all processes.""" + """Lists open files for each processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (1, 2, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From feae6a9aa0f5c7869174e0b906ec83e61b3e14e3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 00:42:52 +1000 Subject: [PATCH 351/979] PR review fixes: Use plugin's open method instead of the builtin open() --- volatility3/framework/plugins/linux/pagecache.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index f062655a6..cf4151c85 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set +from typing import List, Set, Type from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints @@ -408,6 +408,7 @@ class InodePages(plugins.PluginInterface): def write_inode_content_to_file( inode: interfaces.objects.ObjectInterface, filename: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a file @@ -415,6 +416,7 @@ class InodePages(plugins.PluginInterface): Args: inode: The inode to dump filename: Filename for writing the inode content + open_method: class for constructing output files vmlinux_layer: The kernel layer to obtain the page size """ if not inode.is_reg: @@ -426,7 +428,7 @@ class InodePages(plugins.PluginInterface): # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. try: - with open(filename, "wb") as f: + with open_method(filename) as f: inode_size = inode.i_size f.truncate(inode_size) @@ -502,7 +504,7 @@ class InodePages(plugins.PluginInterface): if self.config["dump"]: filename = self.config["dump"] vollog.info("[*] Writing inode at 0x%x to '%s'", inode.vol.offset, filename) - self.write_inode_content_to_file(inode, filename, vmlinux_layer) + self.write_inode_content_to_file(inode, filename, self.open, vmlinux_layer) def run(self): headers = [ From 3c70c1b9f7c2251e35eea276f5d5099b68392a48 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 00:58:04 +1000 Subject: [PATCH 352/979] PR review fixes: Use context and module_name instead of vmlinux in ebpf plugin --- volatility3/framework/plugins/linux/ebpf.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/ebpf.py b/volatility3/framework/plugins/linux/ebpf.py index 70082daf5..2267dd922 100644 --- a/volatility3/framework/plugins/linux/ebpf.py +++ b/volatility3/framework/plugins/linux/ebpf.py @@ -29,15 +29,21 @@ class EBPF(plugins.PluginInterface): ), ] - def get_ebpf_programs(self, vmlinux) -> interfaces.objects.ObjectInterface: + def get_ebpf_programs( + self, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> interfaces.objects.ObjectInterface: """Enumerate eBPF programs walking its IDR. Args: - vmlinux: The kernel symbols object - + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate Yields: eBPF program objects """ + vmlinux = context.modules[vmlinux_module_name] + if not vmlinux.has_symbol("prog_idr"): raise exceptions.VolatilityException( "Cannot find the eBPF prog idr. Unsupported kernel" @@ -49,8 +55,7 @@ class EBPF(plugins.PluginInterface): yield bpf_prog def _generator(self): - vmlinux = self.context.modules[self.config["kernel"]] - for prog in self.get_ebpf_programs(vmlinux): + for prog in self.get_ebpf_programs(self.context, self.config["kernel"]): prog_addr = prog.vol.offset prog_type = prog.get_type() or renderers.NotAvailableValue() prog_tag = prog.get_tag() or renderers.NotAvailableValue() From 695635199e2bd3af11bee522e054719ee1898db6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 01:14:09 +1000 Subject: [PATCH 353/979] PR review fixes: Avoid saving state in the pidhashtable plugin --- .../framework/plugins/linux/pidhashtable.py | 79 +++++++++++-------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 9a2528543..3223aed4a 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -44,17 +44,16 @@ class PIDHashTable(plugins.PluginInterface): ), ] - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.vmlinux = None - self.vmlinux_layer = None - def _is_valid_task(self, task) -> bool: - return bool(task and task.pid > 0 and self.vmlinux_layer.is_valid(task.parent)) + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + return bool(task and task.pid > 0 and vmlinux_layer.is_valid(task.parent)) def _get_pidtype_pid(self): + vmlinux = self.context.modules[self.config["kernel"]] + # The pid_type enumeration is present since 2.5.37, just in case - pid_type_enum = self.vmlinux.get_enumeration("pid_type") + pid_type_enum = vmlinux.get_enumeration("pid_type") if not pid_type_enum: vollog.error("Cannot find pid_type enum. Unsupported kernel") return None @@ -68,38 +67,46 @@ class PIDHashTable(plugins.PluginInterface): return pidtype_pid def _get_pidhash_array(self): - pidhash_shift = self.vmlinux.object_from_symbol("pidhash_shift") + vmlinux = self.context.modules[self.config["kernel"]] + + pidhash_shift = vmlinux.object_from_symbol("pidhash_shift") pidhash_size = 1 << pidhash_shift - array_type_name = self.vmlinux.symbol_table_name + constants.BANG + "array" + array_type_name = vmlinux.symbol_table_name + constants.BANG + "array" - pidhash_ptr = self.vmlinux.object_from_symbol("pid_hash") + pidhash_ptr = vmlinux.object_from_symbol("pid_hash") # pidhash is an array of hlist_heads pidhash = self._context.object( array_type_name, offset=pidhash_ptr, - subtype=self.vmlinux.get_type("hlist_head"), + subtype=vmlinux.get_type("hlist_head"), count=pidhash_size, - layer_name=self.vmlinux.layer_name, + layer_name=vmlinux.layer_name, ) return pidhash def _walk_upid(self, seen_upids, upid): - while upid and self.vmlinux_layer.is_valid(upid.vol.offset): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + + while upid and vmlinux_layer.is_valid(upid.vol.offset): if upid.vol.offset in seen_upids: break seen_upids.add(upid.vol.offset) pid_chain = upid.pid_chain - if not (pid_chain and self.vmlinux_layer.is_valid(pid_chain.vol.offset)): + if not (pid_chain and vmlinux_layer.is_valid(pid_chain.vol.offset)): break upid = linux.LinuxUtilities.container_of( - pid_chain.next, "upid", "pid_chain", self.vmlinux + pid_chain.next, "upid", "pid_chain", vmlinux ) def _get_upids(self): + vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_layer = self.context.layers[vmlinux.layer_name] + # 2.6.24 <= kernels < 4.15 pidhash = self._get_pidhash_array() @@ -108,10 +115,10 @@ class PIDHashTable(plugins.PluginInterface): # each entry in the hlist is a upid which is wrapped in a pid ent = hlist.first - while ent and self.vmlinux_layer.is_valid(ent.vol.offset): + while ent and vmlinux_layer.is_valid(ent.vol.offset): # upid->pid_chain exists 2.6.24 <= kernel < 4.15 upid = linux.LinuxUtilities.container_of( - ent.vol.offset, "upid", "pid_chain", self.vmlinux + ent.vol.offset, "upid", "pid_chain", vmlinux ) if upid.vol.offset in seen_upids: @@ -124,16 +131,14 @@ class PIDHashTable(plugins.PluginInterface): return seen_upids def _pid_hash_implementation(self): + vmlinux = self.context.modules[self.config["kernel"]] + # 2.6.24 <= kernels < 4.15 - task_pids_off = self.vmlinux.get_type("task_struct").relative_child_offset( - "pids" - ) + task_pids_off = vmlinux.get_type("task_struct").relative_child_offset("pids") pidtype_pid = self._get_pidtype_pid() for upid in self._get_upids(): - pid = linux.LinuxUtilities.container_of( - upid, "pid", "numbers", self.vmlinux - ) + pid = linux.LinuxUtilities.container_of(upid, "pid", "numbers", vmlinux) if not pid: continue @@ -141,22 +146,24 @@ class PIDHashTable(plugins.PluginInterface): if not pid_tasks_0: continue - task = self.vmlinux.object( + task = vmlinux.object( "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True ) if self._is_valid_task(task): yield task def _task_for_radix_pid_node(self, nodep): + vmlinux = self.context.modules[self.config["kernel"]] + # kernels >= 4.15 - pid = self.vmlinux.object("pid", offset=nodep, absolute=True) + pid = vmlinux.object("pid", offset=nodep, absolute=True) pidtype_pid = self._get_pidtype_pid() pid_tasks_0 = pid.tasks[pidtype_pid].first if not pid_tasks_0: return None - task_struct_type = self.vmlinux.get_type("task_struct") + task_struct_type = vmlinux.get_type("task_struct") if task_struct_type.has_member("pids"): member = "pids" elif task_struct_type.has_member("pid_links"): @@ -165,15 +172,17 @@ class PIDHashTable(plugins.PluginInterface): return None task_pids_off = task_struct_type.relative_child_offset(member) - task = self.vmlinux.object( + task = vmlinux.object( "task_struct", offset=pid_tasks_0 - task_pids_off, absolute=True ) return task def _pid_namespace_idr(self): + vmlinux = self.context.modules[self.config["kernel"]] + # kernels >= 4.15 - ns_addr = self.vmlinux.get_symbol("init_pid_ns").address - ns = self.vmlinux.object("pid_namespace", offset=ns_addr) + ns_addr = vmlinux.get_symbol("init_pid_ns").address + ns = vmlinux.object("pid_namespace", offset=ns_addr) for page_addr in ns.idr.get_entries(): task = self._task_for_radix_pid_node(page_addr) @@ -181,24 +190,26 @@ class PIDHashTable(plugins.PluginInterface): yield task def _determine_pid_func(self): - pid_hash = self.vmlinux.has_symbol("pid_hash") and self.vmlinux.has_symbol( + vmlinux = self.context.modules[self.config["kernel"]] + + pid_hash = vmlinux.has_symbol("pid_hash") and vmlinux.has_symbol( "pidhash_shift" ) # 2.5.55 <= kernels < 4.15 - has_pid_numbers = self.vmlinux.has_type("pid") and self.vmlinux.get_type( + has_pid_numbers = vmlinux.has_type("pid") and vmlinux.get_type( "pid" ).has_member( "numbers" ) # kernels >= 2.6.24 - has_pid_chain = self.vmlinux.has_type("upid") and self.vmlinux.get_type( + has_pid_chain = vmlinux.has_type("upid") and vmlinux.get_type( "upid" ).has_member( "pid_chain" ) # 2.6.24 <= kernels < 4.15 # kernels >= 4.15 - pid_idr = self.vmlinux.has_type("pid_namespace") and self.vmlinux.get_type( + pid_idr = vmlinux.has_type("pid_namespace") and vmlinux.get_type( "pid_namespace" ).has_member("idr") @@ -217,8 +228,6 @@ class PIDHashTable(plugins.PluginInterface): Yields: task_struct objects """ - self.vmlinux = self.context.modules[self.config["kernel"]] - self.vmlinux_layer = self.context.layers[self.vmlinux.layer_name] pid_func = self._determine_pid_func() if not pid_func: vollog.error("Cannot determine which PID hash table this kernel is using") From d964e6f61de00b5a8608d48d468d9a59b7923ec7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 02:06:06 +1000 Subject: [PATCH 354/979] PR review fixes: Check for LinuxUtilities version everywhere we use it --- .../symbols/linux/extensions/__init__.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d00af7a3f..51dc37d31 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,6 +13,7 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion +from volatility3.framework.configuration import requirements from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1608,6 +1609,19 @@ class xdp_sock(objects.StructType): class bpf_prog(objects.StructType): + def _get_vmlinux(self): + linuxutils_required_version = (2, 1, 1) + linuxutils_current_version = linux.LinuxUtilities._version + if not requirements.VersionRequirement.matches_required( + linuxutils_required_version, linuxutils_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" + ) + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + return vmlinux + def get_type(self) -> Union[str, None]: """Returns a string with the eBPF program type""" @@ -1631,7 +1645,7 @@ class bpf_prog(objects.StructType): if not self.has_member("tag"): return None - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] prog_tag_addr = self.tag.vol.offset @@ -2043,13 +2057,26 @@ class page(objects.StructType): return flags + def _get_vmlinux(self): + linuxutils_required_version = (2, 1, 1) + linuxutils_current_version = linux.LinuxUtilities._version + if not requirements.VersionRequirement.matches_required( + linuxutils_required_version, linuxutils_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" + ) + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + return vmlinux + def to_paddr(self) -> int: """Converts a page's virtual address to its physical address using the current physical memory model. Returns: int: page physical address """ - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] vmemmap_start = None @@ -2100,7 +2127,7 @@ class page(objects.StructType): Returns: The page content """ - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] physical_layer = vmlinux.context.layers["memory_layer"] page_paddr = self.to_paddr() @@ -2118,6 +2145,19 @@ class IDR(objects.StructType): MAX_IDR_SHIFT = INT_SIZE * 8 - 1 MAX_IDR_BIT = 1 << MAX_IDR_SHIFT + def _get_vmlinux(self): + linuxutils_required_version = (2, 1, 1) + linuxutils_current_version = linux.LinuxUtilities._version + if not requirements.VersionRequirement.matches_required( + linuxutils_required_version, linuxutils_current_version + ): + raise exceptions.PluginRequirementException( + f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" + ) + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + return vmlinux + def idr_max(self, num_layers: int) -> int: """Returns the maximum ID which can be allocated given idr::layers @@ -2141,7 +2181,7 @@ class IDR(objects.StructType): Returns: A pointer to the given ID element """ - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux = self._get_vmlinux() if not vmlinux.get_type("idr_layer").has_member("layer"): vollog.info( "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" From d627f243e259efce2f730860f47d9116b3c78f3d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 10:55:01 +1000 Subject: [PATCH 355/979] PR review fixes: Add typing info to the get_inodes() class method. --- volatility3/framework/plugins/linux/pagecache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index cf4151c85..e384cbabf 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type +from typing import List, Set, Type, Iterable from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints @@ -205,7 +205,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, config_path: str, - ): + ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: From 90b327e63253404a98cc5a79e3cecaa1b773048c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 11:45:29 +1000 Subject: [PATCH 356/979] PR review fixes: Make mountinfo.get_superblocks() a classmethod and adapt the code using it. --- .../framework/plugins/linux/mountinfo.py | 19 +++++++++--- .../framework/plugins/linux/pagecache.py | 31 ++++++++++++------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index dfb2e2f52..1eaec77bf 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -37,7 +37,7 @@ class MountInfo(plugins.PluginInterface): _required_framework_version = (2, 2, 0) - _version = (1, 1, 0) + _version = (1, 2, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -143,8 +143,8 @@ class MountInfo(plugins.PluginInterface): sb_opts, ) + @staticmethod def _get_tasks_mountpoints( - self, tasks: Iterable[interfaces.objects.ObjectInterface], filtered_by_pids: bool = False, ): @@ -247,17 +247,26 @@ class MountInfo(plugins.PluginInterface): "Could not filter by mount namespace id. This field is not available in this kernel." ) - def get_superblocks(self): + @classmethod + def get_superblocks( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: """Yield file system superblocks based on the task's mounted filesystems. + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + Yields: super_block: Kernel's struct super_block object """ # No filter so that we get all the mount namespaces from all tasks - tasks = pslist.PsList.list_tasks(self.context, self.config["kernel"]) + tasks = pslist.PsList.list_tasks(context, vmlinux_module_name) seen_sb_ptr = set() - for task, mnt, _mnt_ns_id in self._get_tasks_mountpoints(tasks): + for task, mnt, _mnt_ns_id in cls._get_tasks_mountpoints(tasks): path_root = linux.LinuxUtilities.get_path_mnt(task, mnt) if not path_root: continue diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index e384cbabf..e54891480 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -115,7 +115,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 1, 0) + name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 2, 0) ), requirements.ListRequirement( name="type", @@ -204,22 +204,22 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): def get_inodes( cls, context: interfaces.context.ContextInterface, - config_path: str, + vmlinux_module_name: str, ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: context: The context that the plugin will operate within - config_path: The path to configuration data within the context configuration data + vmlinux_module_name: The name of the kernel module on which to operate Yields: An InodeInternal object """ - superblocks_iter = mountinfo.MountInfo( + superblocks_iter = mountinfo.MountInfo.get_superblocks( context=context, - config_path=config_path, - ).get_superblocks() + vmlinux_module_name=vmlinux_module_name, + ) seen_inodes = set() seen_dentries = set() @@ -289,11 +289,13 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): yield inode_in def _generator(self): - vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] vmlinux_layer = self.context.layers[vmlinux.layer_name] inodes_iter = self.get_inodes( - context=self.context, config_path=self.config_path + context=self.context, + vmlinux_module_name=vmlinux_module_name, ) types_filter = self.config["type"] @@ -316,12 +318,15 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): These need not be generated in any particular order, sorting will be done later """ - vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] vmlinux_layer = self.context.layers[vmlinux.layer_name] inodes_iter = self.get_inodes( - context=self.context, config_path=self.config_path + context=self.context, + vmlinux_module_name=vmlinux_module_name, ) + for inode_in in inodes_iter: inode_out = inode_in.to_user(vmlinux_layer) description = f"Cached Inode for {inode_out.path}" @@ -450,7 +455,8 @@ class InodePages(plugins.PluginInterface): vollog.error("Unable to write to file (%s): %s", filename, e) def _generator(self): - vmlinux = self.context.modules[self.config["kernel"]] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] vmlinux_layer = self.context.layers[vmlinux.layer_name] if self.config["inode"] and self.config["find"]: @@ -459,7 +465,8 @@ class InodePages(plugins.PluginInterface): if self.config["find"]: inodes_iter = Files.get_inodes( - context=self.context, config_path=self.config_path + context=self.context, + vmlinux_module_name=vmlinux_module_name, ) for inode_in in inodes_iter: if inode_in.path == self.config["find"]: From 3bf9f8cec0e1c4c75088abe17baadd0bc6d4c3d6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sat, 24 Aug 2024 12:03:10 +1000 Subject: [PATCH 357/979] PR review fixes: Add typing info to pagecache.Files._follow_symlink() --- volatility3/framework/plugins/linux/pagecache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index e54891480..b6c5f7cc0 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -131,7 +131,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): ] @staticmethod - def _follow_symlink(inode, symlink_path) -> str: + def _follow_symlink( + inode: interfaces.objects.ObjectInterface, + symlink_path: str, + ) -> str: """Follows (fast) symlinks (kernels >= 4.2.x). Fast symlinks are filesystem agnostic. From e97abae156721bc1c625729166c08325c164bd3a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 24 Aug 2024 15:02:31 +0100 Subject: [PATCH 358/979] Remove get_vmlinux calls Sorry, I know I asked for it, but I hadn't quite figured out what was going on. Things in the symbols/linux code are considered part of the framework, and therefore it's the framework version that should have been bumped. Since the framework comes packages with LinuxUtilities we can rely on the version numbers to be suitable. This cleans up the mess I caused, sorry for the extra work! 5:S --- volatility3/framework/constants/_version.py | 4 +- .../symbols/linux/extensions/__init__.py | 47 ++----------------- 2 files changed, 6 insertions(+), 45 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 4df0b9041..d30446d63 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 8 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 9 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 51dc37d31..fdd34403a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1609,19 +1609,6 @@ class xdp_sock(objects.StructType): class bpf_prog(objects.StructType): - def _get_vmlinux(self): - linuxutils_required_version = (2, 1, 1) - linuxutils_current_version = linux.LinuxUtilities._version - if not requirements.VersionRequirement.matches_required( - linuxutils_required_version, linuxutils_current_version - ): - raise exceptions.PluginRequirementException( - f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" - ) - - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - return vmlinux - def get_type(self) -> Union[str, None]: """Returns a string with the eBPF program type""" @@ -1645,7 +1632,7 @@ class bpf_prog(objects.StructType): if not self.has_member("tag"): return None - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] prog_tag_addr = self.tag.vol.offset @@ -2057,26 +2044,13 @@ class page(objects.StructType): return flags - def _get_vmlinux(self): - linuxutils_required_version = (2, 1, 1) - linuxutils_current_version = linux.LinuxUtilities._version - if not requirements.VersionRequirement.matches_required( - linuxutils_required_version, linuxutils_current_version - ): - raise exceptions.PluginRequirementException( - f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" - ) - - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - return vmlinux - def to_paddr(self) -> int: """Converts a page's virtual address to its physical address using the current physical memory model. Returns: int: page physical address """ - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] vmemmap_start = None @@ -2127,7 +2101,7 @@ class page(objects.StructType): Returns: The page content """ - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] physical_layer = vmlinux.context.layers["memory_layer"] page_paddr = self.to_paddr() @@ -2145,19 +2119,6 @@ class IDR(objects.StructType): MAX_IDR_SHIFT = INT_SIZE * 8 - 1 MAX_IDR_BIT = 1 << MAX_IDR_SHIFT - def _get_vmlinux(self): - linuxutils_required_version = (2, 1, 1) - linuxutils_current_version = linux.LinuxUtilities._version - if not requirements.VersionRequirement.matches_required( - linuxutils_required_version, linuxutils_current_version - ): - raise exceptions.PluginRequirementException( - f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" - ) - - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - return vmlinux - def idr_max(self, num_layers: int) -> int: """Returns the maximum ID which can be allocated given idr::layers @@ -2181,7 +2142,7 @@ class IDR(objects.StructType): Returns: A pointer to the given ID element """ - vmlinux = self._get_vmlinux() + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) if not vmlinux.get_type("idr_layer").has_member("layer"): vollog.info( "Unsupported IDR implementation, it should be a very very old kernel, probabably < 2.6" From 6a157a785f3364635de2c7a43eeab054ab7c6b3d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 27 Aug 2024 14:45:52 +0100 Subject: [PATCH 359/979] Bump the modules version number Pull-request #1173 bumped the version of the modules plugin (even though this only needed to be a MINOR version bump, see https://github.com/volatilityfoundation/volatility3/pull/1173#discussion_r1649614761), but failed to verify that other plugins which relied on it were also updated to make use of the new plugin. This was the version system working as intended, but highlighted a review failure that the neither the author, nor the reviewers, verified that the rest of the framework (specifically other plugins which relied on modules) worked correctly with the new code (which this kind of error is designed to fix). Fixes #1244. --- volatility3/framework/plugins/windows/ssdt.py | 2 +- volatility3/framework/plugins/windows/verinfo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/ssdt.py b/volatility3/framework/plugins/windows/ssdt.py index 6a47c36e9..1fcb6cc91 100644 --- a/volatility3/framework/plugins/windows/ssdt.py +++ b/volatility3/framework/plugins/windows/ssdt.py @@ -30,7 +30,7 @@ class SSDT(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="modules", plugin=modules.Modules, version=(1, 0, 0) + name="modules", plugin=modules.Modules, version=(2, 0, 0) ), ] diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 1c6615804..5b3c52bf6 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -46,7 +46,7 @@ class VerInfo(interfaces.plugins.PluginInterface): name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.PluginRequirement( - name="modules", plugin=modules.Modules, version=(1, 0, 0) + name="modules", plugin=modules.Modules, version=(2, 0, 0) ), requirements.VersionRequirement( name="dlllist", component=dlllist.DllList, version=(2, 0, 0) From a2196850bc01d5a1a41a2aa29685cdb1d2146e87 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Wed, 28 Aug 2024 10:39:28 +0200 Subject: [PATCH 360/979] increase python version to 3.8 --- .github/workflows/build-pypi.yml | 2 +- .github/workflows/install.yml | 2 +- .github/workflows/test.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 346d0337e..d1a63b4da 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.7"] + python-version: ["3.8"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index e13161085..398ff8ae3 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: host: [ ubuntu-latest, windows-latest ] - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + python-version: [ "3.8", "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 55b2e4b60..65dfb13c8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.7"] + python-version: ["3.8"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 7a479d617f8107590533eaed9afe105c3cd3b34f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 28 Aug 2024 18:40:41 +0200 Subject: [PATCH 361/979] add OS and framework architectures constants --- .../framework/constants/architectures.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 volatility3/framework/constants/architectures.py diff --git a/volatility3/framework/constants/architectures.py b/volatility3/framework/constants/architectures.py new file mode 100644 index 000000000..a8822d7d3 --- /dev/null +++ b/volatility3/framework/constants/architectures.py @@ -0,0 +1,21 @@ +from volatility3.framework.layers import intel + +WIN_ARCHS = ["Intel32", "Intel64"] +"""Windows supported architectures""" +WIN_ARCHS_LAYERS = [intel.Intel] +"""Windows supported architectures layers""" + +LINUX_ARCHS = ["Intel32", "Intel64"] +"""Linux supported architectures""" +LINUX_ARCHS_LAYERS = [intel.Intel] +"""Linux supported architectures layers""" + +MAC_ARCHS = ["Intel32", "Intel64"] +"""Mac supported architectures""" +MAC_ARCHS_LAYERS = [intel.Intel] +"""Mac supported architectures layers""" + +FRAMEWORK_ARCHS = ["Intel32", "Intel64"] +"""Framework supported architectures""" +FRAMEWORK_ARCHS_LAYERS = [intel.Intel] +"""Framework supported architectures layers""" From 46a26c7dc5f5bc87d93027a6c005acbf85bc8716 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 2 Sep 2024 12:17:16 -0500 Subject: [PATCH 362/979] Address feedback --- .../framework/plugins/windows/orphan_kernel_threads.py | 8 +++++--- volatility3/framework/plugins/windows/thrdscan.py | 3 +-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/orphan_kernel_threads.py b/volatility3/framework/plugins/windows/orphan_kernel_threads.py index 1cdb1dcdf..f4901dc8c 100644 --- a/volatility3/framework/plugins/windows/orphan_kernel_threads.py +++ b/volatility3/framework/plugins/windows/orphan_kernel_threads.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # @@ -48,8 +48,9 @@ class Threads(thrdscan.ThrdScan): """Yields thread objects of kernel threads that do not map to a module Args: - kernel - + cls + context: the context to operate upon + module_name: name of the module to use for scanning Returns: A generator of thread objects of orphaned threads """ @@ -61,6 +62,7 @@ class Threads(thrdscan.ThrdScan): context, layer_name, symbol_table ) + # FIXME - use a proper constant once established # used to filter out smeared pointers if symbols.symbol_table_is_64bit(context, symbol_table): kernel_start = 0xFFFFF80000000000 diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index ad885e8e9..b812a15ff 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -48,8 +48,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) Args: context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - symbol_table: The name of the table containing the kernel symbols + module_name: Name of the module to use for scanning Returns: A list of _ETHREAD objects found by scanning memory for the "Thre" / "Thr\\xE5" pool signatures From 483cb7e4eebbb6b536cdec883caa3f75fa182ce2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 2 Sep 2024 15:08:29 -0500 Subject: [PATCH 363/979] Add smear checks in MFT parsing code --- .../framework/symbols/windows/extensions/mft.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 14c1f08d6..c51dd0348 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -27,6 +27,11 @@ class MFTAttribute(objects.StructType): """This represents an MFT ATTRIBUTE""" def get_resident_filename(self) -> str: + # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems + # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous + if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.NameLength > 512: + return None + # To get the resident name, we jump to relative name offset and read name length * 2 bytes of data try: name = self._context.object( @@ -42,6 +47,11 @@ class MFTAttribute(objects.StructType): return None def get_resident_filecontent(self) -> bytes: + # smear observed in mass testing of samples + # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems + if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.ContentLength > 4194304: + return None + # To get the resident content, we jump to relative content offset and read name length * 2 bytes of data try: bytesobj = self._context.object( From 660e8a7b89387bfc3e883a10a3662708a09fa146 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 2 Sep 2024 15:09:20 -0500 Subject: [PATCH 364/979] Add smear checks in MFT parsing code --- .../framework/symbols/windows/extensions/mft.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index c51dd0348..c0303eafb 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -29,7 +29,10 @@ class MFTAttribute(objects.StructType): def get_resident_filename(self) -> str: # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous - if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.NameLength > 512: + if ( + self.Attr_Header.ContentOffset > 4194304 + or self.Attr_Header.NameLength > 512 + ): return None # To get the resident name, we jump to relative name offset and read name length * 2 bytes of data @@ -49,7 +52,10 @@ class MFTAttribute(objects.StructType): def get_resident_filecontent(self) -> bytes: # smear observed in mass testing of samples # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems - if self.Attr_Header.ContentOffset > 4194304 or self.Attr_Header.ContentLength > 4194304: + if ( + self.Attr_Header.ContentOffset > 4194304 + or self.Attr_Header.ContentLength > 4194304 + ): return None # To get the resident content, we jump to relative content offset and read name length * 2 bytes of data From 9c058936a88893d0b5dd6a3ee68ab87b037bacf4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 3 Sep 2024 10:30:05 -0500 Subject: [PATCH 365/979] Address feedback --- .../framework/symbols/windows/extensions/mft.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index c0303eafb..ebba882c0 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,6 +2,8 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +from typing import Optional + from volatility3.framework import objects, constants, exceptions @@ -26,11 +28,11 @@ class MFTFileName(objects.StructType): class MFTAttribute(objects.StructType): """This represents an MFT ATTRIBUTE""" - def get_resident_filename(self) -> str: + def get_resident_filename(self) -> Optional[str]: # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems # Length as 512 as its 256*2, which is the maximum size for an entire file path, so this is even generous if ( - self.Attr_Header.ContentOffset > 4194304 + self.Attr_Header.ContentOffset > 0x400000 or self.Attr_Header.NameLength > 512 ): return None @@ -49,12 +51,12 @@ class MFTAttribute(objects.StructType): except exceptions.InvalidAddressException: return None - def get_resident_filecontent(self) -> bytes: + def get_resident_filecontent(self) -> Optional[bytes]: # smear observed in mass testing of samples # 4MB chosen as cutoff instead of 4KB to allow for recovery from format /L created file systems if ( - self.Attr_Header.ContentOffset > 4194304 - or self.Attr_Header.ContentLength > 4194304 + self.Attr_Header.ContentOffset > 0x400000 + or self.Attr_Header.ContentLength > 0x400000 ): return None From 69e6e59daf39ec7a0cf9a82fb13762d4389c95a4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:01:21 -0500 Subject: [PATCH 366/979] Add new pe_symbols API, debug registers plugin, unhooked system calls plugin --- .../plugins/windows/debugregisters.py | 223 ++++++ .../framework/plugins/windows/pe_symbols.py | 732 ++++++++++++++++++ .../plugins/windows/unhooked_system_calls.py | 183 +++++ .../framework/plugins/windows/vadinfo.py | 88 ++- 4 files changed, 1219 insertions(+), 7 deletions(-) create mode 100644 volatility3/framework/plugins/windows/debugregisters.py create mode 100644 volatility3/framework/plugins/windows/pe_symbols.py create mode 100644 volatility3/framework/plugins/windows/unhooked_system_calls.py diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py new file mode 100644 index 000000000..5f3c2e519 --- /dev/null +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -0,0 +1,223 @@ +import logging + +from typing import Tuple, Optional, Generator, List, Dict + +from functools import partial + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +import volatility3.plugins.windows.pslist as pslist +import volatility3.plugins.windows.threads as threads +import volatility3.plugins.windows.vadinfo as vadinfo +import volatility3.plugins.windows.pe_symbols as pe_symbols + +vollog = logging.getLogger(__name__) + + +class DebugRegisters(interfaces.plugins.PluginInterface): + # version 2.6.0 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 6, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) + ), + ] + + def _get_debug_info( + self, ethread: interfaces.objects.ObjectInterface + ) -> Optional[Tuple[interfaces.objects.ObjectInterface, int, int, int, int, int]]: + """ + Gathers information related to the debug registers for the given thread + """ + try: + dr7 = ethread.Tcb.TrapFrame.Dr7 + state = ethread.Tcb.State + except exceptions.InvalidAddressException: + return None + + # 0 = debug registers not active + # 4 = terminated + if dr7 == 0 or state == 4: + return None + + try: + owner_proc = ethread.owning_process() + except (AttributeError, exceptions.InvalidAddressException): + return None + + dr0 = ethread.Tcb.TrapFrame.Dr0 + dr1 = ethread.Tcb.TrapFrame.Dr1 + dr2 = ethread.Tcb.TrapFrame.Dr2 + dr3 = ethread.Tcb.TrapFrame.Dr3 + + # bail if all are 0 + if not (dr0 or dr1 or dr2 or dr3): + return None + + return owner_proc, dr7, dr0, dr1, dr2, dr3 + + def _get_vads( + self, + vads_cache: Dict[int, List[Tuple[int, int, str]]], + owner_proc: interfaces.objects.ObjectInterface, + ) -> Optional[List[Tuple[int, int, str]]]: + if owner_proc.vol.offset in vads_cache: + vads = vads_cache[owner_proc.vol.offset] + else: + vads = vadinfo.VadInfo.get_proc_vads_with_file_paths(owner_proc) + vads_cache[owner_proc.vol.offset] = vads + + # smear or terminated process + if len(vads) == 0: + return None + + return vads + + def _generator( + self, + ) -> Generator[ + Tuple[ + int, + Tuple[ + str, + int, + int, + int, + int, + format_hints.Hex, + str, + str, + format_hints.Hex, + str, + str, + format_hints.Hex, + str, + str, + format_hints.Hex, + str, + str, + ], + ], + None, + None, + ]: + kernel = self.context.modules[self.config["kernel"]] + + vads_cache: Dict[int, List[Tuple[int, int, str]]] = {} + + proc_modules = None + + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ) + + for proc in procs: + for thread in threads.Threads.list_threads(kernel, proc): + debug_info = self._get_debug_info(thread) + if not debug_info: + continue + + owner_proc, dr7, dr0, dr1, dr2, dr3 = debug_info + + vads = self._get_vads(vads_cache, owner_proc) + if not vads: + continue + + # this lookup takes a while, so only perform if we need to + if not proc_modules: + proc_modules = pe_symbols.PESymbols.get_process_modules( + self.context, kernel.layer_name, kernel.symbol_table_name, None + ) + path_and_symbol = partial( + pe_symbols.PESymbols.path_and_symbol_for_address, + self.context, + self.config_path, + proc_modules, + ) + + file0, sym0 = path_and_symbol(vads, dr0) + file1, sym1 = path_and_symbol(vads, dr1) + file2, sym2 = path_and_symbol(vads, dr2) + file3, sym3 = path_and_symbol(vads, dr3) + + # if none map to an actual file VAD then bail + if not ( + isinstance(file0, str) + or isinstance(file1, str) + or isinstance(file2, str) + or isinstance(file3, str) + ): + continue + + process_name = owner_proc.ImageFileName.cast( + "string", + max_length=owner_proc.ImageFileName.vol.count, + errors="replace", + ) + + thread_tid = thread.Cid.UniqueThread + + yield ( + 0, + ( + process_name, + owner_proc.UniqueProcessId, + thread_tid, + thread.Tcb.State, + dr7, + format_hints.Hex(dr0), + file0, + sym0, + format_hints.Hex(dr1), + file1, + sym1, + format_hints.Hex(dr2), + file2, + sym2, + format_hints.Hex(dr3), + file3, + sym3, + ), + ) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("TID", int), + ("State", int), + ("Dr7", int), + ("Dr0", format_hints.Hex), + ("Range0", str), + ("Symbol0", str), + ("Dr1", format_hints.Hex), + ("Range1", str), + ("Symbol1", str), + ("Dr2", format_hints.Hex), + ("Range2", str), + ("Symbol2", str), + ("Dr3", format_hints.Hex), + ("Range3", str), + ("Symbol3", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py new file mode 100644 index 000000000..b7faeaf55 --- /dev/null +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -0,0 +1,732 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + + +import io +import logging + +from typing import Dict, Tuple, Optional, List, Generator, Union + +import pefile + +from volatility3.framework import interfaces, exceptions +from volatility3.framework import renderers, constants +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.symbols.windows.extensions import pe +from volatility3.plugins.windows import pslist, vadinfo, modules + +vollog = logging.getLogger(__name__) + + +class PESymbolFinder: + """ + Interface for PE symbol finding classes + This interface provides a standard way for the calling code to + lookup symbols by name or address + """ + + cached_str = Union[str, None] + cached_str_dict = Dict[str, cached_str] + + cached_int = Union[int, None] + cached_int_dict = Dict[str, cached_int] + + cached_value = Union[int, str, None] + cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + + def __init__( + self, + layer_name: str, + mod_name: str, + module_start: int, + symbol_module: Union[interfaces.context.ModuleInterface, pefile.ExportDirData], + ): + self._layer_name = layer_name + self._mod_name = mod_name + self._module_start = module_start + self._symbol_module = symbol_module + + self._address_cache: PESymbolFinder.cached_int_dict = {} + self._name_cache: PESymbolFinder.cached_str_dict = {} + + def _get_cache_key(self, value: cached_value) -> str: + """ + Maintain a cache for symbol lookups to avoid re-walking of PDB symbols or export tables + within the same module for the same address in the same layer + """ + return f"{self._layer_name}|{self._mod_name}|{value}" + + def get_name_for_address(self, address: int) -> cached_str: + cached_key = self._get_cache_key(address) + if cached_key not in self._name_cache: + name = self._do_get_name(address) + self._name_cache[cached_key] = name + + return self._name_cache[cached_key] + + def get_address_for_name(self, name: str) -> cached_int: + cached_key = self._get_cache_key(name) + if cached_key not in self._address_cache: + address = self._do_get_address(name) + self._address_cache[cached_key] = address + + return self._address_cache[cached_key] + + def _do_get_name(self, address: int) -> cached_str: + raise NotImplementedError("_do_get_name must be overwritten") + + def _do_get_address(self, name: str) -> cached_int: + raise NotImplementedError("_do_get_address must be overwritten") + + +class PDBSymbolFinder(PESymbolFinder): + """ + PESymbolFinder implementation for PDB modules + """ + + def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + try: + return self._symbol_module.get_absolute_symbol_address(name) + except exceptions.SymbolError: + return None + + def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + try: + name = self._symbol_module.get_symbols_by_absolute_location(address)[0] + return name.split(constants.BANG)[1] + except (exceptions.SymbolError, IndexError): + return None + + +class ExportSymbolFinder(PESymbolFinder): + """ + PESymbolFinder implementation for PDB modules + """ + + def _get_name(self, export: pefile.ExportData) -> Optional[str]: + # AttributeError throws on empty or ordinal-only exports + try: + return export.name.decode("ascii") + except AttributeError: + return None + + def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + for export in self._symbol_module: + if export.address + self._module_start == address: + return self._get_name(export) + + return None + + def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + for export in self._symbol_module: + sym_name = self._get_name(export) + if sym_name and sym_name == name: + return self._module_start + export.address + + return None + + +class PESymbols(interfaces.plugins.PluginInterface): + """Prints symbols in PE files in process and kernel memory""" + + _required_framework_version = (2, 7, 0) + + _version = (1, 0, 0) + + # used for special handling of the kernel PDB file. See later notes + os_module_name = "ntoskrnl.exe" + + # keys for specifying wanted names and/or addresses + # used for consistent access between the API and plugins + wanted_names = "names" + wanted_addresses = "addresses" + + # how wanted modules/symbols are specified, such as: + # {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} + # {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} + filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + + # holds resolved symbols + # {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} + found_symbols_type = Dict[str, List[Tuple[str, int]]] + + # used to hold informatin about a range (VAD or kernel module) + # (start address, size, file path) + range_type = Tuple[int, int, str] + ranges_type = List[range_type] + + @classmethod + def get_requirements(cls) -> List: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="modules", component=modules.Modules, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pdbutil", component=pdbutil.PDBUtility, version=(1, 0, 0) + ), + requirements.ChoiceRequirement( + name="source", + description="Where to resolve symbols.", + choices=["kernel", "processes"], + optional=False, + ), + requirements.StringRequirement( + name="module", + description='Module in which to resolve symbols. Use "ntoskrnl.exe" to resolve in the base kernel executable.', + optional=False, + ), + requirements.StringRequirement( + name="symbol", + description="Symbol name to resolve", + optional=True, + ), + requirements.IntRequirement( + name="address", + description="Address of symbol to resolve", + optional=True, + ), + ] + + @staticmethod + def _get_pefile_obj( + context: interfaces.context.ContextInterface, + pe_table_name: str, + layer_name: str, + base_address: int, + ) -> Optional[pefile.PE]: + """ + Attempts to pefile object from the bytes of the PE file + + Args: + pe_table_name: name of the pe types table + layer_name: name of the process layer + base_address: base address of the module + + Returns: + the constructed pefile object + """ + pe_data = io.BytesIO() + + try: + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base_address, + layer_name=layer_name, + ) + + for offset, data in dos_header.reconstruct(): + pe_data.seek(offset) + pe_data.write(data) + + pe_ret = pefile.PE(data=pe_data.getvalue(), fast_load=True) + + except exceptions.InvalidAddressException: + pe_ret = None + + return pe_ret + + @staticmethod + def range_info_for_address( + ranges: ranges_type, address: int + ) -> Optional[range_type]: + """ + Helper for getting the range information for an address + """ + for start, size, filepath in ranges: + if start <= address < start + size: + return start, size, filepath + + return None + + @staticmethod + def filepath_for_address(ranges: ranges_type, address: int) -> Optional[str]: + """ + Helper to get the file path for an address + """ + info = PESymbols.range_info_for_address(ranges, address) + if info: + return info[2] + + return None + + @staticmethod + def filename_for_path(filepath: str) -> str: + """ + Consistent way to get the filename + """ + return filepath.split("\\")[-1] + + @staticmethod + def addresses_for_process_symbols( + context: interfaces.context.ContextInterface, + config_path: str, + layer_name: str, + symbol_table_name: str, + symbols: filter_modules_type, + ) -> found_symbols_type: + collected_modules = PESymbols.get_process_modules( + context, layer_name, symbol_table_name, symbols + ) + + found_symbols = PESymbols.find_symbols( + context, config_path, symbols, collected_modules + ) + + for mod_name, unresolved_symbols in symbols.items(): + for symbol in unresolved_symbols: + vollog.debug(f"Unable to resolve symbol {symbol} in module {mod_name}") + + return found_symbols + + @staticmethod + def path_and_symbol_for_address( + context: interfaces.context.ContextInterface, + config_path: str, + collected_modules: Dict[str, List[Tuple[str, int, int]]], + ranges: ranges_type, + address: int, + ) -> Tuple[str, str]: + """ + Method for plugins to determine the file path and symbol name for a given address + + collected_modules: return value from `get_kernel_modules` or `get_process_modules` + ranges: the memory ranges to examine in this layer. + address: address to resolve to its symbol name + """ + + if not address: + return renderers.NotApplicableValue(), renderers.NotApplicableValue() + + filepath = PESymbols.filepath_for_address(ranges, address) + + if not filepath: + return renderers.NotAvailableValue(), renderers.NotAvailableValue() + + filename = PESymbols.filename_for_path(filepath).lower() + + # setup to resolve the address + filter_module: PESymbols.filter_modules_type = { + filename: {PESymbols.wanted_addresses: [address]} + } + + found_symbols = PESymbols.find_symbols( + context, config_path, filter_module, collected_modules + ) + + if not found_symbols or not found_symbols[filename]: + return renderers.NotAvailableValue(), renderers.NotAvailableValue() + + return filepath, found_symbols[filename][0][0] + + @staticmethod + def _get_exported_symbols( + context: interfaces.context.ContextInterface, + pe_table_name: str, + mod_name: str, + module_info: Tuple[str, int, int], + ) -> Optional[ExportSymbolFinder]: + """ + Attempts to locate symbols based on export analysis + + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + """ + + layer_name = module_info[0] + module_start = module_info[1] + + # we need a valid PE with an export table + pe_module = PESymbols._get_pefile_obj( + context, pe_table_name, layer_name, module_start + ) + if not pe_module: + return None + + pe_module.parse_data_directories( + directories=[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_EXPORT"]] + ) + if not hasattr(pe_module, "DIRECTORY_ENTRY_EXPORT"): + return None + + return ExportSymbolFinder( + layer_name, mod_name, module_start, pe_module.DIRECTORY_ENTRY_EXPORT.symbols + ) + + @staticmethod + def _get_pdb_module( + context: interfaces.context.ContextInterface, + config_path: str, + mod_name: str, + module_info: Tuple[str, int, int], + ) -> Optional[PDBSymbolFinder]: + """ + Attempts to locate symbols based on PDB analysis + + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + """ + + mod_symbols = None + + layer_name, module_start, module_size = module_info + + # the PDB name of the kernel file is not consistent for an exe, for example, + # a `ntoskrnl.exe` can have an internal PDB name of any of the ones in the following list + # The code attempts to find all possible PDBs to ensure the best chance of recovery + if mod_name == PESymbols.os_module_name: + pdb_names = ["ntkrnlmp.pdb", "ntkrnlpa.pdb", "ntkrpamp.pdb", "ntoskrnl.pdb"] + + # for non-kernel files, replace the exe, sys, or dll extension with pdb + else: + mod_name = mod_name[:-3] + "pdb" + first_upper = mod_name[0].upper() + mod_name[1:] + pdb_names = [mod_name, first_upper] + + # loop through each PDB name (will be just one for all but the kernel) + for pdb_name in pdb_names: + try: + mod_symbols = pdbutil.PDBUtility.symbol_table_from_pdb( + context, + interfaces.configuration.path_join(config_path, mod_name), + layer_name, + pdb_name, + module_start, + module_size, + ) + + if mod_symbols: + break + + # this exception is expected when the PDB can't be found or downloaded + except exceptions.VolatilityException: + continue + + # this is not expected - it means pdbconv broke when parsing the PDB + except TypeError as e: + vollog.error( + f"Unable to parse PDB file for module {pdb_name} -> {e}. Please file a bug on the GitHub issue tracker." + ) + + # cannot do anything without the symbols + if not mod_symbols: + return None + + pdb_module = context.module( + mod_symbols, layer_name=layer_name, offset=module_start + ) + + return PDBSymbolFinder(layer_name, mod_name, module_start, pdb_module) + + @staticmethod + def _find_symbols_through_pdb( + context: interfaces.context.ContextInterface, + config_path: str, + module_instances: List[Tuple[str, int, int]], + mod_name: str, + ) -> Generator[PDBSymbolFinder, None, None]: + """ + Attempts to resolve the symbols in `wanted_symbols` through PDB analysis + """ + for module_info in module_instances: + mod_module = PESymbols._get_pdb_module( + context, config_path, mod_name, module_info + ) + if mod_module: + yield mod_module + + @staticmethod + def _find_symbols_through_exports( + context: interfaces.context.ContextInterface, + config_path: str, + module_instances: List[Tuple[str, int, int]], + mod_name: str, + ) -> Generator[ExportSymbolFinder, None, None]: + """ + Attempts to resolve the symbols in `wanted_symbols` through export analysis + """ + pe_table_name = intermed.IntermediateSymbolTable.create( + context, config_path, "windows", "pe", class_types=pe.class_types + ) + + # for each process layer and VAD, construct a PE and examine the export table + for module_info in module_instances: + exported_symbols = PESymbols._get_exported_symbols( + context, pe_table_name, mod_name, module_info + ) + if exported_symbols: + yield exported_symbols + + @staticmethod + def _get_symbol_value( + wanted_modules: PESymbolFinder.cached_value_dict, + mod_name: str, + symbol_resolver: PESymbolFinder, + ) -> Generator[Tuple[str, int], None, None]: + """ + Enumerates the symbols specified as wanted by the calling plugin + """ + wanted_symbols = wanted_modules[mod_name] + + if ( + PESymbols.wanted_names not in wanted_symbols + and PESymbols.wanted_addresses not in wanted_symbols + ): + vollog.warning( + f"Invalid `wanted_symbols` sent to `find_symbols` for module {mod_name}. addresses and names keys both misssing." + ) + return + + symbol_keys = [ + (PESymbols.wanted_names, "get_address_for_name"), + (PESymbols.wanted_addresses, "get_name_for_address"), + ] + + for symbol_key, symbol_getter in symbol_keys: + # address or name + if symbol_key in wanted_symbols: + # walk each wanted address or name + for wanted_value in wanted_symbols[symbol_key]: + symbol_value = symbol_resolver.__getattribute__(symbol_getter)( + wanted_value + ) + if symbol_value: + # yield out symbol name, symbol address + if symbol_key == PESymbols.wanted_names: + yield wanted_value, symbol_value # type: ignore + else: + yield symbol_value, wanted_value # type: ignore + + index = wanted_modules[mod_name][symbol_key].index(wanted_value) # type: ignore + + del wanted_modules[mod_name][symbol_key][index] + + # if all names or addresses from a module are found, delete the key + if not wanted_modules[mod_name][symbol_key]: + del wanted_modules[mod_name][symbol_key] + break + + @staticmethod + def _resolve_symbols_through_methods( + context: interfaces.context.ContextInterface, + config_path: str, + module_instances: List[Tuple[str, int, int]], + wanted_modules: PESymbolFinder.cached_value_dict, + mod_name: str, + ) -> Generator[Tuple[str, int], None, None]: + """ + Attempts to resolve every wanted symbol in `mod_name` + Every layer is enumerated for maximum chance of recovery + """ + symbol_resolving_methods = [ + PESymbols._find_symbols_through_pdb, + PESymbols._find_symbols_through_exports, + ] + + for method in symbol_resolving_methods: + for symbol_resolver in method( + context, config_path, module_instances, mod_name + ): + vollog.debug(f"Have resolver for method {method}") + yield from PESymbols._get_symbol_value( + wanted_modules, mod_name, symbol_resolver + ) + + if not wanted_modules[mod_name]: + break + + if not wanted_modules[mod_name]: + break + + @staticmethod + def find_symbols( + context: interfaces.context.ContextInterface, + config_path: str, + wanted_modules: PESymbolFinder.cached_value_dict, + collected_modules: Dict[str, List[Tuple[str, int, int]]], + ) -> found_symbols_type: + """ + Loops through each method of symbol analysis until each wanted symbol is found + Returns the resolved symbols as a dictionary that includes the name and runtime address + """ + found_symbols: PESymbols.found_symbols_type = {} + + for mod_name in wanted_modules: + if mod_name not in collected_modules: + continue + + module_instances = collected_modules[mod_name] + + # try to resolve the symbols for `mod_name` through each method (PDB and export table currently) + for symbol_name, address in PESymbols._resolve_symbols_through_methods( + context, config_path, module_instances, wanted_modules, mod_name + ): + if mod_name not in found_symbols: + found_symbols[mod_name] = [] + + found_symbols[mod_name].append((symbol_name, address)) + + # stop processing the layers (processes) if we found all the symbols for this module + if not wanted_modules[mod_name]: + break + + # stop processing this module if/when all symbols are found + if not wanted_modules[mod_name]: + del wanted_modules[mod_name] + break + + return found_symbols + + @staticmethod + def get_kernel_modules( + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + filter_modules: Optional[filter_modules_type], + ) -> Dict[str, List[Tuple[str, int, int]]]: + """ + Walks the kernel module list and finds the session layer, base, and size of each wanted module + """ + found_modules: Dict[str, List[Tuple[str, int, int]]] = {} + + if filter_modules: + # create a tuple of module names for use with `endswith` + filter_modules_check = tuple([key.lower() for key in filter_modules.keys()]) + else: + filter_modules_check = None + + session_layers = list( + modules.Modules.get_session_layers(context, layer_name, symbol_table) + ) + + # special handling for the kernel + gather_kernel = ( + filter_modules_check and PESymbols.os_module_name in filter_modules_check + ) + + for index, mod in enumerate( + modules.Modules.list_modules(context, layer_name, symbol_table) + ): + try: + mod_name = str(mod.BaseDllName.get_string().lower()) + except exceptions.InvalidAddressException: + continue + + # to analyze, it must either be the kernel or a wanted module + if not filter_modules_check or (gather_kernel and index == 0): + mod_name = PESymbols.os_module_name + elif filter_modules_check and not mod_name.endswith(filter_modules_check): + continue + + # we won't find symbol information if we can't analyze the module + session_layer_name = modules.Modules.find_session_layer( + context, session_layers, mod.DllBase + ) + if not session_layer_name: + continue + + if mod_name not in found_modules: + found_modules[mod_name] = [] + + found_modules[mod_name].append( + (session_layer_name, mod.DllBase, mod.SizeOfImage) + ) + + return found_modules + + @staticmethod + def get_process_modules( + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + filter_modules: Optional[filter_modules_type], + ) -> Dict[str, List[Tuple[str, int, int]]]: + """ + Walks the process list and each process' VAD to determine the base address and size of wanted modules + """ + proc_modules: Dict[str, List[Tuple[str, int, int]]] = {} + + if filter_modules: + # create a tuple of module names for use with `endswith` + filter_modules_check = tuple([key.lower() for key in filter_modules.keys()]) + else: + filter_modules_check = None + + for _, proc_layer_name, vads in vadinfo.VadInfo.get_all_vads_with_file_paths( + context, layer_name, symbol_table + ): + for vad_start, vad_size, filepath in vads: + filename = PESymbols.filename_for_path(filepath) + + if filter_modules_check and not filename.endswith(filter_modules_check): + continue + + # track each module along with the process layer and range to find it + if filename not in proc_modules: + proc_modules[filename] = [] + + proc_modules[filename].append((proc_layer_name, vad_start, vad_size)) + + return proc_modules + + def _generator(self) -> Generator[Tuple[int, Tuple[str, str, int]], None, None]: + kernel = self.context.modules[self.config["kernel"]] + + if self.config["symbol"]: + filter_module = { + self.config["module"].lower(): { + PESymbols.wanted_names: [self.config["symbol"]] + } + } + + elif self.config["address"]: + filter_module = { + self.config["module"].lower(): { + PESymbols.wanted_addresses: [self.config["address"]] + } + } + + else: + vollog.error("--address or --symbol must be specified") + return + + if self.config["source"] == "kernel": + module_resolver = self.get_kernel_modules + else: + module_resolver = self.get_process_modules + + collected_modules = module_resolver( + self.context, kernel.layer_name, kernel.symbol_table_name, filter_module + ) + + found_symbols = PESymbols.find_symbols( + self.context, self.config_path, filter_module, collected_modules + ) + + for module, symbols in found_symbols.items(): + for symbol, address in symbols: + yield (0, (module, symbol, format_hints.Hex(address))) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Module", str), + ("Symbol", str), + ("Address", format_hints.Hex), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py new file mode 100644 index 000000000..0438bc9e3 --- /dev/null +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -0,0 +1,183 @@ +import logging + +from typing import Dict, Tuple, List, Generator + +from volatility3.framework import interfaces, exceptions +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility +from volatility3.plugins.windows import pslist, pe_symbols + +vollog = logging.getLogger(__name__) + + +class unhooked_system_calls(interfaces.plugins.PluginInterface): + """Looks for signs of Skeleton Key malware""" + + _required_framework_version = (2, 4, 0) + + system_calls = { + "ntdll.dll": { + pe_symbols.PESymbols.wanted_names: [ + "NtCreateThread", + "NtProtectVirtualMemory", + "NtReadVirtualMemory", + "NtOpenProcess", + "NtWriteFile", + "NtQueryVirtualMemory", + "NtAllocateVirtualMemory", + "NtWorkerFactoryWorkerReady", + "NtAcceptConnectPort", + "NtAddDriverEntry", + "NtAdjustPrivilegesToken", + "NtAlpcCreatePort", + "NtClose", + "NtCreateFile", + "NtCreateMutant", + "NtOpenFile", + "NtOpenIoCompletion", + "NtOpenJobObject", + "NtOpenKey", + "NtOpenKeyEx", + "NtOpenThread", + "NtOpenThreadToken", + "NtOpenThreadTokenEx", + "NtWriteVirtualMemory", + "NtTraceEvent", + "NtTranslateFilePath", + "NtUmsThreadYield", + "NtUnloadDriver", + "NtUnloadKey", + "NtUnloadKey2", + "NtUnloadKeyEx", + "NtCreateKey", + "NtCreateSection", + "NtDeleteKey", + "NtDeleteValueKey", + "NtDuplicateObject", + "NtQueryValueKey", + "NtReplaceKey", + "NtRequestWaitReplyPort", + "NtRestoreKey", + "NtSetContextThread", + "NtSetSecurityObject", + "NtSetValueKey", + "NtSystemDebugControl", + "NtTerminateProcess", + ] + } + } + + _code_bytes_type = Dict[str, Dict[str, Dict[bytes, List[Tuple[int, str]]]]] + + @classmethod + def get_requirements(cls) -> List: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="pe_symbols", plugin=pe_symbols.PESymbols, version=(1, 0, 0) + ), + ] + + def _gather_code_bytes( + self, + kernel: interfaces.context.ModuleInterface, + found_symbols: pe_symbols.PESymbols.found_symbols_type, + ) -> _code_bytes_type: + """ + Enumerates the desired DLLs and function implementations in each process + Groups based on unique implementations of each DLLs' functions + The purpose is to detect when a function has different implementations (code) + in different processes. + This very effectively detects code injection. + """ + code_bytes: unhooked_system_calls._code_bytes_type = {} + + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ) + + for proc in procs: + try: + proc_id = proc.UniqueProcessId + proc_name = utility.array_to_string(proc.ImageFileName) + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + for dll_name, functions in found_symbols.items(): + for func_name, func_addr in functions: + try: + fbytes = self.context.layers[proc_layer_name].read( + func_addr, 0x20 + ) + except exceptions.InvalidAddressException: + continue + + if dll_name not in code_bytes: + code_bytes[dll_name] = {} + + if func_name not in code_bytes[dll_name]: + code_bytes[dll_name][func_name] = {} + + if fbytes not in code_bytes[dll_name][func_name]: + code_bytes[dll_name][func_name][fbytes] = [] + + code_bytes[dll_name][func_name][fbytes].append((proc_id, proc_name)) + + return code_bytes + + def _generator(self) -> Generator[Tuple[int, Tuple[str, str, int]], None, None]: + kernel = self.context.modules[self.config["kernel"]] + + found_symbols = pe_symbols.PESymbols.addresses_for_process_symbols( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + unhooked_system_calls.system_calls, + ) + + # code_bytes[dll_name][func_name][func_bytes] + code_bytes = self._gather_code_bytes(kernel, found_symbols) + + for functions in code_bytes.values(): + for func_name, cbb in functions.items(): + cb = list(cbb.values()) + + # same implementation in all + if len(cb) == 1: + yield 0, (func_name, "", len(cb[0])) + else: + # find the processes that are hooked for reporting + max_idx = 0 if len(cb[0]) > len(cb[1]) else 1 + small_idx = (~max_idx) & 1 + + ps = [] + + for pid, pname in cb[small_idx]: + ps.append("{:d}:{}".format(pid, pname)) + + proc_names = ", ".join(ps) + + yield 0, (func_name, proc_names, len(cb[max_idx])) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Function", str), + ("Distinct Implementations", str), + ("Total Implementations", int), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index abc6142fe..97a2f2455 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -3,13 +3,13 @@ # import logging -from typing import Callable, List, Generator, Iterable, Type, Optional +from typing import Callable, List, Generator, Iterable, Type, Optional, Tuple -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import renderers, interfaces, exceptions, symbols from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist +from volatility3.plugins.windows import pslist, pe_symbols vollog = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class VadInfo(interfaces.plugins.PluginInterface): _version = (2, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): # type: ignore super().__init__(*args, **kwargs) self._protect_values = None @@ -107,6 +107,58 @@ class VadInfo(interfaces.plugins.PluginInterface): ) return values # type: ignore + @staticmethod + def get_proc_vads_with_file_paths( + proc: interfaces.objects.ObjectInterface, + ) -> pe_symbols.PESymbols.ranges_type: + """ + Returns a list of the process' vads that map a file + """ + vads = [] + + for vad in proc.get_vad_root().traverse(): + filepath = vad.get_file_name() + if not isinstance(filepath, str) or filepath.count("\\") == 0: + continue + + vads.append((vad.get_start(), vad.get_size(), filepath)) + + return vads + + @classmethod + def get_all_vads_with_file_paths( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table_name: str, + ) -> Generator[ + Tuple[ + interfaces.objects.ObjectInterface, str, pe_symbols.PESymbols.ranges_type + ], + None, + None, + ]: + """ + Yields each set of vads for a process that have a file mapped, along with the process itself and its layer + """ + is_32bit_arch = not symbols.symbol_table_is_64bit(context, symbol_table_name) + + procs = pslist.PsList.list_processes( + context=context, + layer_name=layer_name, + symbol_table=symbol_table_name, + ) + + for proc in procs: + try: + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + vads = cls.get_proc_vads_with_file_paths(proc) + + yield proc, proc_layer_name, vads + @classmethod def list_vads( cls, @@ -196,11 +248,33 @@ class VadInfo(interfaces.plugins.PluginInterface): return file_handle - def _generator(self, procs): + def _generator( + self, procs: List[interfaces.objects.ObjectInterface] + ) -> Generator[ + Tuple[ + int, + Tuple[ + int, + str, + format_hints.Hex, + format_hints.Hex, + format_hints.Hex, + str, + str, + int, + int, + format_hints.Hex, + str, + str, + ], + ], + None, + None, + ]: kernel = self.context.modules[self.config["kernel"]] kernel_layer = self.context.layers[kernel.layer_name] - def passthrough(_: interfaces.objects.ObjectInterface) -> bool: + def passthrough(x: interfaces.objects.ObjectInterface) -> bool: return False filter_func = passthrough @@ -250,7 +324,7 @@ class VadInfo(interfaces.plugins.PluginInterface): ), ) - def run(self): + def run(self) -> renderers.TreeGrid: kernel = self.context.modules[self.config["kernel"]] filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) From 3bb9264a5770d6828487129b6ebfec509ade8680 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:11:16 -0500 Subject: [PATCH 367/979] formatting --- volatility3/framework/plugins/windows/debugregisters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 5f3c2e519..c70f922ae 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -37,7 +37,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) ), - ] + ] def _get_debug_info( self, ethread: interfaces.objects.ObjectInterface From 30cb5bd3ab4bac9f3ebb224d2f37cb98d6960545 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:14:06 -0500 Subject: [PATCH 368/979] Formatting that my local black doesn't understand --- volatility3/framework/plugins/windows/vadinfo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 97a2f2455..b3b7bf5fb 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -249,8 +249,7 @@ class VadInfo(interfaces.plugins.PluginInterface): return file_handle def _generator( - self, procs: List[interfaces.objects.ObjectInterface] - ) -> Generator[ + self, procs: List[interfaces.objects.ObjectInterface]) -> Generator[ Tuple[ int, Tuple[ From c223ac6e762d1072f04c21276645d7c2dda79dde Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:31:55 -0500 Subject: [PATCH 369/979] more black help --- volatility3/framework/plugins/windows/vadinfo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index b3b7bf5fb..655e54be6 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -141,8 +141,6 @@ class VadInfo(interfaces.plugins.PluginInterface): """ Yields each set of vads for a process that have a file mapped, along with the process itself and its layer """ - is_32bit_arch = not symbols.symbol_table_is_64bit(context, symbol_table_name) - procs = pslist.PsList.list_processes( context=context, layer_name=layer_name, From 7b48ee4be489b4667c57cca55da84abd3c1c3d12 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:36:13 -0500 Subject: [PATCH 370/979] more black help --- volatility3/framework/plugins/windows/vadinfo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 655e54be6..594c0d67c 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -246,8 +246,7 @@ class VadInfo(interfaces.plugins.PluginInterface): return file_handle - def _generator( - self, procs: List[interfaces.objects.ObjectInterface]) -> Generator[ + def _generator(self, procs: List[interfaces.objects.ObjectInterface]) -> Generator[ Tuple[ int, Tuple[ From bcd93616c0628fdb991866290845bbb861889f8f Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:36:35 -0500 Subject: [PATCH 371/979] more black help --- volatility3/framework/plugins/windows/vadinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 594c0d67c..e129ad4ef 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -5,7 +5,7 @@ import logging from typing import Callable, List, Generator, Iterable, Type, Optional, Tuple -from volatility3.framework import renderers, interfaces, exceptions, symbols +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints From fd8777287736baee75e6f1a8843b471a47b6ced2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:47:42 -0500 Subject: [PATCH 372/979] Move VAD enumeration into pe_symbols --- .../framework/plugins/windows/pe_symbols.py | 57 +++++++++++++++++-- .../framework/plugins/windows/vadinfo.py | 50 ---------------- 2 files changed, 52 insertions(+), 55 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index b7faeaf55..51558f71b 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -16,7 +16,7 @@ from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbutil from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import pslist, vadinfo, modules +from volatility3.plugins.windows import pslist, modules vollog = logging.getLogger(__name__) @@ -170,9 +170,6 @@ class PESymbols(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) - ), requirements.VersionRequirement( name="modules", component=modules.Modules, version=(2, 0, 0) ), @@ -648,6 +645,56 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_modules + @staticmethod + def get_proc_vads_with_file_paths( + proc: interfaces.objects.ObjectInterface, + ) -> ranges_type: + """ + Returns a list of the process' vads that map a file + """ + vads = [] + + for vad in proc.get_vad_root().traverse(): + filepath = vad.get_file_name() + if not isinstance(filepath, str) or filepath.count("\\") == 0: + continue + + vads.append((vad.get_start(), vad.get_size(), filepath)) + + return vads + + @classmethod + def get_all_vads_with_file_paths( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table_name: str, + ) -> Generator[ + Tuple[ + interfaces.objects.ObjectInterface, str, ranges_type + ], + None, + None, + ]: + """ + Yields each set of vads for a process that have a file mapped, along with the process itself and its layer + """ + procs = pslist.PsList.list_processes( + context=context, + layer_name=layer_name, + symbol_table=symbol_table_name, + ) + + for proc in procs: + try: + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + vads = PESymbols.get_proc_vads_with_file_paths(proc) + + yield proc, proc_layer_name, vads + @staticmethod def get_process_modules( context: interfaces.context.ContextInterface, @@ -666,7 +713,7 @@ class PESymbols(interfaces.plugins.PluginInterface): else: filter_modules_check = None - for _, proc_layer_name, vads in vadinfo.VadInfo.get_all_vads_with_file_paths( + for _, proc_layer_name, vads in PESymbols.get_all_vads_with_file_paths( context, layer_name, symbol_table ): for vad_start, vad_size, filepath in vads: diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index e129ad4ef..974793a71 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -107,56 +107,6 @@ class VadInfo(interfaces.plugins.PluginInterface): ) return values # type: ignore - @staticmethod - def get_proc_vads_with_file_paths( - proc: interfaces.objects.ObjectInterface, - ) -> pe_symbols.PESymbols.ranges_type: - """ - Returns a list of the process' vads that map a file - """ - vads = [] - - for vad in proc.get_vad_root().traverse(): - filepath = vad.get_file_name() - if not isinstance(filepath, str) or filepath.count("\\") == 0: - continue - - vads.append((vad.get_start(), vad.get_size(), filepath)) - - return vads - - @classmethod - def get_all_vads_with_file_paths( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - symbol_table_name: str, - ) -> Generator[ - Tuple[ - interfaces.objects.ObjectInterface, str, pe_symbols.PESymbols.ranges_type - ], - None, - None, - ]: - """ - Yields each set of vads for a process that have a file mapped, along with the process itself and its layer - """ - procs = pslist.PsList.list_processes( - context=context, - layer_name=layer_name, - symbol_table=symbol_table_name, - ) - - for proc in procs: - try: - proc_layer_name = proc.add_process_layer() - except exceptions.InvalidAddressException: - continue - - vads = cls.get_proc_vads_with_file_paths(proc) - - yield proc, proc_layer_name, vads - @classmethod def list_vads( cls, From 61cf58d97794359e3e093a97f3b8d2562661aea4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 12:49:01 -0500 Subject: [PATCH 373/979] black fixes --- volatility3/framework/plugins/windows/pe_symbols.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 51558f71b..10e87c552 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -670,9 +670,7 @@ class PESymbols(interfaces.plugins.PluginInterface): layer_name: str, symbol_table_name: str, ) -> Generator[ - Tuple[ - interfaces.objects.ObjectInterface, str, ranges_type - ], + Tuple[interfaces.objects.ObjectInterface, str, ranges_type], None, None, ]: From 802fc024b5a7666f1e56a641af4fe219c856f6b4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 6 Sep 2024 13:27:04 -0500 Subject: [PATCH 374/979] switch api place --- volatility3/framework/plugins/windows/debugregisters.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index c70f922ae..931ccef37 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -9,7 +9,6 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints import volatility3.plugins.windows.pslist as pslist import volatility3.plugins.windows.threads as threads -import volatility3.plugins.windows.vadinfo as vadinfo import volatility3.plugins.windows.pe_symbols as pe_symbols vollog = logging.getLogger(__name__) @@ -31,9 +30,6 @@ class DebugRegisters(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) - ), requirements.VersionRequirement( name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) ), @@ -80,7 +76,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): if owner_proc.vol.offset in vads_cache: vads = vads_cache[owner_proc.vol.offset] else: - vads = vadinfo.VadInfo.get_proc_vads_with_file_paths(owner_proc) + vads = pe_symbols.PESymbols.get_proc_vads_with_file_paths(owner_proc) vads_cache[owner_proc.vol.offset] = vads # smear or terminated process From 26cd73115b58cf5b1adfb430c9f1070e45dbe727 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 13:31:09 +0100 Subject: [PATCH 375/979] CLI: Filter on rendered values --- volatility3/cli/text_renderer.py | 38 +++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ab9e44141..1f9cfa9d8 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -179,7 +179,15 @@ class QuickTextRenderer(CLIRenderer): outfd.write("\n{}\n".format("\t".join(line))) def visitor(node: interfaces.renderers.TreeNode, accumulator): - if self.filter and self.filter.filter(node.values): + line = [] + for column_index in range(len(grid.columns)): + column = grid.columns[column_index] + renderer = self._type_renderers.get( + column.type, self._type_renderers["default"] + ) + line.append(renderer(node.values[column_index])) + + if self.filter and self.filter.filter(line): return accumulator accumulator.write("\n") @@ -188,13 +196,6 @@ class QuickTextRenderer(CLIRenderer): "*" * max(0, node.path_depth - 1) + ("" if (node.path_depth <= 1) else " ") ) - line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] - renderer = self._type_renderers.get( - column.type, self._type_renderers["default"] - ) - line.append(renderer(node.values[column_index])) accumulator.write("{}".format("\t".join(line))) accumulator.flush() return accumulator @@ -259,12 +260,18 @@ class CSVRenderer(CLIRenderer): def visitor(node: interfaces.renderers.TreeNode, accumulator): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case row = {"TreeDepth": str(max(0, node.path_depth - 1))} + line = [] for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) row[f"{column.name}"] = renderer(node.values[column_index]) + line.append(row[f"{column.name}"]) + + if self.filter and self.filter.filter(line): + return accumulator + accumulator.writerow(row) return accumulator @@ -317,10 +324,8 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths.get(tree_indent_column, 0), node.path_depth ) - if self.filter and self.filter.filter(node.values): - return accumulator - line = {} + rendered_line = [] for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get( @@ -334,6 +339,11 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths.get(column.name, len(column.name)), field_width ) line[column] = data.split("\n") + rendered_line.append(data) + + if self.filter and self.filter.filter(rendered_line): + return accumulator + accumulator.append((node.path_depth, line)) return accumulator @@ -437,6 +447,7 @@ class JsonRenderer(CLIRenderer): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case acc_map, final_tree = accumulator node_dict: Dict[str, Any] = {"__children": []} + line = [] for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get( @@ -446,6 +457,11 @@ class JsonRenderer(CLIRenderer): if isinstance(data, interfaces.renderers.BaseAbsentValue): data = None node_dict[column.name] = data + line.append(data) + + if self.filter and self.filter.filter(line): + return accumulator + if node.parent: acc_map[node.parent.path]["__children"].append(node_dict) else: From 8259ca90199f548e1592aa3ab05444c9353fe66a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 13:47:26 +0100 Subject: [PATCH 376/979] Core: Bump the framework number so we can differentiate CLI versions --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index d30446d63..a131609c9 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 9 # Number of changes that only add to the interface +VERSION_MINOR = 10 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 4361c3857393b7f7311e9b600d1ea5d4966cdb93 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 14:55:04 +0100 Subject: [PATCH 377/979] Core: Verify plugin requirements of plugins --- .../framework/configuration/requirements.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 1c0622574..931995bb8 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -585,6 +585,24 @@ class PluginRequirement(VersionRequirement): version=version, ) + def unsatisfied( + self, context: interfaces.context.ContextInterface, config_path: str + ) -> Dict[str, interfaces.configuration.RequirementInterface]: + result = super().unsatisfied(context, config_path) + if not result: + component: Type[interfaces.plugins.PluginInterface] = self._component + for requirement in component.get_requirements(): + if isinstance(requirement, PluginRequirement): + result.update( + requirement.unsatisfied( + context, + interfaces.configuration.path_join(config_path, self.name), + ) + ) + if result: + result[config_path] = self + return result + class ModuleRequirement( interfaces.configuration.ConstructableRequirementInterface, From f3085b6c59b05190320722feef58f276357d2155 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 16:06:23 +0100 Subject: [PATCH 378/979] Core: Move the pluginrequirement check to generic versionrequirement --- .../framework/configuration/requirements.py | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 931995bb8..f95dac307 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -527,12 +527,14 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, optional: bool = False, component: Type[interfaces.configuration.VersionableInterface] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: + if description is None: + description = f"Version {".".join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) @@ -550,9 +552,29 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} + + # Check for child requirements + if issubclass(self._component, interfaces.configuration.ConfigurableInterface): + result = {} + for requirement in self._component.get_requirements(): + if not requirement.optional and isinstance( + requirement, VersionRequirement + ): + result.update( + requirement.unsatisfied( + context, + config_path, + ) + ) + + if result: + result.update({config_path: self}) + return result + context.config[interfaces.configuration.path_join(config_path, self.name)] = ( True ) + return {} @classmethod @@ -585,24 +607,6 @@ class PluginRequirement(VersionRequirement): version=version, ) - def unsatisfied( - self, context: interfaces.context.ContextInterface, config_path: str - ) -> Dict[str, interfaces.configuration.RequirementInterface]: - result = super().unsatisfied(context, config_path) - if not result: - component: Type[interfaces.plugins.PluginInterface] = self._component - for requirement in component.get_requirements(): - if isinstance(requirement, PluginRequirement): - result.update( - requirement.unsatisfied( - context, - interfaces.configuration.path_join(config_path, self.name), - ) - ) - if result: - result[config_path] = self - return result - class ModuleRequirement( interfaces.configuration.ConstructableRequirementInterface, From ace590e8f0669797b5770773eeadcbcb27752a78 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 16:08:48 +0100 Subject: [PATCH 379/979] Core: Fix up f-string containing a string --- volatility3/framework/configuration/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index f95dac307..3a862a132 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -534,7 +534,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): version: Optional[Tuple[int, ...]] = None, ) -> None: if description is None: - description = f"Version {".".join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" + description = f"Version {'.'.join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) From bf000ff0a0bafc0197ee1dd074f1df450362d02f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 21:42:59 +0100 Subject: [PATCH 380/979] Core: Add recursion protection to VersionRequirement check --- .../framework/configuration/requirements.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 3a862a132..49ca49b59 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -546,13 +546,26 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): self._version = version def unsatisfied( - self, context: interfaces.context.ContextInterface, config_path: str + self, + context: interfaces.context.ContextInterface, + config_path: str, + accumulator: Optional[ + List[interfaces.configuration.VersionableInterface] + ] = None, ) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} + if accumulator is None: + accumulator = set([self._component]) + else: + if self._component in accumulator: + return {config_path: self} + else: + accumulator.add(self._component) + # Check for child requirements if issubclass(self._component, interfaces.configuration.ConfigurableInterface): result = {} @@ -562,8 +575,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): ): result.update( requirement.unsatisfied( - context, - config_path, + context, config_path, accumulator.copy() ) ) From 9152f33181fd347366468d00c47a2c22ae71cfc2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 8 Sep 2024 21:58:09 +0100 Subject: [PATCH 381/979] Core: Allow circular dependencies as long as they are all met --- volatility3/framework/configuration/requirements.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 49ca49b59..dcb7505ea 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -558,16 +558,20 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): if not self.matches_required(self._version, self._component.version): return {config_path: self} + recurse = True if accumulator is None: accumulator = set([self._component]) else: if self._component in accumulator: - return {config_path: self} + recurse = False else: accumulator.add(self._component) # Check for child requirements - if issubclass(self._component, interfaces.configuration.ConfigurableInterface): + if ( + issubclass(self._component, interfaces.configuration.ConfigurableInterface) + and recurse + ): result = {} for requirement in self._component.get_requirements(): if not requirement.optional and isinstance( From fd2c97e1bafb5b30403a1788eae4c32723438746 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 9 Sep 2024 00:44:00 +0100 Subject: [PATCH 382/979] CLI: Use enumerate for renderers (thanks @gcmoreira) --- volatility3/cli/text_renderer.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 1f9cfa9d8..da0cdf62a 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -180,8 +180,7 @@ class QuickTextRenderer(CLIRenderer): def visitor(node: interfaces.renderers.TreeNode, accumulator): line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) @@ -261,8 +260,7 @@ class CSVRenderer(CLIRenderer): # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case row = {"TreeDepth": str(max(0, node.path_depth - 1))} line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) @@ -326,8 +324,7 @@ class PrettyTextRenderer(CLIRenderer): line = {} rendered_line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) @@ -357,8 +354,7 @@ class PrettyTextRenderer(CLIRenderer): format_string_list = [ "{0:<" + str(max_column_widths.get(tree_indent_column, 0)) + "s}" ] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): format_string_list.append( "{" + str(column_index + 1) @@ -448,8 +444,7 @@ class JsonRenderer(CLIRenderer): acc_map, final_tree = accumulator node_dict: Dict[str, Any] = {"__children": []} line = [] - for column_index in range(len(grid.columns)): - column = grid.columns[column_index] + for column_index, column in enumerate(grid.columns): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) From b7e604d63f667663f9a00415493ec4f8b12a5024 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 10 Sep 2024 14:50:40 -0500 Subject: [PATCH 383/979] Address feedback --- .../plugins/windows/debugregisters.py | 33 +- .../framework/plugins/windows/pe_symbols.py | 394 ++++++++++++++---- .../plugins/windows/unhooked_system_calls.py | 7 +- .../framework/plugins/windows/vadinfo.py | 4 +- 4 files changed, 331 insertions(+), 107 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 931ccef37..65b2e625b 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -1,3 +1,6 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + import logging from typing import Tuple, Optional, Generator, List, Dict @@ -35,11 +38,16 @@ class DebugRegisters(interfaces.plugins.PluginInterface): ), ] + @staticmethod def _get_debug_info( - self, ethread: interfaces.objects.ObjectInterface + ethread: interfaces.objects.ObjectInterface, ) -> Optional[Tuple[interfaces.objects.ObjectInterface, int, int, int, int, int]]: """ Gathers information related to the debug registers for the given thread + Args: + ethread: the thread (_ETHREAD) to examine + Returns: + Tuple[interfaces.objects.ObjectInterface, int, int, int, int, int]: The owner process of the thread and the values for dr7, dr0, dr1, dr2, dr3 """ try: dr7 = ethread.Tcb.TrapFrame.Dr7 @@ -68,23 +76,6 @@ class DebugRegisters(interfaces.plugins.PluginInterface): return owner_proc, dr7, dr0, dr1, dr2, dr3 - def _get_vads( - self, - vads_cache: Dict[int, List[Tuple[int, int, str]]], - owner_proc: interfaces.objects.ObjectInterface, - ) -> Optional[List[Tuple[int, int, str]]]: - if owner_proc.vol.offset in vads_cache: - vads = vads_cache[owner_proc.vol.offset] - else: - vads = pe_symbols.PESymbols.get_proc_vads_with_file_paths(owner_proc) - vads_cache[owner_proc.vol.offset] = vads - - # smear or terminated process - if len(vads) == 0: - return None - - return vads - def _generator( self, ) -> Generator[ @@ -115,7 +106,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): ]: kernel = self.context.modules[self.config["kernel"]] - vads_cache: Dict[int, List[Tuple[int, int, str]]] = {} + vads_cache: Dict[int, pe_symbols.ranges_type] = {} proc_modules = None @@ -133,7 +124,9 @@ class DebugRegisters(interfaces.plugins.PluginInterface): owner_proc, dr7, dr0, dr1, dr2, dr3 = debug_info - vads = self._get_vads(vads_cache, owner_proc) + vads = pe_symbols.PESymbols.get_vads_for_process_cache( + vads_cache, owner_proc + ) if not vads: continue diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 10e87c552..c9785a1c9 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -1,9 +1,9 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 - import io import logging +import ntpath from typing import Dict, Tuple, Optional, List, Generator, Union @@ -17,9 +17,37 @@ from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbutil from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import pslist, modules +from volatility3.framework.constants.windows import KERNEL_MODULE_NAMES vollog = logging.getLogger(__name__) +# keys for specifying wanted names and/or addresses +# used for consistent access between the API and plugins +wanted_names_identifier = "names" +wanted_addresses_identifier = "addresses" + +# how wanted modules/symbols are specified, such as: +# {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} +# {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} +filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + +# holds resolved symbols +# {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} +found_symbols_type = Dict[str, List[Tuple[str, int]]] + +# used to hold informatin about a range (VAD or kernel module) +# (start address, size, file path) +range_type = Tuple[int, int, str] +ranges_type = List[range_type] + +# collected_modules are modules and their symbols found when walking vads or kernel modules +# Tuple of (process or kernel layer name, range start, range size) +collected_module_instance = Tuple[str, int, int] +collected_modules_info = List[collected_module_instance] +collected_modules_type = Dict[str, collected_modules_info] + +PESymbolFinders = Union[interfaces.context.ModuleInterface, pefile.ExportDirData] + class PESymbolFinder: """ @@ -28,21 +56,19 @@ class PESymbolFinder: lookup symbols by name or address """ - cached_str = Union[str, None] - cached_str_dict = Dict[str, cached_str] + cached_str_dict = Dict[str, Optional[str]] - cached_int = Union[int, None] - cached_int_dict = Dict[str, cached_int] + cached_int_dict = Dict[str, Optional[int]] cached_value = Union[int, str, None] - cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + cached_value_dict = Dict[str, Dict[str, List[str]] | Dict[str, List[int]]] def __init__( self, layer_name: str, mod_name: str, module_start: int, - symbol_module: Union[interfaces.context.ModuleInterface, pefile.ExportDirData], + symbol_module: PESymbolFinders, ): self._layer_name = layer_name self._mod_name = mod_name @@ -56,10 +82,25 @@ class PESymbolFinder: """ Maintain a cache for symbol lookups to avoid re-walking of PDB symbols or export tables within the same module for the same address in the same layer + + Args: + value: The value (address or name) being cached + + Returns: + str: The constructed cache key that includes the layer and module name """ return f"{self._layer_name}|{self._mod_name}|{value}" - def get_name_for_address(self, address: int) -> cached_str: + def get_name_for_address(self, address: int) -> Optional[str]: + """ + Returns the name for the given address within the particular layer and module + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ cached_key = self._get_cache_key(address) if cached_key not in self._name_cache: name = self._do_get_name(address) @@ -67,7 +108,16 @@ class PESymbolFinder: return self._name_cache[cached_key] - def get_address_for_name(self, name: str) -> cached_int: + def get_address_for_name(self, name: str) -> Optional[int]: + """ + Returns the name for the given address within the particular layer and module + + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ cached_key = self._get_cache_key(name) if cached_key not in self._address_cache: address = self._do_get_address(name) @@ -75,10 +125,30 @@ class PESymbolFinder: return self._address_cache[cached_key] - def _do_get_name(self, address: int) -> cached_str: + def _do_get_name(self, address: int) -> Optional[str]: + """ + Returns the name for the given address within the particular layer and module. + This method must be overwritten by sub classes. + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ raise NotImplementedError("_do_get_name must be overwritten") - def _do_get_address(self, name: str) -> cached_int: + def _do_get_address(self, name: str) -> Optional[int]: + """ + Returns the name for the given address within the particular layer and module + This method must be overwritten by sub classes. + + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ raise NotImplementedError("_do_get_address must be overwritten") @@ -87,13 +157,31 @@ class PDBSymbolFinder(PESymbolFinder): PESymbolFinder implementation for PDB modules """ - def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + def _do_get_address(self, name: str) -> Optional[int]: + """ + _do_get_address implementation for PDBSymbolFinder + + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ try: return self._symbol_module.get_absolute_symbol_address(name) except exceptions.SymbolError: return None - def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + def _do_get_name(self, address: int) -> Optional[str]: + """ + _do_get_name implementation for PDBSymbolFinder + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ try: name = self._symbol_module.get_symbols_by_absolute_location(address)[0] return name.split(constants.BANG)[1] @@ -113,14 +201,32 @@ class ExportSymbolFinder(PESymbolFinder): except AttributeError: return None - def _do_get_name(self, address: int) -> PESymbolFinder.cached_str: + def _do_get_name(self, address: int) -> Optional[str]: + """ + _do_get_name implementation for ExportSymbolFinder + + Args: + address: the address to resolve within the module + + Returns: + str: the name of the symbol, if found + """ for export in self._symbol_module: if export.address + self._module_start == address: return self._get_name(export) return None - def _do_get_address(self, name: str) -> PESymbolFinder.cached_int: + def _do_get_address(self, name: str) -> Optional[int]: + """ + _do_get_address implementation for ExportSymbolFinder + Args: + str: the name of the symbol to resolve + + Returns: + address: the address of the symbol, if found + """ + for export in self._symbol_module: sym_name = self._get_name(export) if sym_name and sym_name == name: @@ -139,25 +245,6 @@ class PESymbols(interfaces.plugins.PluginInterface): # used for special handling of the kernel PDB file. See later notes os_module_name = "ntoskrnl.exe" - # keys for specifying wanted names and/or addresses - # used for consistent access between the API and plugins - wanted_names = "names" - wanted_addresses = "addresses" - - # how wanted modules/symbols are specified, such as: - # {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} - # {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} - filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] - - # holds resolved symbols - # {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} - found_symbols_type = Dict[str, List[Tuple[str, int]]] - - # used to hold informatin about a range (VAD or kernel module) - # (start address, size, file path) - range_type = Tuple[int, int, str] - ranges_type = List[range_type] - @classmethod def get_requirements(cls) -> List: # Since we're calling the plugin, make sure we have the plugin's requirements @@ -187,13 +274,15 @@ class PESymbols(interfaces.plugins.PluginInterface): description='Module in which to resolve symbols. Use "ntoskrnl.exe" to resolve in the base kernel executable.', optional=False, ), - requirements.StringRequirement( - name="symbol", + requirements.ListRequirement( + name="symbols", + element_type=str, description="Symbol name to resolve", optional=True, ), - requirements.IntRequirement( - name="address", + requirements.ListRequirement( + name="addresses", + element_type=int, description="Address of symbol to resolve", optional=True, ), @@ -242,7 +331,15 @@ class PESymbols(interfaces.plugins.PluginInterface): ranges: ranges_type, address: int ) -> Optional[range_type]: """ - Helper for getting the range information for an address + Helper for getting the range information for an address. + Finds the range holding the `address` parameter + + Args: + address: the address to find the range for + + Returns: + Tuple[int, int, str]: The starting address, size, and file path of the range + """ for start, size, filepath in ranges: if start <= address < start + size: @@ -254,6 +351,13 @@ class PESymbols(interfaces.plugins.PluginInterface): def filepath_for_address(ranges: ranges_type, address: int) -> Optional[str]: """ Helper to get the file path for an address + + Args: + ranges: The set of VADs with mapped files to find the address + address: The address to find inside of the VADs set + + Returns: + str: The full path of the file, if found and present """ info = PESymbols.range_info_for_address(ranges, address) if info: @@ -264,9 +368,15 @@ class PESymbols(interfaces.plugins.PluginInterface): @staticmethod def filename_for_path(filepath: str) -> str: """ - Consistent way to get the filename + Consistent way to get the filename regardless of platform + + Args: + str: the file path from `filepath_for_address` + + Returns: + str: the bsae file name of the full path """ - return filepath.split("\\")[-1] + return ntpath.basename(filepath) @staticmethod def addresses_for_process_symbols( @@ -276,6 +386,18 @@ class PESymbols(interfaces.plugins.PluginInterface): symbol_table_name: str, symbols: filter_modules_type, ) -> found_symbols_type: + """ + Used to easily resolve the addresses of names inside of modules. + + See the usage of this function for system call resolution in unhooked_system_calls.py + for an easy to understand example. + + Args: + symbols: The dictionary of symbols requested by the caller + + Returns: + found_symbols_type: The dictionary of symbols that were resolved + """ collected_modules = PESymbols.get_process_modules( context, layer_name, symbol_table_name, symbols ) @@ -294,16 +416,22 @@ class PESymbols(interfaces.plugins.PluginInterface): def path_and_symbol_for_address( context: interfaces.context.ContextInterface, config_path: str, - collected_modules: Dict[str, List[Tuple[str, int, int]]], + collected_modules: collected_modules_type, ranges: ranges_type, address: int, ) -> Tuple[str, str]: """ Method for plugins to determine the file path and symbol name for a given address - collected_modules: return value from `get_kernel_modules` or `get_process_modules` - ranges: the memory ranges to examine in this layer. - address: address to resolve to its symbol name + See debugregisters.py for an example of how this function is used along with get_vads_for_process_cache + for resolving symbols in processes. + + Args: + collected_modules: return value from `get_kernel_modules` or `get_process_modules` + ranges: the memory ranges to examine in this layer. + address: address to resolve to its symbol name + Returns: + Tuple[str|renderers.NotApplicableValue|renderers.NotAvailableValue, str|renderers.NotApplicableValue|renderers.NotAvailableValue] """ if not address: @@ -317,15 +445,15 @@ class PESymbols(interfaces.plugins.PluginInterface): filename = PESymbols.filename_for_path(filepath).lower() # setup to resolve the address - filter_module: PESymbols.filter_modules_type = { - filename: {PESymbols.wanted_addresses: [address]} + filter_module: filter_modules_type = { + filename: {wanted_addresses_identifier: [address]} } found_symbols = PESymbols.find_symbols( context, config_path, filter_module, collected_modules ) - if not found_symbols or not found_symbols[filename]: + if not found_symbols or filename not in found_symbols: return renderers.NotAvailableValue(), renderers.NotAvailableValue() return filepath, found_symbols[filename][0][0] @@ -335,13 +463,18 @@ class PESymbols(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, pe_table_name: str, mod_name: str, - module_info: Tuple[str, int, int], + module_info: collected_module_instance, ) -> Optional[ExportSymbolFinder]: """ Attempts to locate symbols based on export analysis - mod_name: lower case name of the module to resolve symbols in - module_info: (layer_name, module_start, module_size) of the module to examine + Args: + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + + Returns: + Optional[ExportSymbolFinder]: If the export table can be resolved, then the ExportSymbolFinder + instance for it """ layer_name = module_info[0] @@ -361,7 +494,10 @@ class PESymbols(interfaces.plugins.PluginInterface): return None return ExportSymbolFinder( - layer_name, mod_name, module_start, pe_module.DIRECTORY_ENTRY_EXPORT.symbols + layer_name, + mod_name.lower(), + module_start, + pe_module.DIRECTORY_ENTRY_EXPORT.symbols, ) @staticmethod @@ -369,13 +505,17 @@ class PESymbols(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, config_path: str, mod_name: str, - module_info: Tuple[str, int, int], + module_info: collected_module_instance, ) -> Optional[PDBSymbolFinder]: """ - Attempts to locate symbols based on PDB analysis + Attempts to locate symbols based on PDB analysis through each layer where the mod_name module was found - mod_name: lower case name of the module to resolve symbols in - module_info: (layer_name, module_start, module_size) of the module to examine + Args: + mod_name: lower case name of the module to resolve symbols in + module_info: (layer_name, module_start, module_size) of the module to examine + + Returns: + Optional[PDBSymbolFinder]: If the export table can be resolved, then the ExportSymbolFinder """ mod_symbols = None @@ -386,15 +526,17 @@ class PESymbols(interfaces.plugins.PluginInterface): # a `ntoskrnl.exe` can have an internal PDB name of any of the ones in the following list # The code attempts to find all possible PDBs to ensure the best chance of recovery if mod_name == PESymbols.os_module_name: - pdb_names = ["ntkrnlmp.pdb", "ntkrnlpa.pdb", "ntkrpamp.pdb", "ntoskrnl.pdb"] + pdb_names = [fn + ".pdb" for fn in KERNEL_MODULE_NAMES] # for non-kernel files, replace the exe, sys, or dll extension with pdb else: + # in testing we found where some DLLs, such amsi.dll, have its PDB string as Amsi.dll + # in certain Windows versions mod_name = mod_name[:-3] + "pdb" first_upper = mod_name[0].upper() + mod_name[1:] pdb_names = [mod_name, first_upper] - # loop through each PDB name (will be just one for all but the kernel) + # loop through each PDB name (all the kernel names or the dll name as lower() + first char upper case) for pdb_name in pdb_names: try: mod_symbols = pdbutil.PDBUtility.symbol_table_from_pdb( @@ -433,11 +575,17 @@ class PESymbols(interfaces.plugins.PluginInterface): def _find_symbols_through_pdb( context: interfaces.context.ContextInterface, config_path: str, - module_instances: List[Tuple[str, int, int]], + module_instances: collected_modules_info, mod_name: str, ) -> Generator[PDBSymbolFinder, None, None]: """ - Attempts to resolve the symbols in `wanted_symbols` through PDB analysis + Attempts to resolve the symbols in `mod_name` through PDB analysis + + Args: + module_instances: the set of layers in which the module was found + mod_name: name of the module to resolve symbols in + Returns: + Generator[PDBSymbolFinder]: a PDBSymbolFinder instance for each layer in which the module was found """ for module_info in module_instances: mod_module = PESymbols._get_pdb_module( @@ -450,11 +598,17 @@ class PESymbols(interfaces.plugins.PluginInterface): def _find_symbols_through_exports( context: interfaces.context.ContextInterface, config_path: str, - module_instances: List[Tuple[str, int, int]], + module_instances: collected_modules_info, mod_name: str, ) -> Generator[ExportSymbolFinder, None, None]: """ - Attempts to resolve the symbols in `wanted_symbols` through export analysis + Attempts to resolve the symbols in `mod_name` through export analysis + + Args: + module_instances: the set of layers in which the module was found + mod_name: name of the module to resolve symbols in + Returns: + Generator[ExportSymbolFinder]: an ExportSymbolFinder instance for each layer in which the module was found """ pe_table_name = intermed.IntermediateSymbolTable.create( context, config_path, "windows", "pe", class_types=pe.class_types @@ -476,12 +630,21 @@ class PESymbols(interfaces.plugins.PluginInterface): ) -> Generator[Tuple[str, int], None, None]: """ Enumerates the symbols specified as wanted by the calling plugin + + removes entries from wanted_modules as they found to avoid PDB or export analysis after resolving all symbols + + Args: + wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. + mod_name: the name of module to resolve symbols in + + Returns: + Tuple[str, int]: the name and address of resolved symbols """ wanted_symbols = wanted_modules[mod_name] if ( - PESymbols.wanted_names not in wanted_symbols - and PESymbols.wanted_addresses not in wanted_symbols + wanted_names_identifier not in wanted_symbols + and wanted_addresses_identifier not in wanted_symbols ): vollog.warning( f"Invalid `wanted_symbols` sent to `find_symbols` for module {mod_name}. addresses and names keys both misssing." @@ -489,8 +652,8 @@ class PESymbols(interfaces.plugins.PluginInterface): return symbol_keys = [ - (PESymbols.wanted_names, "get_address_for_name"), - (PESymbols.wanted_addresses, "get_name_for_address"), + (wanted_names_identifier, "get_address_for_name"), + (wanted_addresses_identifier, "get_name_for_address"), ] for symbol_key, symbol_getter in symbol_keys: @@ -503,7 +666,7 @@ class PESymbols(interfaces.plugins.PluginInterface): ) if symbol_value: # yield out symbol name, symbol address - if symbol_key == PESymbols.wanted_names: + if symbol_key == wanted_names_identifier: yield wanted_value, symbol_value # type: ignore else: yield symbol_value, wanted_value # type: ignore @@ -521,13 +684,20 @@ class PESymbols(interfaces.plugins.PluginInterface): def _resolve_symbols_through_methods( context: interfaces.context.ContextInterface, config_path: str, - module_instances: List[Tuple[str, int, int]], + module_instances: collected_modules_info, wanted_modules: PESymbolFinder.cached_value_dict, mod_name: str, ) -> Generator[Tuple[str, int], None, None]: """ Attempts to resolve every wanted symbol in `mod_name` Every layer is enumerated for maximum chance of recovery + + Args: + module_instances: the set of layers in which the module was found + wanted_modules: The symbols to resolve tied to their module names + mod_name: name of the module to resolve symbols in + Returns: + Generator[Tuple[str, int]]: resolved symbol names and addresses """ symbol_resolving_methods = [ PESymbols._find_symbols_through_pdb, @@ -554,13 +724,19 @@ class PESymbols(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, config_path: str, wanted_modules: PESymbolFinder.cached_value_dict, - collected_modules: Dict[str, List[Tuple[str, int, int]]], + collected_modules: collected_modules_type, ) -> found_symbols_type: """ Loops through each method of symbol analysis until each wanted symbol is found Returns the resolved symbols as a dictionary that includes the name and runtime address + + Args: + wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. + collected_modules: return value from `get_kernel_modules` or `get_process_modules` + Returns: + found_symbols_type: The set of symbols resolved to their name and/or address """ - found_symbols: PESymbols.found_symbols_type = {} + found_symbols: found_symbols_type = {} for mod_name in wanted_modules: if mod_name not in collected_modules: @@ -594,11 +770,16 @@ class PESymbols(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, filter_modules: Optional[filter_modules_type], - ) -> Dict[str, List[Tuple[str, int, int]]]: + ) -> collected_modules_type: """ Walks the kernel module list and finds the session layer, base, and size of each wanted module + + Args: + filter_modules: The modules to filter the gathering to. If left as None, all kernel modules are gathered. + Returns: + collected_modules_type: The collection of modules found with at least one layer present """ - found_modules: Dict[str, List[Tuple[str, int, int]]] = {} + found_modules: collected_modules_type = {} if filter_modules: # create a tuple of module names for use with `endswith` @@ -645,16 +826,55 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_modules + @staticmethod + def get_vads_for_process_cache( + vads_cache: Dict[int, ranges_type], + owner_proc: interfaces.objects.ObjectInterface, + ) -> Optional[ranges_type]: + """ + Creates and utilizes a cache of a process' VADs for efficient lookups + + Returns the vad information of the VAD hosting the address, if found + + Args: + vads_cache: The existing cache of VADs + owner_proc: The process being inspected + Returns: + Optional[ranges_type]: The range holding the address, if found + """ + if owner_proc.vol.offset in vads_cache: + vads = vads_cache[owner_proc.vol.offset] + else: + vads = PESymbols.get_proc_vads_with_file_paths(owner_proc) + vads_cache[owner_proc.vol.offset] = vads + + # smear or terminated process + if len(vads) == 0: + return None + + return vads + @staticmethod def get_proc_vads_with_file_paths( proc: interfaces.objects.ObjectInterface, ) -> ranges_type: """ Returns a list of the process' vads that map a file - """ - vads = [] - for vad in proc.get_vad_root().traverse(): + Args: + proc: The process to gather the VADs for + + Returns: + ranges_type: The list of VADs for this process that map a file + """ + vads: ranges_type = [] + + try: + vad_root = proc.get_vad_root() + except exceptions.InvalidAddressException: + return vads + + for vad in vad_root.traverse(): filepath = vad.get_file_name() if not isinstance(filepath, str) or filepath.count("\\") == 0: continue @@ -676,6 +896,9 @@ class PESymbols(interfaces.plugins.PluginInterface): ]: """ Yields each set of vads for a process that have a file mapped, along with the process itself and its layer + + Args: + Generator[Tuple[interfaces.objects.ObjectInterface, str, ranges_type]]: Yields tuple of process objects, layers, and VADs mapping files """ procs = pslist.PsList.list_processes( context=context, @@ -689,7 +912,7 @@ class PESymbols(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: continue - vads = PESymbols.get_proc_vads_with_file_paths(proc) + vads = cls.get_proc_vads_with_file_paths(proc) yield proc, proc_layer_name, vads @@ -699,11 +922,16 @@ class PESymbols(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, filter_modules: Optional[filter_modules_type], - ) -> Dict[str, List[Tuple[str, int, int]]]: + ) -> collected_modules_type: """ Walks the process list and each process' VAD to determine the base address and size of wanted modules + + Args: + filter_modules: The modules to filter the gathering to. If left as None, all process modules are gathered. + Returns: + collected_modules_type: The collection of modules found with at least one layer present """ - proc_modules: Dict[str, List[Tuple[str, int, int]]] = {} + proc_modules: collected_modules_type = {} if filter_modules: # create a tuple of module names for use with `endswith` @@ -711,7 +939,7 @@ class PESymbols(interfaces.plugins.PluginInterface): else: filter_modules_check = None - for _, proc_layer_name, vads in PESymbols.get_all_vads_with_file_paths( + for _proc, proc_layer_name, vads in PESymbols.get_all_vads_with_file_paths( context, layer_name, symbol_table ): for vad_start, vad_size, filepath in vads: @@ -731,17 +959,17 @@ class PESymbols(interfaces.plugins.PluginInterface): def _generator(self) -> Generator[Tuple[int, Tuple[str, str, int]], None, None]: kernel = self.context.modules[self.config["kernel"]] - if self.config["symbol"]: + if self.config["symbols"]: filter_module = { self.config["module"].lower(): { - PESymbols.wanted_names: [self.config["symbol"]] + wanted_names_identifier: self.config["symbols"] } } - elif self.config["address"]: + elif self.config["addresses"]: filter_module = { self.config["module"].lower(): { - PESymbols.wanted_addresses: [self.config["address"]] + wanted_addresses_identifier: self.config["addresses"] } } diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index 0438bc9e3..68f4c4b80 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -1,3 +1,6 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + import logging from typing import Dict, Tuple, List, Generator @@ -18,7 +21,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): system_calls = { "ntdll.dll": { - pe_symbols.PESymbols.wanted_names: [ + pe_symbols.wanted_names_identifier: [ "NtCreateThread", "NtProtectVirtualMemory", "NtReadVirtualMemory", @@ -90,7 +93,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): def _gather_code_bytes( self, kernel: interfaces.context.ModuleInterface, - found_symbols: pe_symbols.PESymbols.found_symbols_type, + found_symbols: pe_symbols.found_symbols_type, ) -> _code_bytes_type: """ Enumerates the desired DLLs and function implementations in each process diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 974793a71..2c6ed4daf 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -9,7 +9,7 @@ from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist, pe_symbols +from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class VadInfo(interfaces.plugins.PluginInterface): _version = (2, 0, 0) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb - def __init__(self, *args, **kwargs): # type: ignore + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._protect_values = None From b40c20dd7fe0d1e348f94531c290a9aceee34dcb Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 10 Sep 2024 14:55:00 -0500 Subject: [PATCH 384/979] Revert back to union to avoid failed tests --- volatility3/framework/plugins/windows/pe_symbols.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index c9785a1c9..30e9b49d1 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -61,7 +61,7 @@ class PESymbolFinder: cached_int_dict = Dict[str, Optional[int]] cached_value = Union[int, str, None] - cached_value_dict = Dict[str, Dict[str, List[str]] | Dict[str, List[int]]] + cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] def __init__( self, From c86d973329e1795ceb0be15e70d9411a38b0f60e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 11 Sep 2024 15:54:17 +1000 Subject: [PATCH 385/979] Linux page cache plugins pointer verification improvements --- .../framework/plugins/linux/mountinfo.py | 11 +++++--- .../framework/plugins/linux/pagecache.py | 27 ++++++++++++------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 1eaec77bf..2499f009e 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -37,7 +37,7 @@ class MountInfo(plugins.PluginInterface): _required_framework_version = (2, 2, 0) - _version = (1, 2, 0) + _version = (1, 2, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -272,11 +272,16 @@ class MountInfo(plugins.PluginInterface): continue sb_ptr = mnt.get_mnt_sb() - if not sb_ptr or sb_ptr in seen_sb_ptr: + if not (sb_ptr and sb_ptr.is_readable()): + continue + + if sb_ptr in seen_sb_ptr: continue seen_sb_ptr.add(sb_ptr) - yield sb_ptr.dereference(), path_root + superblock = sb_ptr.dereference() + + yield superblock, path_root def run(self): pids = self.config.get("pids") diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index b6c5f7cc0..9b6f05c4d 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -104,7 +104,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -186,8 +186,12 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): seen_dentries.add(dentry_addr) - inode = dentry.d_inode - if not (inode and inode.is_valid()): + inode_ptr = dentry.d_inode + if not (inode_ptr and inode_ptr.is_readable()): + continue + + inode = inode_ptr.dereference() + if not inode.is_valid(): continue # This allows us to have consistent paths @@ -242,8 +246,9 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): # More dentry/inode sanity checks root_inode_ptr = root_dentry.d_inode - if not root_inode_ptr: + if not (root_inode_ptr and root_inode_ptr.is_readable()): continue + root_inode = root_inode_ptr.dereference() if not root_inode.is_valid(): continue @@ -269,10 +274,12 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): ): if not file_dentry: continue + # Dentry/inode sanity checks file_inode_ptr = file_dentry.d_inode - if not file_inode_ptr: + if not (file_inode_ptr and file_inode_ptr.is_readable()): continue + file_inode = file_inode_ptr.dereference() if not file_inode.is_valid(): continue @@ -382,7 +389,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -482,15 +489,15 @@ class InodePages(plugins.PluginInterface): vollog.error("You must use either --inode or --find") return + if not inode.is_valid(): + vollog.error("Invalid inode at 0x%x", inode.vol.offset) + return + if not inode.is_reg: vollog.error("The inode is not a regular file") return inode_size = inode.i_size - if not inode.is_valid(): - vollog.error("Invalid inode at 0x%x", self.config["inode"]) - return - for page_obj in inode.get_pages(): page_vaddr = page_obj.vol.offset page_paddr = page_obj.to_paddr() From 037eb1ce036ae7dea48c427742ae889c9b2ec7a3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 11 Sep 2024 16:30:41 +1000 Subject: [PATCH 386/979] Linux Check_creds plugins pointer verification improvements --- volatility3/framework/plugins/linux/check_creds.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index ab6ee4935..45df966d2 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -16,6 +16,8 @@ class Check_creds(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) + @classmethod def get_requirements(cls): return [ @@ -46,7 +48,11 @@ class Check_creds(interfaces.plugins.PluginInterface): tasks = pslist.PsList.list_tasks(self.context, vmlinux.name) for task in tasks: - cred_addr = task.cred.dereference().vol.offset + task_cred_ptr = task.cred + if not (task_cred_ptr and task_cred_ptr.is_readable()): + continue + + cred_addr = task_cred_ptr.dereference().vol.offset if cred_addr not in creds: creds[cred_addr] = [] From c77c662b70c6751087bf947c400a045c81e7a8ec Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 11 Sep 2024 21:09:08 +1000 Subject: [PATCH 387/979] Linux pidhashtable plugin pointer verification improvements --- .../framework/plugins/linux/pidhashtable.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 3223aed4a..edafe97e0 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -20,7 +20,7 @@ class PIDHashTable(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -45,9 +45,7 @@ class PIDHashTable(plugins.PluginInterface): ] def _is_valid_task(self, task) -> bool: - vmlinux = self.context.modules[self.config["kernel"]] - vmlinux_layer = self.context.layers[vmlinux.layer_name] - return bool(task and task.pid > 0 and vmlinux_layer.is_valid(task.parent)) + return bool(task and task.pid > 0 and task.parent.is_readable()) def _get_pidtype_pid(self): vmlinux = self.context.modules[self.config["kernel"]] @@ -96,7 +94,7 @@ class PIDHashTable(plugins.PluginInterface): seen_upids.add(upid.vol.offset) pid_chain = upid.pid_chain - if not (pid_chain and vmlinux_layer.is_valid(pid_chain.vol.offset)): + if not (pid_chain.next and pid_chain.next.is_readable()): break upid = linux.LinuxUtilities.container_of( @@ -105,7 +103,6 @@ class PIDHashTable(plugins.PluginInterface): def _get_upids(self): vmlinux = self.context.modules[self.config["kernel"]] - vmlinux_layer = self.context.layers[vmlinux.layer_name] # 2.6.24 <= kernels < 4.15 pidhash = self._get_pidhash_array() @@ -115,7 +112,7 @@ class PIDHashTable(plugins.PluginInterface): # each entry in the hlist is a upid which is wrapped in a pid ent = hlist.first - while ent and vmlinux_layer.is_valid(ent.vol.offset): + while ent and ent.is_readable(): # upid->pid_chain exists 2.6.24 <= kernel < 4.15 upid = linux.LinuxUtilities.container_of( ent.vol.offset, "upid", "pid_chain", vmlinux @@ -143,7 +140,7 @@ class PIDHashTable(plugins.PluginInterface): continue pid_tasks_0 = pid.tasks[pidtype_pid].first - if not pid_tasks_0: + if not (pid_tasks_0 and pid_tasks_0.is_readable()): continue task = vmlinux.object( @@ -160,7 +157,7 @@ class PIDHashTable(plugins.PluginInterface): pidtype_pid = self._get_pidtype_pid() pid_tasks_0 = pid.tasks[pidtype_pid].first - if not pid_tasks_0: + if not (pid_tasks_0 and pid_tasks_0.is_readable()): return None task_struct_type = vmlinux.get_type("task_struct") From 9e8471799adcf090e20ea98a20309076193e9009 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 10:44:43 +1000 Subject: [PATCH 388/979] Improving code and adding the credential virtual addresses to the output. --- .../framework/plugins/linux/check_creds.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 45df966d2..3e292ae33 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -2,21 +2,18 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import logging - from volatility3.framework import interfaces, renderers +from volatility3.framework.renderers import format_hints from volatility3.framework.configuration import requirements from volatility3.plugins.linux import pslist -vollog = logging.getLogger(__name__) - class Check_creds(interfaces.plugins.PluginInterface): """Checks if any processes are sharing credential structures""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls): @@ -54,18 +51,22 @@ class Check_creds(interfaces.plugins.PluginInterface): cred_addr = task_cred_ptr.dereference().vol.offset - if cred_addr not in creds: - creds[cred_addr] = [] - + creds.setdefault(cred_addr, []) creds[cred_addr].append(task.pid) - for _, pids in creds.items(): + for cred_addr, pids in creds.items(): if len(pids) > 1: - pid_str = "" - for pid in pids: - pid_str = pid_str + f"{pid:d}, " - pid_str = pid_str[:-2] - yield (0, [str(pid_str)]) + pid_str = ", ".join([str(pid) for pid in pids]) + + fields = [ + format_hints.Hex(cred_addr), + pid_str, + ] + yield (0, fields) def run(self): - return renderers.TreeGrid([("PIDs", str)], self._generator()) + headers = [ + ("CredVAddr", format_hints.Hex), + ("PIDs", str), + ] + return renderers.TreeGrid(headers, self._generator()) From 57de357ffdfe87dbcad8c219228a4a0d0e17c173 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 16:05:40 +1000 Subject: [PATCH 389/979] Timeliner plugin: Fix issue with filtering TimeLinerInterface plugins and using the filter argument --- volatility3/framework/plugins/timeliner.py | 23 +++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index f657a2918..abe802e8f 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -45,6 +45,7 @@ class Timeliner(interfaces.plugins.PluginInterface): orders the results by time.""" _required_framework_version = (2, 0, 0) + _version = (1, 1, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -245,6 +246,17 @@ class Timeliner(interfaces.plugins.PluginInterface): filter_list = self.config["plugin-filter"] # Identify plugins that we can run which output datetimes for plugin_class in self.usable_plugins: + if not issubclass(plugin_class, TimeLinerInterface): + continue + + if filter_list and not any( + [ + filter in plugin_class.__module__ + "." + plugin_class.__name__ + for filter in filter_list + ] + ): + continue + try: automagics = automagic.choose_automagic(self.automagics, plugin_class) @@ -276,15 +288,8 @@ class Timeliner(interfaces.plugins.PluginInterface): config_value, ) - if isinstance(plugin, TimeLinerInterface): - if not len(filter_list) or any( - [ - filter - in plugin.__module__ + "." + plugin.__class__.__name__ - for filter in filter_list - ] - ): - plugins_to_run.append(plugin) + plugins_to_run.append(plugin) + except exceptions.UnsatisfiedException as excp: # Remove the failed plugin from the list and continue vollog.debug( From be05ace29b134156fe5f7584921887426fc2f41f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 16:06:40 +1000 Subject: [PATCH 390/979] Timeliner plugin: Add exception information --- volatility3/framework/plugins/timeliner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index abe802e8f..56fe465e4 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -199,9 +199,10 @@ class Timeliner(interfaces.plugins.PluginInterface): ), ) ) - except Exception: + except Exception as e: vollog.log( - logging.INFO, f"Exception occurred running plugin: {plugin_name}" + logging.INFO, + f"Exception occurred running plugin: {plugin_name}: {e}", ) vollog.log(logging.DEBUG, traceback.format_exc()) From a7dcd6d9e8adfe3124aa13db1b1536e6799d822c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 16:19:13 +1000 Subject: [PATCH 391/979] Minor: Add comment on TimeLinerInterface subclass filter --- volatility3/framework/plugins/timeliner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 56fe465e4..d1cb9f460 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -248,6 +248,7 @@ class Timeliner(interfaces.plugins.PluginInterface): # Identify plugins that we can run which output datetimes for plugin_class in self.usable_plugins: if not issubclass(plugin_class, TimeLinerInterface): + # get_usable_plugins() should filter this, but adding a safeguard just in case continue if filter_list and not any( From ebb9f0c20163fe215c72d510e9188dbf1d3c318a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 17:05:45 +1000 Subject: [PATCH 392/979] Fix typo in timeliner function --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 9b6f05c4d..ad2c3b1be 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -342,7 +342,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): description = f"Cached Inode for {inode_out.path}" yield description, timeliner.TimeLinerType.ACCESSED, inode_out.access_time yield description, timeliner.TimeLinerType.MODIFIED, inode_out.modification_time - yield description, timeliner.TimeLinerType.CHANGE, inode_out.change_time + yield description, timeliner.TimeLinerType.CHANGED, inode_out.change_time @staticmethod def format_fields_with_headers(headers, generator): From 7b2e259c6dca79b1d36acdc82bc8d7b76f7aa00b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 17:06:29 +1000 Subject: [PATCH 393/979] fix wrong variable names --- volatility3/framework/plugins/linux/pagecache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index ad2c3b1be..81e1f3601 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -78,8 +78,8 @@ class InodeInternal: cached_pages = int(self.inode.i_mapping.nrpages) file_mode = self.inode.get_file_mode() access_time_dt = self.inode.get_access_time() - modification_time_str = self.inode.get_modification_time() - change_time_str = self.inode.get_change_time() + modification_time_dt = self.inode.get_modification_time() + change_time_dt = self.inode.get_change_time() inode_user = InodeUser( superblock_addr=superblock_addr, @@ -92,8 +92,8 @@ class InodeInternal: cached_pages=cached_pages, file_mode=file_mode, access_time=access_time_dt, - modification_time=modification_time_str, - change_time=change_time_str, + modification_time=modification_time_dt, + change_time=change_time_dt, path=self.path, ) return inode_user From 48ae43d64edd457b65eb40174fb00b54202aabda Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Sep 2024 17:22:52 +1000 Subject: [PATCH 394/979] Bumping the major version since the output changed --- volatility3/framework/plugins/linux/check_creds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 3e292ae33..b7f73c3eb 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -13,7 +13,7 @@ class Check_creds(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): From 867c8e3f1bdeb0fa57dd3e02056f32b526a81fb4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 13 Sep 2024 15:33:19 +1000 Subject: [PATCH 395/979] Linux: Add support for threads in both lsof and sockstat plugins. - lsof plugin: source code refactored - lsof plugin: Added the 'device' column to complete the inode information. An inode number is specific to the filesystem/device it belongs to. - lsof/sockstat plugins: Add threads support. Threads may or may not share the file descriptor table with the thread group leader, depending on whether the CLONE_FILES flag is included in the clone() syscall. Also, once started, a thread can unshare the fd table with its parent. Refer to the unshare() libc syscall wrapper man page, unshare(2). Additionally, note that the Linux lsof command in user space includes thread listings by default as well. Now, there are two columns to identify the thread group ID (PID) and the task/thread ID (TID). - Added inode getters from both, the dentry and file structs. From kernels +3.9 the file struct cached the inode pointer. So, when possible, we get this value. - Improve smear protection in these plugins and various APIs (https://github.com/volatilityfoundation/volatility3/pull/1243) --- volatility3/framework/plugins/linux/lsof.py | 229 +++++++++++------- .../framework/plugins/linux/sockstat.py | 18 +- .../framework/symbols/linux/__init__.py | 4 +- .../symbols/linux/extensions/__init__.py | 42 +++- 4 files changed, 185 insertions(+), 108 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 360f89749..0325ebd4c 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -1,12 +1,12 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" -import logging, datetime +import logging +from datetime import datetime +from dataclasses import dataclass, astuple, field from typing import List, Callable -from volatility3.framework import renderers, interfaces, constants, exceptions +from volatility3.framework import renderers, interfaces, constants from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -17,11 +17,94 @@ from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) +@dataclass +class FDUser: + """FD user representation, featuring augmented information and formatted fields. + This is the data the plugin will eventually display. + """ + + task_tgid: int + task_tid: int + task_comm: str + fd_num: int + full_path: str + device: str = field(default=renderers.NotAvailableValue()) + inode_num: int = field(default=renderers.NotAvailableValue()) + inode_type: str = field(default=renderers.NotAvailableValue()) + file_mode: str = field(default=renderers.NotAvailableValue()) + change_time: datetime = field(default=renderers.NotAvailableValue()) + modification_time: datetime = field(default=renderers.NotAvailableValue()) + access_time: datetime = field(default=renderers.NotAvailableValue()) + inode_size: int = field(default=renderers.NotAvailableValue()) + + +@dataclass +class FDInternal: + """FD internal representation containing only the core objects + + Fields: + task: 'task_truct' object + fd_fields: FD fields as obtained from LinuxUtilities.files_descriptors_for_process() + """ + + task: interfaces.objects.ObjectInterface + fd_fields: tuple[int, int, str] + + def to_user(self) -> FDUser: + """Augment the FD information to be presented to the user + + Returns: + An InodeUser dataclass + """ + # Ensure all types are atomic immutable. Otherwise, astuple() will take a long + # time doing a deepcopy of the Volatility objects. + task_tgid = int(self.task.tgid) + task_tid = int(self.task.pid) + task_comm = utility.array_to_string(self.task.comm) + fd_num, filp, full_path = self.fd_fields + fd_num = int(fd_num) + full_path = str(full_path) + inode = filp.get_inode() + if inode: + superblock_ptr = inode.i_sb + if superblock_ptr and superblock_ptr.is_readable(): + device = f"{superblock_ptr.major}:{superblock_ptr.minor}" + else: + device = renderers.NotAvailableValue() + + fd_user = FDUser( + task_tgid=task_tgid, + task_tid=task_tid, + task_comm=task_comm, + fd_num=fd_num, + full_path=full_path, + device=device, + inode_num=int(inode.i_ino), + inode_type=inode.get_inode_type() or renderers.UnparsableValue(), + file_mode=inode.get_file_mode(), + change_time=inode.get_change_time(), + modification_time=inode.get_modification_time(), + access_time=inode.get_access_time(), + inode_size=int(inode.i_size), + ) + else: + # We use the dataclasses' default values + fd_user = FDUser( + task_tgid=task_tgid, + task_tid=task_tid, + task_comm=task_comm, + fd_num=fd_num, + full_path=full_path, + ) + + return fd_user + + class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists open files for each processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 2, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -45,126 +128,86 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] - @classmethod - def get_inode_metadata(cls, filp: interfaces.objects.ObjectInterface): - try: - dentry = filp.get_dentry() - if dentry: - inode_object = dentry.d_inode - if inode_object and inode_object.is_valid(): - itype = ( - inode_object.get_inode_type() or renderers.NotAvailableValue() - ) - return ( - inode_object.i_ino, - itype, - inode_object.i_size, - inode_object.get_file_mode(), - inode_object.get_change_time(), - inode_object.get_modification_time(), - inode_object.get_access_time(), - ) - except (exceptions.InvalidAddressException, AttributeError) as e: - vollog.warning(f"Can't get inode metadata: {e}") - return None - @classmethod def list_fds( cls, context: interfaces.context.ContextInterface, - symbol_table: str, + vmlinux_module_name: str, filter_func: Callable[[int], bool] = lambda _: False, - ): + ) -> FDInternal: + """Enumerates open file descriptors in tasks + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + filter_func: A function which takes a process object and returns True if the process + should be ignored/filtered + + Yields: + A FDInternal object + """ linuxutils_symbol_table = None - for task in pslist.PsList.list_tasks(context, symbol_table, filter_func): + for task in pslist.PsList.list_tasks( + context, vmlinux_module_name, filter_func, include_threads=True + ): if linuxutils_symbol_table is None: if constants.BANG not in task.vol.type_name: raise ValueError("Task is not part of a symbol table") linuxutils_symbol_table = task.vol.type_name.split(constants.BANG)[0] - task_comm = utility.array_to_string(task.comm) - pid = int(task.pid) - fd_generator = linux.LinuxUtilities.files_descriptors_for_process( context, linuxutils_symbol_table, task ) for fd_fields in fd_generator: - yield pid, task_comm, task, fd_fields + yield FDInternal(task=task, fd_fields=fd_fields) - @classmethod - def list_fds_and_inodes( - cls, - context: interfaces.context.ContextInterface, - symbol_table: str, - filter_func: Callable[[int], bool] = lambda _: False, - ): - for pid, task_comm, task, (fd_num, filp, full_path) in cls.list_fds( - context, symbol_table, filter_func - ): - inode_metadata = cls.get_inode_metadata(filp) - if inode_metadata is None: - inode_metadata = tuple( - interfaces.renderers.BaseAbsentValue() for _ in range(7) - ) - yield pid, task_comm, task, fd_num, filp, full_path, inode_metadata - - def _generator(self, pids, symbol_table): + def _generator(self, pids, vmlinux_module_name): filter_func = pslist.PsList.create_pid_filter(pids) - fds_generator = self.list_fds_and_inodes( - self.context, symbol_table, filter_func=filter_func - ) - - for ( - pid, - task_comm, - task, - fd_num, - filp, - full_path, - inode_metadata, - ) in fds_generator: - inode_num, itype, file_size, imode, ctime, mtime, atime = inode_metadata - fields = ( - pid, - task_comm, - fd_num, - full_path, - inode_num, - itype, - imode, - ctime, - mtime, - atime, - file_size, - ) - yield (0, fields) + for fd_internal in self.list_fds( + self.context, vmlinux_module_name, filter_func=filter_func + ): + fd_user = fd_internal.to_user() + yield (0, astuple(fd_user)) def run(self): pids = self.config.get("pid", None) - symbol_table = self.config["kernel"] + vmlinux_module_name = self.config["kernel"] tree_grid_args = [ ("PID", int), + ("TID", int), ("Process", str), ("FD", int), ("Path", str), + ("Device", str), ("Inode", int), ("Type", str), ("Mode", str), - ("Changed", datetime.datetime), - ("Modified", datetime.datetime), - ("Accessed", datetime.datetime), + ("Changed", datetime), + ("Modified", datetime), + ("Accessed", datetime), ("Size", int), ] - return renderers.TreeGrid(tree_grid_args, self._generator(pids, symbol_table)) + return renderers.TreeGrid( + tree_grid_args, self._generator(pids, vmlinux_module_name) + ) def generate_timeline(self): pids = self.config.get("pid", None) - symbol_table = self.config["kernel"] - for row in self._generator(pids, symbol_table): - _depth, row_data = row - description = f'Process {row_data[1]} ({row_data[0]}) Open "{row_data[3]}"' - yield description, timeliner.TimeLinerType.CHANGED, row_data[7] - yield description, timeliner.TimeLinerType.MODIFIED, row_data[8] - yield description, timeliner.TimeLinerType.ACCESSED, row_data[9] + vmlinux_module_name = self.config["kernel"] + + filter_func = pslist.PsList.create_pid_filter(pids) + for fd_internal in self.list_fds( + self.context, vmlinux_module_name, filter_func=filter_func + ): + fd_user = fd_internal.to_user() + + description = ( + f"Process {fd_user.task_comm} ({fd_user.task_tgid}/{fd_user.task_tid}) " + f"Open '{fd_user.full_path}'" + ) + + yield description, timeliner.TimeLinerType.CHANGED, fd_user.change_time + yield description, timeliner.TimeLinerType.MODIFIED, fd_user.modification_time + yield description, timeliner.TimeLinerType.ACCESSED, fd_user.access_time diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index b0503b105..c76e0a6b4 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -22,9 +22,10 @@ class SockHandlers(interfaces.configuration.VersionableInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) - def __init__(self, vmlinux, task): + def __init__(self, vmlinux, task, *args, **kwargs): + super().__init__(*args, **kwargs) self._vmlinux = vmlinux self._task = task @@ -438,7 +439,7 @@ class Sockstat(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -452,7 +453,7 @@ class Sockstat(plugins.PluginInterface): name="SockHandlers", component=SockHandlers, version=(1, 0, 0) ), requirements.PluginRequirement( - name="lsof", plugin=lsof.Lsof, version=(1, 1, 0) + name="lsof", plugin=lsof.Lsof, version=(2, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) @@ -507,8 +508,9 @@ class Sockstat(plugins.PluginInterface): dfop_addr = vmlinux.object_from_symbol("sockfs_dentry_operations").vol.offset fd_generator = lsof.Lsof.list_fds(context, vmlinux.name, filter_func) - for _pid, _task_comm, task, fd_fields in fd_generator: - fd_num, filp, _full_path = fd_fields + for fd_internal in fd_generator: + fd_num, filp, _full_path = fd_internal.fd_fields + task = fd_internal.task if filp.f_op not in (sfop_addr, dfop_addr): continue @@ -617,6 +619,7 @@ class Sockstat(plugins.PluginInterface): fields = ( netns_id, + task.tgid, task.pid, fd_num, format_hints.Hex(sock.vol.offset), @@ -636,7 +639,8 @@ class Sockstat(plugins.PluginInterface): tree_grid_args = [ ("NetNS", int), - ("Pid", int), + ("PID", int), + ("TID", int), ("FD", int), ("Sock Offset", format_hints.Hex), ("Family", str), diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 91abf7db4..6e9297517 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -256,7 +256,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): task: interfaces.objects.ObjectInterface, ): # task.files can be null - if not task.files: + if not (task.files and task.files.is_readable()): return None fd_table = task.files.get_fds() @@ -276,7 +276,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ) for fd_num, filp in enumerate(fds): - if filp != 0: + if filp and filp.is_readable(): full_path = LinuxUtilities.path_for_file(context, task, filp) yield fd_num, filp, full_path diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fdd34403a..fd5ddcffa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -843,25 +843,55 @@ class dentry(objects.StructType): dentry_type_name = self.get_symbol_table_name() + constants.BANG + "dentry" yield from list_head_member.to_list(dentry_type_name, walk_member) + def get_inode(self) -> interfaces.objects.ObjectInterface: + """Returns the inode associated with this dentry""" + + inode_ptr = self.d_inode + if not (inode_ptr and inode_ptr.is_valid()): + return None + + return inode_ptr.dereference() + class struct_file(objects.StructType): def get_dentry(self) -> interfaces.objects.ObjectInterface: - if self.has_member("f_dentry"): - return self.f_dentry - elif self.has_member("f_path"): + """Returns a pointer to the dentry associated with this file""" + if self.has_member("f_path"): return self.f_path.dentry + elif self.has_member("f_dentry"): + return self.f_dentry else: raise AttributeError("Unable to find file -> dentry") def get_vfsmnt(self) -> interfaces.objects.ObjectInterface: """Returns the fs (vfsmount) where this file is mounted""" - if self.has_member("f_vfsmnt"): - return self.f_vfsmnt - elif self.has_member("f_path"): + if self.has_member("f_path"): return self.f_path.mnt + elif self.has_member("f_vfsmnt"): + return self.f_vfsmnt else: raise AttributeError("Unable to find file -> vfs mount") + def get_inode(self) -> interfaces.objects.ObjectInterface: + """Returns an inode associated with this file""" + + inode_ptr = None + if self.has_member("f_inode") and self.f_inode and self.f_inode.is_readable(): + # Try first the cached value, kernels +3.9 + inode_ptr = self.f_inode + + if not (inode_ptr and inode_ptr.is_valid()): + dentry_ptr = self.get_dentry() + if not (dentry_ptr and dentry_ptr.is_readable()): + return None + + inode_ptr = dentry_ptr.d_inode + + if not (inode_ptr and inode_ptr.is_valid()): + return None + + return inode_ptr.dereference() + class list_head(objects.StructType, collections.abc.Iterable): def to_list( From aecd31f0953142993dc10126be77441b8178dc51 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 13 Sep 2024 16:09:49 +1000 Subject: [PATCH 396/979] Add missing pointer verification --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fd5ddcffa..46bed72aa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -847,7 +847,7 @@ class dentry(objects.StructType): """Returns the inode associated with this dentry""" inode_ptr = self.d_inode - if not (inode_ptr and inode_ptr.is_valid()): + if not (inode_ptr and inode_ptr.is_readable() and inode_ptr.is_valid()): return None return inode_ptr.dereference() @@ -880,14 +880,14 @@ class struct_file(objects.StructType): # Try first the cached value, kernels +3.9 inode_ptr = self.f_inode - if not (inode_ptr and inode_ptr.is_valid()): + if not (inode_ptr and inode_ptr.is_readable() and inode_ptr.is_valid()): dentry_ptr = self.get_dentry() if not (dentry_ptr and dentry_ptr.is_readable()): return None inode_ptr = dentry_ptr.d_inode - if not (inode_ptr and inode_ptr.is_valid()): + if not (inode_ptr and inode_ptr.is_readable() and inode_ptr.is_valid()): return None return inode_ptr.dereference() From 32eaeeb7d2b631b4ff9c89a2bea2793e2ca36135 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 13 Sep 2024 16:13:00 +1000 Subject: [PATCH 397/979] Fix typing typo --- volatility3/framework/plugins/linux/lsof.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 0325ebd4c..527815738 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -4,7 +4,7 @@ import logging from datetime import datetime from dataclasses import dataclass, astuple, field -from typing import List, Callable +from typing import List, Callable, Tuple from volatility3.framework import renderers, interfaces, constants from volatility3.framework.configuration import requirements @@ -48,7 +48,7 @@ class FDInternal: """ task: interfaces.objects.ObjectInterface - fd_fields: tuple[int, int, str] + fd_fields: Tuple[int, int, str] def to_user(self) -> FDUser: """Augment the FD information to be presented to the user From e084086ee5f2d9dad29182bc4616912cbfe53f1e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 13 Sep 2024 16:18:26 +1000 Subject: [PATCH 398/979] Improve pointer verification in super_block->get_type() --- .../symbols/linux/extensions/__init__.py | 62 ++++++++++++++++--- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fdd34403a..aa18b8262 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -638,11 +638,23 @@ class super_block(objects.StructType): ] return sb_opts - def get_type(self): - mnt_sb_type = utility.pointer_to_string(self.s_type.name, count=255) - if self.s_subtype: - mnt_sb_subtype = utility.pointer_to_string(self.s_subtype, count=255) + def get_type(self) -> Optional[str]: + """Gets the superblock filesystem type string""" + + s_type_ptr = self.s_type + if not (s_type_ptr and s_type_ptr.is_readable()): + return None + + s_type_name_ptr = s_type_ptr.name + if not (s_type_name_ptr and s_type_name_ptr.is_readable()): + return None + + mnt_sb_type = utility.pointer_to_string(s_type_name_ptr, count=255) + s_subtype_ptr = self.s_subtype + if s_subtype_ptr and s_subtype_ptr.is_readable(): + mnt_sb_subtype = utility.pointer_to_string(s_subtype_ptr, count=255) mnt_sb_type += "." + mnt_sb_subtype + return mnt_sb_type @@ -843,25 +855,55 @@ class dentry(objects.StructType): dentry_type_name = self.get_symbol_table_name() + constants.BANG + "dentry" yield from list_head_member.to_list(dentry_type_name, walk_member) + def get_inode(self) -> interfaces.objects.ObjectInterface: + """Returns the inode associated with this dentry""" + + inode_ptr = self.d_inode + if not (inode_ptr and inode_ptr.is_readable() and inode_ptr.is_valid()): + return None + + return inode_ptr.dereference() + class struct_file(objects.StructType): def get_dentry(self) -> interfaces.objects.ObjectInterface: - if self.has_member("f_dentry"): - return self.f_dentry - elif self.has_member("f_path"): + """Returns a pointer to the dentry associated with this file""" + if self.has_member("f_path"): return self.f_path.dentry + elif self.has_member("f_dentry"): + return self.f_dentry else: raise AttributeError("Unable to find file -> dentry") def get_vfsmnt(self) -> interfaces.objects.ObjectInterface: """Returns the fs (vfsmount) where this file is mounted""" - if self.has_member("f_vfsmnt"): - return self.f_vfsmnt - elif self.has_member("f_path"): + if self.has_member("f_path"): return self.f_path.mnt + elif self.has_member("f_vfsmnt"): + return self.f_vfsmnt else: raise AttributeError("Unable to find file -> vfs mount") + def get_inode(self) -> interfaces.objects.ObjectInterface: + """Returns an inode associated with this file""" + + inode_ptr = None + if self.has_member("f_inode") and self.f_inode and self.f_inode.is_readable(): + # Try first the cached value, kernels +3.9 + inode_ptr = self.f_inode + + if not (inode_ptr and inode_ptr.is_readable() and inode_ptr.is_valid()): + dentry_ptr = self.get_dentry() + if not (dentry_ptr and dentry_ptr.is_readable()): + return None + + inode_ptr = dentry_ptr.d_inode + + if not (inode_ptr and inode_ptr.is_readable() and inode_ptr.is_valid()): + return None + + return inode_ptr.dereference() + class list_head(objects.StructType, collections.abc.Iterable): def to_list( From 997abeda6d9014a028f6c2b7a9e11352320c142f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 13 Sep 2024 17:27:02 +1000 Subject: [PATCH 399/979] Linux lsof: Add namespace dentry name --- .../framework/symbols/linux/__init__.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 91abf7db4..57b45667e 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -169,13 +169,30 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): Returns: str: Sock pipe pathname relative to the task's root directory. """ + # FIXME: This function must be moved to the 'dentry' object extension + # Also, the scope of this function went beyond the sock pipe path, so we need to rename this. + # Once https://github.com/volatilityfoundation/volatility3/pull/1263 is merged, replace the + # dentry inode getters + + if not (filp and filp.is_readable()): + return f" {filp:x}" + dentry = filp.get_dentry() + if not (dentry and dentry.is_readable()): + return f" {dentry:x}" kernel_module = cls.get_module_from_volobj_type(context, dentry) sym_addr = dentry.d_op.d_dname + if not (sym_addr and sym_addr.is_readable()): + return f" {sym_addr:x}" + symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) + inode = dentry.d_inode + if not (inode and inode.is_readable() and inode.is_valid()): + return f" {inode:x}" + if len(symbs) == 1: sym = symbs[0].split(constants.BANG)[1] @@ -191,13 +208,36 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): elif sym == "simple_dname": pre_name = cls._get_path_file(task, filp) - else: - pre_name = f"" + elif sym == "ns_dname": + # From Kernels 3.19 - ret = f"{pre_name}:[{dentry.d_inode.i_ino:d}]" + # In Kernels >= 6.9, see Linux kernel commit 1fa08aece42512be072351f482096d5796edf7ca + # ns_common->stashed change from 'atomic64_t' to 'dentry*' + try: + ns_common_type = kernel_module.get_type("ns_common") + stashed_template = ns_common_type.child_template("stashed") + stashed_type_full_name = stashed_template.vol.type_name + stashed_type_name = stashed_type_full_name.split(constants.BANG)[-1] + if stashed_type_name == "atomic64_t": + # 3.19 <= Kernels < 6.9 + ns_ops = dentry.d_fsdata.dereference().cast( + "proc_ns_operations" + ) + else: + # Kernels >= 6.9 + ns_common = inode.i_private.dereference().cast("ns_common") + ns_ops = ns_common.ops + + pre_name = utility.pointer_to_string(ns_ops.name, 255) + except IndexError: + ret = "" + else: + pre_name = f" {sym}" + + ret = f"{pre_name}:[{inode.i_ino:d}]" else: - ret = f" {sym_addr:x}" + ret = f" {sym_addr:x}" return ret From cd2af74e6d0c554e81d1e67a8020195cfca59983 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 13 Sep 2024 17:58:21 +1000 Subject: [PATCH 400/979] Improve pointers address verification and return message chain --- .../framework/symbols/linux/__init__.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 57b45667e..2410d2627 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -217,29 +217,32 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ns_common_type = kernel_module.get_type("ns_common") stashed_template = ns_common_type.child_template("stashed") stashed_type_full_name = stashed_template.vol.type_name - stashed_type_name = stashed_type_full_name.split(constants.BANG)[-1] + stashed_type_name = stashed_type_full_name.split(constants.BANG)[1] if stashed_type_name == "atomic64_t": # 3.19 <= Kernels < 6.9 - ns_ops = dentry.d_fsdata.dereference().cast( - "proc_ns_operations" - ) + fsdata_ptr = dentry.d_fsdata + if not (fsdata_ptr and fsdata_ptr.is_readable()): + raise IndexError + + ns_ops = fsdata_ptr.dereference().cast("proc_ns_operations") else: # Kernels >= 6.9 - ns_common = inode.i_private.dereference().cast("ns_common") + private_ptr = inode.i_private + if not (private_ptr and private_ptr.is_readable()): + raise IndexError + + ns_common = private_ptr.dereference().cast("ns_common") ns_ops = ns_common.ops pre_name = utility.pointer_to_string(ns_ops.name, 255) except IndexError: - ret = "" + pre_name = "" else: pre_name = f" {sym}" - - ret = f"{pre_name}:[{inode.i_ino:d}]" - else: - ret = f" {sym_addr:x}" + pre_name = f" {sym_addr:x}" - return ret + return f"{pre_name}:[{inode.i_ino:d}]" @classmethod def path_for_file(cls, context, task, filp) -> str: From 67ee382c3229f10d4e29958b6a5bf257e29ed8f2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 13 Sep 2024 18:53:33 +0200 Subject: [PATCH 401/979] use default req value in config_value call --- volatility3/framework/interfaces/configuration.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index 3bb3cb019..da0a4556c 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -494,8 +494,7 @@ class SimpleTypeRequirement(RequirementInterface): """Validates the instance requirement based upon its `instance_type`.""" config_path = path_join(config_path, self.name) - - value = self.config_value(context, config_path, None) + value = self.config_value(context, config_path, self.default) if not isinstance(value, self.instance_type): vollog.log( constants.LOGLEVEL_V, @@ -536,7 +535,7 @@ class ClassRequirement(RequirementInterface): """Checks to see if a class can be recovered.""" config_path = path_join(config_path, self.name) - value = self.config_value(context, config_path, None) + value = self.config_value(context, config_path, self.default) self._cls = None if value is not None and isinstance(value, str): if "." in value: From ba0c975e73207ee6555bd80067460ddbea6426a2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 13 Sep 2024 18:50:47 -0500 Subject: [PATCH 402/979] Address all feedback --- .../framework/plugins/windows/pe_symbols.py | 126 +++++++++--------- 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 30e9b49d1..85bfb572e 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -1,11 +1,12 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +import copy import io import logging import ntpath -from typing import Dict, Tuple, Optional, List, Generator, Union +from typing import Dict, Tuple, Optional, List, Generator, Union, Callable import pefile @@ -29,11 +30,13 @@ wanted_addresses_identifier = "addresses" # how wanted modules/symbols are specified, such as: # {"ntdll.dll" : {wanted_addresses : [42, 43, 43]}} # {"ntdll.dll" : {wanted_names : ["NtCreateThread"]}} -filter_modules_type = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] +filter_module_info = Union[Dict[str, List[str]], Dict[str, List[int]]] +filter_modules_type = Dict[str, filter_module_info] # holds resolved symbols # {"ntdll.dll": [("Bob", 123), ("Alice", 456)]} -found_symbols_type = Dict[str, List[Tuple[str, int]]] +found_symbols_module = List[Tuple[str, int]] +found_symbols_type = Dict[str, found_symbols_module] # used to hold informatin about a range (VAD or kernel module) # (start address, size, file path) @@ -61,7 +64,8 @@ class PESymbolFinder: cached_int_dict = Dict[str, Optional[int]] cached_value = Union[int, str, None] - cached_value_dict = Dict[str, Union[Dict[str, List[str]], Dict[str, List[int]]]] + cached_module_lists = Union[Dict[str, List[str]], Dict[str, List[int]]] + cached_value_dict = Dict[str, cached_module_lists] def __init__( self, @@ -402,11 +406,11 @@ class PESymbols(interfaces.plugins.PluginInterface): context, layer_name, symbol_table_name, symbols ) - found_symbols = PESymbols.find_symbols( + found_symbols, missing_symbols = PESymbols.find_symbols( context, config_path, symbols, collected_modules ) - for mod_name, unresolved_symbols in symbols.items(): + for mod_name, unresolved_symbols in missing_symbols.items(): for symbol in unresolved_symbols: vollog.debug(f"Unable to resolve symbol {symbol} in module {mod_name}") @@ -449,7 +453,7 @@ class PESymbols(interfaces.plugins.PluginInterface): filename: {wanted_addresses_identifier: [address]} } - found_symbols = PESymbols.find_symbols( + found_symbols, _missing_msybols = PESymbols.find_symbols( context, config_path, filter_module, collected_modules ) @@ -624,61 +628,46 @@ class PESymbols(interfaces.plugins.PluginInterface): @staticmethod def _get_symbol_value( - wanted_modules: PESymbolFinder.cached_value_dict, - mod_name: str, + wanted_symbols: filter_module_info, symbol_resolver: PESymbolFinder, - ) -> Generator[Tuple[str, int], None, None]: + ) -> Generator[Tuple[str, int, str, int], None, None]: """ Enumerates the symbols specified as wanted by the calling plugin - removes entries from wanted_modules as they found to avoid PDB or export analysis after resolving all symbols - Args: - wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. - mod_name: the name of module to resolve symbols in + wanted_symbols: the set of symbols for a particular module + symbol_resolver: method in a layer to resolve the symbols Returns: - Tuple[str, int]: the name and address of resolved symbols + Tuple[str, int, str, int]: the index and value of the found symbol in the wanted list, and the name and address of resolved symbol """ - wanted_symbols = wanted_modules[mod_name] - if ( wanted_names_identifier not in wanted_symbols and wanted_addresses_identifier not in wanted_symbols ): vollog.warning( - f"Invalid `wanted_symbols` sent to `find_symbols` for module {mod_name}. addresses and names keys both misssing." + f"Invalid `wanted_symbols` sent to `find_symbols`. addresses and names keys both misssing." ) return - symbol_keys = [ - (wanted_names_identifier, "get_address_for_name"), - (wanted_addresses_identifier, "get_name_for_address"), + symbol_keys: List[Tuple[str, Callable]] = [ + (wanted_names_identifier, symbol_resolver.get_address_for_name), + (wanted_addresses_identifier, symbol_resolver.get_name_for_address), ] for symbol_key, symbol_getter in symbol_keys: # address or name if symbol_key in wanted_symbols: # walk each wanted address or name - for wanted_value in wanted_symbols[symbol_key]: - symbol_value = symbol_resolver.__getattribute__(symbol_getter)( - wanted_value - ) + for value_index, wanted_value in enumerate(wanted_symbols[symbol_key]): + symbol_value = symbol_getter(wanted_value) + if symbol_value: - # yield out symbol name, symbol address + # yield out deleteion key, deletion index, symbol name, symbol address if symbol_key == wanted_names_identifier: - yield wanted_value, symbol_value # type: ignore + yield symbol_key, value_index, wanted_value, symbol_value # type: ignore else: - yield symbol_value, wanted_value # type: ignore - - index = wanted_modules[mod_name][symbol_key].index(wanted_value) # type: ignore - - del wanted_modules[mod_name][symbol_key][index] - - # if all names or addresses from a module are found, delete the key - if not wanted_modules[mod_name][symbol_key]: - del wanted_modules[mod_name][symbol_key] - break + yield symbol_key, value_index, symbol_value, wanted_value # type: ignore @staticmethod def _resolve_symbols_through_methods( @@ -687,7 +676,7 @@ class PESymbols(interfaces.plugins.PluginInterface): module_instances: collected_modules_info, wanted_modules: PESymbolFinder.cached_value_dict, mod_name: str, - ) -> Generator[Tuple[str, int], None, None]: + ) -> Tuple[found_symbols_module, PESymbolFinder.cached_module_lists]: """ Attempts to resolve every wanted symbol in `mod_name` Every layer is enumerated for maximum chance of recovery @@ -697,35 +686,53 @@ class PESymbols(interfaces.plugins.PluginInterface): wanted_modules: The symbols to resolve tied to their module names mod_name: name of the module to resolve symbols in Returns: - Generator[Tuple[str, int]]: resolved symbol names and addresses + Tuple[found_symbols_module, PESymbolFinder.cached_module_lists]: The set of found symbols and the ones that could not be resolved """ symbol_resolving_methods = [ PESymbols._find_symbols_through_pdb, PESymbols._find_symbols_through_exports, ] + found: found_symbols_module = [] + + # the symbols wanted from this module by the caller + wanted = wanted_modules[mod_name] + + # make a copy to remove from inside this function for returning to the caller + remaining = copy.deepcopy(wanted) + for method in symbol_resolving_methods: + # every layer where this module was found through the given method for symbol_resolver in method( context, config_path, module_instances, mod_name ): vollog.debug(f"Have resolver for method {method}") - yield from PESymbols._get_symbol_value( - wanted_modules, mod_name, symbol_resolver - ) + for ( + symbol_key, + value_index, + symbol_name, + symbol_address, + ) in PESymbols._get_symbol_value(remaining, symbol_resolver): + found.append((symbol_name, symbol_address)) + del remaining[symbol_key][value_index] - if not wanted_modules[mod_name]: + # everything was resolved, stop this resolver + if not remaining: break - if not wanted_modules[mod_name]: + # stop all resolving + if not remaining: break + return found, remaining + @staticmethod def find_symbols( context: interfaces.context.ContextInterface, config_path: str, wanted_modules: PESymbolFinder.cached_value_dict, collected_modules: collected_modules_type, - ) -> found_symbols_type: + ) -> Tuple[found_symbols_type, PESymbolFinder.cached_value_dict]: """ Loops through each method of symbol analysis until each wanted symbol is found Returns the resolved symbols as a dictionary that includes the name and runtime address @@ -734,9 +741,10 @@ class PESymbols(interfaces.plugins.PluginInterface): wanted_modules: the dictionary of modules and symbols to resolve. Modified to remove symbols as they are resolved. collected_modules: return value from `get_kernel_modules` or `get_process_modules` Returns: - found_symbols_type: The set of symbols resolved to their name and/or address + Tuple[found_symbols_type, PESymbolFinder.cached_value_dict]: The set of found symbols but the ones that could not be resolved """ found_symbols: found_symbols_type = {} + missing_symbols: PESymbolFinder.cached_value_dict = {} for mod_name in wanted_modules: if mod_name not in collected_modules: @@ -745,24 +753,20 @@ class PESymbols(interfaces.plugins.PluginInterface): module_instances = collected_modules[mod_name] # try to resolve the symbols for `mod_name` through each method (PDB and export table currently) - for symbol_name, address in PESymbols._resolve_symbols_through_methods( + ( + found_in_module, + missing_in_module, + ) = PESymbols._resolve_symbols_through_methods( context, config_path, module_instances, wanted_modules, mod_name - ): - if mod_name not in found_symbols: - found_symbols[mod_name] = [] + ) - found_symbols[mod_name].append((symbol_name, address)) + if found_in_module: + found_symbols[mod_name] = found_in_module - # stop processing the layers (processes) if we found all the symbols for this module - if not wanted_modules[mod_name]: - break + if missing_in_module: + missing_symbols[mod_name] = missing_in_module - # stop processing this module if/when all symbols are found - if not wanted_modules[mod_name]: - del wanted_modules[mod_name] - break - - return found_symbols + return found_symbols, missing_symbols @staticmethod def get_kernel_modules( @@ -986,7 +990,7 @@ class PESymbols(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, filter_module ) - found_symbols = PESymbols.find_symbols( + found_symbols, _missing_symbols = PESymbols.find_symbols( self.context, self.config_path, filter_module, collected_modules ) From 21d21cf4b8ec3aafa5e98c563b207b21c41d5c3f Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 16:26:38 -0500 Subject: [PATCH 403/979] Break properly in all paths. Help callers to ensure always lower case module name matching. --- volatility3/framework/plugins/windows/pe_symbols.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 85bfb572e..d0ddcac57 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -380,7 +380,7 @@ class PESymbols(interfaces.plugins.PluginInterface): Returns: str: the bsae file name of the full path """ - return ntpath.basename(filepath) + return ntpath.basename(filepath).lower() @staticmethod def addresses_for_process_symbols( @@ -717,11 +717,12 @@ class PESymbols(interfaces.plugins.PluginInterface): del remaining[symbol_key][value_index] # everything was resolved, stop this resolver - if not remaining: + if not remaining[symbol_key]: break # stop all resolving - if not remaining: + if not remaining[symbol_key]: + del remaining[symbol_key] break return found, remaining From 291bc878ad771388f3514b2c66d31b7325b2bc01 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 16:31:33 -0500 Subject: [PATCH 404/979] Break in a cleaner flow --- volatility3/framework/plugins/windows/pe_symbols.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index d0ddcac57..44254513f 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -701,6 +701,8 @@ class PESymbols(interfaces.plugins.PluginInterface): # make a copy to remove from inside this function for returning to the caller remaining = copy.deepcopy(wanted) + done_processing = False + for method in symbol_resolving_methods: # every layer where this module was found through the given method for symbol_resolver in method( @@ -717,12 +719,14 @@ class PESymbols(interfaces.plugins.PluginInterface): del remaining[symbol_key][value_index] # everything was resolved, stop this resolver + # remove this key from the remaining symbols to resolve if not remaining[symbol_key]: + del remaining[symbol_key] + done_processing = True break # stop all resolving - if not remaining[symbol_key]: - del remaining[symbol_key] + if done_processing: break return found, remaining From 322f79fb5040e3e7ebdfa9de8c82a639729a39f2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 17:18:26 -0500 Subject: [PATCH 405/979] Bail as early as possible --- .../framework/plugins/windows/pe_symbols.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 44254513f..0bf03e7d6 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -437,7 +437,6 @@ class PESymbols(interfaces.plugins.PluginInterface): Returns: Tuple[str|renderers.NotApplicableValue|renderers.NotAvailableValue, str|renderers.NotApplicableValue|renderers.NotAvailableValue] """ - if not address: return renderers.NotApplicableValue(), renderers.NotApplicableValue() @@ -458,7 +457,7 @@ class PESymbols(interfaces.plugins.PluginInterface): ) if not found_symbols or filename not in found_symbols: - return renderers.NotAvailableValue(), renderers.NotAvailableValue() + return filepath, renderers.NotAvailableValue() return filepath, found_symbols[filename][0][0] @@ -718,11 +717,14 @@ class PESymbols(interfaces.plugins.PluginInterface): found.append((symbol_name, symbol_address)) del remaining[symbol_key][value_index] - # everything was resolved, stop this resolver - # remove this key from the remaining symbols to resolve - if not remaining[symbol_key]: - del remaining[symbol_key] - done_processing = True + # everything was resolved, stop this resolver + # remove this key from the remaining symbols to resolve + if not remaining[symbol_key]: + del remaining[symbol_key] + done_processing = True + break + + if done_processing: break # stop all resolving @@ -885,6 +887,7 @@ class PESymbols(interfaces.plugins.PluginInterface): for vad in vad_root.traverse(): filepath = vad.get_file_name() + if not isinstance(filepath, str) or filepath.count("\\") == 0: continue From 79b8ff7d05b56316d90be682f56db22401f2c265 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 15 Sep 2024 18:29:05 -0500 Subject: [PATCH 406/979] Address final feedback --- .../plugins/windows/debugregisters.py | 23 ++++++++----------- .../framework/plugins/windows/pe_symbols.py | 10 ++++---- .../plugins/windows/unhooked_system_calls.py | 8 +++++++ 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 65b2e625b..945ba1df0 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -148,12 +148,7 @@ class DebugRegisters(interfaces.plugins.PluginInterface): file3, sym3 = path_and_symbol(vads, dr3) # if none map to an actual file VAD then bail - if not ( - isinstance(file0, str) - or isinstance(file1, str) - or isinstance(file2, str) - or isinstance(file3, str) - ): + if not (file0 or file1 or file2 or file3): continue process_name = owner_proc.ImageFileName.cast( @@ -173,17 +168,17 @@ class DebugRegisters(interfaces.plugins.PluginInterface): thread.Tcb.State, dr7, format_hints.Hex(dr0), - file0, - sym0, + file0 or renderers.NotApplicableValue(), + sym0 or renderers.NotApplicableValue(), format_hints.Hex(dr1), - file1, - sym1, + file1 or renderers.NotApplicableValue(), + sym1 or renderers.NotApplicableValue(), format_hints.Hex(dr2), - file2, - sym2, + file2 or renderers.NotApplicableValue(), + sym2 or renderers.NotApplicableValue(), format_hints.Hex(dr3), - file3, - sym3, + file3 or renderers.NotApplicableValue(), + sym3 or renderers.NotApplicableValue(), ), ) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 0bf03e7d6..955098d6b 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -423,7 +423,7 @@ class PESymbols(interfaces.plugins.PluginInterface): collected_modules: collected_modules_type, ranges: ranges_type, address: int, - ) -> Tuple[str, str]: + ) -> Tuple[Optional[str], Optional[str]]: """ Method for plugins to determine the file path and symbol name for a given address @@ -438,12 +438,12 @@ class PESymbols(interfaces.plugins.PluginInterface): Tuple[str|renderers.NotApplicableValue|renderers.NotAvailableValue, str|renderers.NotApplicableValue|renderers.NotAvailableValue] """ if not address: - return renderers.NotApplicableValue(), renderers.NotApplicableValue() + return None, None filepath = PESymbols.filepath_for_address(ranges, address) if not filepath: - return renderers.NotAvailableValue(), renderers.NotAvailableValue() + return None, None filename = PESymbols.filename_for_path(filepath).lower() @@ -452,12 +452,12 @@ class PESymbols(interfaces.plugins.PluginInterface): filename: {wanted_addresses_identifier: [address]} } - found_symbols, _missing_msybols = PESymbols.find_symbols( + found_symbols, _missing_symbols = PESymbols.find_symbols( context, config_path, filter_module, collected_modules ) if not found_symbols or filename not in found_symbols: - return filepath, renderers.NotAvailableValue() + return filepath, None return filepath, found_symbols[filename][0][0] diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index 68f4c4b80..c3d98254d 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -71,6 +71,13 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): } } + # This data structure is used to track unique implementations of functions across processes + # The outer dictionary holds the module name (e.g., ntdll.dll) + # The next dictionary holds the function names (NtTerminateProcess, NtSetValueKey, etc.) inside a module + # The innermost dictionary holds the unique implementation (bytes) of a function across processes + # Each implementation is tracked along with the process(es) that host it + # For systems without malware, all functions should have the same implementation + # When API hooking/module unhooking is done, the victim (infected) processes will have unique implementations _code_bytes_type = Dict[str, Dict[str, Dict[bytes, List[Tuple[int, str]]]]] @classmethod @@ -127,6 +134,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: continue + # see the definition of _code_bytes_type for details of this data structure if dll_name not in code_bytes: code_bytes[dll_name] = {} From b788733683256e2cfd436750f616054f707252e9 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 15 Sep 2024 19:48:55 -0500 Subject: [PATCH 407/979] More comments on unhooked system calls --- .../plugins/windows/debugregisters.py | 4 ++++ .../plugins/windows/unhooked_system_calls.py | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/debugregisters.py b/volatility3/framework/plugins/windows/debugregisters.py index 945ba1df0..57dd1822c 100644 --- a/volatility3/framework/plugins/windows/debugregisters.py +++ b/volatility3/framework/plugins/windows/debugregisters.py @@ -1,6 +1,10 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# Full details on the techniques used in these plugins to detect EDR-evading malware +# can be found in our 20 page whitepaper submitted to DEFCON along with the presentation +# https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + import logging from typing import Tuple, Optional, Generator, List, Dict diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index c3d98254d..1a1e59940 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -1,6 +1,10 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# Full details on the techniques used in these plugins to detect EDR-evading malware +# can be found in our 20 page whitepaper submitted to DEFCON along with the presentation +# https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + import logging from typing import Dict, Tuple, List, Generator @@ -162,20 +166,30 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): # code_bytes[dll_name][func_name][func_bytes] code_bytes = self._gather_code_bytes(kernel, found_symbols) + # walk the functions that were evaluated for functions in code_bytes.values(): + # cbb is the distinct groups of bytes (instructions) + # for this function across processes for func_name, cbb in functions.items(): + # the dict key here is the raw instructions, which is not helpful to look at + # the values are the list of tuples for the (proc_id, proc_name) pairs for this set of bytes (instructions) cb = list(cbb.values()) - # same implementation in all + # if all processes map to the same implementation, then no malware is present if len(cb) == 1: yield 0, (func_name, "", len(cb[0])) else: - # find the processes that are hooked for reporting + # if there are differing implementations then it means + # that malware has overwritten system call(s) in infected processes + # max_idx and small_idx find which implementation of a system call has the least processes + # as all observed malware and open source projects only infected a few targets, leaving the + # rest with the original EDR hooks in place max_idx = 0 if len(cb[0]) > len(cb[1]) else 1 small_idx = (~max_idx) & 1 ps = [] + # gather processes on small_idx since these are the malware infected ones for pid, pname in cb[small_idx]: ps.append("{:d}:{}".format(pid, pname)) From 10ac21da2cbca02d47dc9aa938c2fe0d560af23d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 16 Sep 2024 12:16:43 +0100 Subject: [PATCH 408/979] Windows: Remove the unnecessary requirement on verinfo Fixes #1267 --- volatility3/framework/plugins/windows/verinfo.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 5b3c52bf6..57b8dcd3f 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -48,9 +48,6 @@ class VerInfo(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="modules", plugin=modules.Modules, version=(2, 0, 0) ), - requirements.VersionRequirement( - name="dlllist", component=dlllist.DllList, version=(2, 0, 0) - ), requirements.BooleanRequirement( name="extensive", description="Search physical layer for version information", From f77003b670be71e012d0caa52f936a5adba8f162 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Sep 2024 12:50:58 +1000 Subject: [PATCH 409/979] Fix changes introduced to volatility3.framework.constants in PRs #838 and #1247 --- volatility3/framework/constants/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 8743a64b0..27fae4ba1 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -134,4 +134,5 @@ def __getattr__(name): ]: warnings.warn(f"{name} is deprecated", FutureWarning) return globals()[f"{deprecated_tag}{name}"] - return None + + return getattr(__import__(__name__), name) From 09fa859a92878b5d035e0dd5c44c0521661630fc Mon Sep 17 00:00:00 2001 From: eve Date: Wed, 25 Sep 2024 09:02:51 +0100 Subject: [PATCH 410/979] Windows: change warnings around large memory maps to debug level as per issue #1256 --- volatility3/framework/plugins/windows/vadyarascan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 7bc3377c3..efcc70d07 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -18,7 +18,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 1, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -68,7 +68,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): layer = self.context.layers[layer_name] for start, size in self.get_vad_maps(task): if size > sanity_check: - vollog.warn( + vollog.debug( f"VAD at 0x{start:x} over sanity-check size, not scanning" ) continue From 7f37135739c9ff951e73680f7cdf4333b47ee231 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 25 Sep 2024 13:13:33 -0500 Subject: [PATCH 411/979] Linux: Update sockstat to render process names Currently, process names are not displayed for sockets in the sockstat plugin, making analysis more painful than it needs to be. This updates the `list_sockets` classmethod and the `generator` method to return the process name in addition to the PID. Because this is changing the public interface, this commit includes a major version bump for `linux.sockstat.Sockstat`. --- volatility3/framework/plugins/linux/sockstat.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index b0503b105..d3efc78dd 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -22,7 +22,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) def __init__(self, vmlinux, task): self._vmlinux = vmlinux @@ -507,7 +507,7 @@ class Sockstat(plugins.PluginInterface): dfop_addr = vmlinux.object_from_symbol("sockfs_dentry_operations").vol.offset fd_generator = lsof.Lsof.list_fds(context, vmlinux.name, filter_func) - for _pid, _task_comm, task, fd_fields in fd_generator: + for _pid, task_comm, task, fd_fields in fd_generator: fd_num, filp, _full_path = fd_fields if filp.f_op not in (sfop_addr, dfop_addr): @@ -548,7 +548,7 @@ class Sockstat(plugins.PluginInterface): except AttributeError: netns_id = NotAvailableValue() - yield task, netns_id, fd_num, family, sock_type, protocol, sock_fields + yield task_comm, task, netns_id, fd_num, family, sock_type, protocol, sock_fields def _format_fields(self, sock_stat, protocol): """Prepare the socket fields to be rendered @@ -595,6 +595,7 @@ class Sockstat(plugins.PluginInterface): ) for ( + task_comm, task, netns_id, fd_num, @@ -617,6 +618,7 @@ class Sockstat(plugins.PluginInterface): fields = ( netns_id, + task_comm, task.pid, fd_num, format_hints.Hex(sock.vol.offset), @@ -636,6 +638,7 @@ class Sockstat(plugins.PluginInterface): tree_grid_args = [ ("NetNS", int), + ("Process Name", str), ("Pid", int), ("FD", int), ("Sock Offset", format_hints.Hex), From 48d4048b487b665051c8f21a24ff8e77333ee682 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 27 Sep 2024 10:02:36 -0500 Subject: [PATCH 412/979] #816 - fixes and additional windows versions --- .../framework/plugins/windows/consoles.py | 291 ++++--- .../framework/plugins/windows/verinfo.py | 2 +- .../consoles-win10-17763-3232-x64.json | 71 +- .../consoles/consoles-win10-17763-x64.json | 71 +- .../consoles/consoles-win10-18362-x64.json | 73 +- .../consoles/consoles-win10-19041-x64.json | 73 +- .../consoles-win10-20348-1970-x64.json | 108 ++- .../consoles-win10-20348-2461-x64.json | 108 ++- .../consoles/consoles-win10-20348-x64.json | 108 ++- .../consoles/consoles-win10-22000-x64.json | 722 +++++++++++++++++ .../consoles-win10-22621-3672-x64.json | 722 +++++++++++++++++ .../consoles/consoles-win10-22621-x64.json | 722 +++++++++++++++++ .../consoles/consoles-win10-25398-x64.json | 723 ++++++++++++++++++ .../symbols/windows/extensions/consoles.py | 112 ++- 14 files changed, 3774 insertions(+), 132 deletions(-) create mode 100644 volatility3/framework/symbols/windows/consoles/consoles-win10-22000-x64.json create mode 100644 volatility3/framework/symbols/windows/consoles/consoles-win10-22621-3672-x64.json create mode 100644 volatility3/framework/symbols/windows/consoles/consoles-win10-22621-x64.json create mode 100644 volatility3/framework/symbols/windows/consoles/consoles-win10-25398-x64.json diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index c9e8522a4..46699aa31 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -11,12 +11,13 @@ from typing import Tuple, Generator, Set, Dict, Any, Type from volatility3.framework import interfaces, symbols, exceptions from volatility3.framework import renderers +from volatility3.framework.interfaces import configuration from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed -from volatility3.framework.symbols.windows import pdbutil, versions +from volatility3.framework.symbols.windows import pdbutil from volatility3.framework.symbols.windows.extensions import pe, consoles from volatility3.plugins.windows import pslist, vadinfo, info, verinfo from volatility3.plugins.windows.registry import hivelist @@ -49,6 +50,9 @@ class Consoles(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="verinfo", component=verinfo.VerInfo, version=(1, 0, 0) + ), requirements.VersionRequirement( name="pdbutil", component=pdbutil.PDBUtility, version=(1, 0, 0) ), @@ -92,18 +96,19 @@ class Consoles(interfaces.plugins.PluginInterface): """ for proc in proc_list: - try: - proc_id = proc.UniqueProcessId - proc_layer_name = proc.add_process_layer() + if utility.array_to_string(proc.ImageFileName).lower() == "conhost.exe": + try: + proc_id = proc.UniqueProcessId + proc_layer_name = proc.add_process_layer() - yield proc, proc_layer_name + yield proc, proc_layer_name - except exceptions.InvalidAddressException as excp: - vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) ) - ) @classmethod def find_conhostexe( @@ -122,7 +127,6 @@ class Consoles(interfaces.plugins.PluginInterface): """ for vad in conhost_proc.get_vad_root().traverse(): filename = vad.get_file_name() - if isinstance(filename, str) and filename.lower().endswith("conhost.exe"): base = vad.get_start() return base, vad.get_size() @@ -135,6 +139,9 @@ class Consoles(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, layer_name: str, nt_symbol_table: str, + config_path: str, + conhost_layer_name: str, + conhost_base: int, ) -> Tuple[str, Type]: """Tries to determine which symbol filename to use for the image's console information. This is similar to the netstat plugin. @@ -143,6 +150,9 @@ class Consoles(interfaces.plugins.PluginInterface): context: The context to retrieve required elements (layers, symbol tables) from layer_name: The name of the layer on which to operate nt_symbol_table: The name of the table containing the kernel symbols + config_path: The config path where to find symbol files + conhost_layer_name: The name of the conhot process memory layer + conhost_base: the base address of conhost.exe Returns: The filename of the symbol table to use and the associated class types. @@ -150,10 +160,6 @@ class Consoles(interfaces.plugins.PluginInterface): is_64bit = symbols.symbol_table_is_64bit(context, nt_symbol_table) - is_18363_or_later = versions.is_win10_18363_or_later( - context=context, symbol_table=nt_symbol_table - ) - if is_64bit: arch = "x64" else: @@ -211,23 +217,15 @@ class Consoles(interfaces.plugins.PluginInterface): (10, 0, 20348, 1970): "consoles-win10-20348-1970-x64", (10, 0, 20348, 2461): "consoles-win10-20348-2461-x64", (10, 0, 20348, 2520): "consoles-win10-20348-2461-x64", + (10, 0, 22000, 0): "consoles-win10-22000-x64", + (10, 0, 22621, 1): "consoles-win10-22621-x64", + (10, 0, 22621, 3672): "consoles-win10-22621-3672-x64", + (10, 0, 25398, 0): "consoles-win10-22000-x64", } # we do not need to check for conhost's specific FileVersion in every case conhost_mod_version = 0 # keep it 0 as a default - # special use cases - - # Win10_18363 is not recognized by windows.info as 18363 - # because all kernel file headers and debug structures report 18363 as - # "10.0.18362.1198" with the last part being incremented. However, we can use - # os_distinguisher to differentiate between 18362 and 18363 - if vers_minor_version == 18362 and is_18363_or_later: - vollog.debug( - "Detected 18363 data structures: working with 18363 symbol table." - ) - vers_minor_version = 18363 - # we need to define additional version numbers (which are then found via conhost.exe's FileVersion header) in case there is # ambiguity _within_ an OS version. If such a version number (last number of the tuple) is defined for the current OS # we need to inspect conhost.exe's headers to see if we can grab the precise version @@ -240,24 +238,40 @@ class Consoles(interfaces.plugins.PluginInterface): vollog.debug( "Requiring further version inspection due to OS version by checking conhost.exe's FileVersion header" ) - # the following is IntelLayer specific and might need to be adapted to other architectures. - physical_layer_name = context.layers[layer_name].config.get( - "memory_layer", None - ) - if physical_layer_name: - ver = verinfo.VerInfo.find_version_info( - context, physical_layer_name, "CONHOST.EXE" - ) - if ver: - conhost_mod_version = ver[3] - vollog.debug( - "Determined conhost.exe's FileVersion: {}".format( - conhost_mod_version - ) + pe_table_name = intermed.IntermediateSymbolTable.create( + context, + configuration.path_join(config_path, "conhost"), + "windows", + "pe", + class_types=pe.class_types + ) + + try: + (major, minor, product, build) = verinfo.VerInfo.get_version_information( + context, pe_table_name, conhost_layer_name, conhost_base + ) + conhost_mod_version = build + vollog.debug(f"Found conhost.exe version {major}.{minor}.{product}.{build} in {conhost_layer_name} at base {conhost_base:#x}") + except (exceptions.InvalidAddressException, TypeError, AttributeError): + # the following is IntelLayer specific and might need to be adapted to other architectures. + physical_layer_name = context.layers[layer_name].config.get( + "memory_layer", None + ) + if physical_layer_name: + ver = verinfo.VerInfo.find_version_info( + context, physical_layer_name, "CONHOST.EXE" ) - else: - vollog.debug("Could not determine conhost.exe's FileVersion.") + + if ver: + conhost_mod_version = ver[3] + vollog.debug( + "Determined conhost.exe's FileVersion: {}".format( + conhost_mod_version + ) + ) + else: + vollog.debug("Could not determine conhost.exe's FileVersion.") else: vollog.debug( "Unable to retrieve physical memory layer, skipping FileVersion check." @@ -287,6 +301,7 @@ class Consoles(interfaces.plugins.PluginInterface): for nt_maj, nt_min, vers_min, conhost_ver in version_dict if nt_maj == nt_major_version and nt_min == nt_minor_version + and vers_min <= vers_minor_version and conhost_ver <= conhost_mod_version ] current_versions.sort() @@ -321,6 +336,8 @@ class Consoles(interfaces.plugins.PluginInterface): layer_name: str, nt_symbol_table: str, config_path: str, + conhost_layer_name: str, + conhost_base: int, ) -> str: """Creates a symbol table for conhost structures. @@ -339,13 +356,16 @@ class Consoles(interfaces.plugins.PluginInterface): context, layer_name, nt_symbol_table, + config_path, + conhost_layer_name, + conhost_base, ) vollog.debug(f"Using symbol file '{symbol_filename}' and types {class_types}") return intermed.IntermediateSymbolTable.create( context, - config_path, + configuration.path_join(config_path, "conhost"), os.path.join("windows", "consoles"), symbol_filename, class_types=class_types, @@ -383,9 +403,7 @@ class Consoles(interfaces.plugins.PluginInterface): that console information structure. """ - conhost_symbol_table = cls.create_conhost_symbol_table( - context, kernel_layer_name, kernel_table_name, config_path - ) + conhost_symbol_table = None for conhost_proc, proc_layer_name in cls.find_conhost_proc(procs): if not conhost_proc: @@ -407,6 +425,16 @@ class Consoles(interfaces.plugins.PluginInterface): proc_layer = context.layers[proc_layer_name] + if conhost_symbol_table is None: + conhost_symbol_table = cls.create_conhost_symbol_table( + context, + kernel_layer_name, + kernel_table_name, + config_path, + proc_layer_name, + conhostexe_base + ) + conhost_module = context.module( conhost_symbol_table, proc_layer_name, offset=conhostexe_base ) @@ -560,6 +588,60 @@ class Consoles(interfaces.plugins.PluginInterface): } ) + vollog.debug( + f"Getting ExeAliasList entries for {console_info.ExeAliasList}" + ) + console_properties.append( + { + "level": 1, + "name": "_CONSOLE_INFORMATION.ExeAliasList", + "address": console_info.ExeAliasList.vol.offset, + "data": "", + } + ) + if console_info.ExeAliasList: + for index, exe_alias_list in enumerate( + console_info.get_exe_aliases() + ): + try: + console_properties.append( + { + "level": 2, + "name": f"_CONSOLE_INFORMATION.ExeAliasList.AliasList_{index}", + "address": exe_alias_list.vol.offset, + "data": "", + } + ) + console_properties.append( + { + "level": 2, + "name": f"_CONSOLE_INFORMATION.ExeAliasList.AliasList_{index}.ExeName", + "address": exe_alias_list.ExeName.vol.offset, + "data": exe_alias_list.get_exename(), + } + ) + for alias_index, alias in enumerate(exe_alias_list.get_aliases()): + console_properties.append( + { + "level": 3, + "name": f"_CONSOLE_INFORMATION.ExeAliasList.AliasList_{index}.Alias_{alias_index}.Source", + "address": alias.Source.vol.offset, + "data": alias.get_source(), + } + ) + console_properties.append( + { + "level": 3, + "name": f"_CONSOLE_INFORMATION.ExeAliasList.AliasList_{index}.Alias_{alias_index}.Target", + "address": alias.Target.vol.offset, + "data": alias.get_target(), + } + ) + except Exception as e: + vollog.debug( + f"reading {exe_alias_list} encountered exception {e}" + ) + vollog.debug( f"Getting HistoryList entries for {console_info.HistoryList}" ) @@ -639,60 +721,66 @@ class Consoles(interfaces.plugins.PluginInterface): f"reading {command_history} encountered exception {e}" ) - vollog.debug(f"Getting ScreenBuffer entries for {console_info}") - console_properties.append( - { - "level": 1, - "name": "_CONSOLE_INFORMATION.CurrentScreenBuffer", - "address": console_info.CurrentScreenBuffer.vol.offset, - "data": "", - } - ) - for screen_index, screen_info in enumerate( - console_info.get_screens() - ): - try: - console_properties.append( - { - "level": 2, - "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}", - "address": screen_info, - "data": "", - } - ) - console_properties.append( - { - "level": 2, - "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.ScreenX", - "address": None, - "data": screen_info.ScreenX, - } - ) - console_properties.append( - { - "level": 2, - "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.ScreenY", - "address": None, - "data": screen_info.ScreenY, - } - ) - console_properties.append( - { - "level": 2, - "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.Dump", - "address": None, - "data": "\n".join(screen_info.get_buffer()), - } - ) - except Exception as e: - vollog.debug( - f"reading {screen_info} encountered exception {e}" - ) + try: + vollog.debug(f"Getting ScreenBuffer entries for {console_info}") + console_properties.append( + { + "level": 1, + "name": "_CONSOLE_INFORMATION.CurrentScreenBuffer", + "address": console_info.CurrentScreenBuffer.vol.offset, + "data": "", + } + ) + for screen_index, screen_info in enumerate( + console_info.get_screens() + ): + try: + console_properties.append( + { + "level": 2, + "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}", + "address": screen_info, + "data": "", + } + ) + console_properties.append( + { + "level": 2, + "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.ScreenX", + "address": None, + "data": screen_info.ScreenX, + } + ) + console_properties.append( + { + "level": 2, + "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.ScreenY", + "address": None, + "data": screen_info.ScreenY, + } + ) + console_properties.append( + { + "level": 2, + "name": f"_CONSOLE_INFORMATION.ScreenBuffer_{screen_index}.Dump", + "address": None, + "data": "\n".join(screen_info.get_buffer()), + } + ) + except Exception as e: + vollog.debug( + f"reading {screen_info} encountered exception {e}" + ) + except Exception as e: + vollog.debug( + f"reading _CONSOLE_INFORMATION.CurrentScreenBuffer encountered exception {e}" + ) except exceptions.PagedInvalidAddressException as exp: vollog.debug( f"Required memory at {exp.invalid_address:#x} is not valid" ) + continue yield conhost_proc, console_info, console_properties @@ -776,6 +864,7 @@ class Consoles(interfaces.plugins.PluginInterface): vollog.debug(f"Possible CommandHistorySize values: {max_history}") vollog.debug(f"Possible HistoryBufferMax values: {max_buffers}") + proc = None for proc, console_info, console_properties in self.get_console_info( self.context, kernel.layer_name, @@ -786,13 +875,14 @@ class Consoles(interfaces.plugins.PluginInterface): max_buffers, ): process_name = utility.array_to_string(proc.ImageFileName) + process_pid = proc.UniqueProcessId if console_info and console_properties: for console_property in console_properties: yield ( console_property["level"], ( - proc.UniqueProcessId, + process_pid, process_name, format_hints.Hex(console_info.vol.offset), console_property["name"], @@ -804,6 +894,11 @@ class Consoles(interfaces.plugins.PluginInterface): str(console_property["data"]), ), ) + else: + vollog.warn(f"_CONSOLE_INFORMATION not found for {process_name} with pid {process_pid}.") + + if proc is None: + vollog.warn("No conhost.exe processes found.") def _conhost_proc_filter(self, proc): """ @@ -811,7 +906,7 @@ class Consoles(interfaces.plugins.PluginInterface): """ process_name = utility.array_to_string(proc.ImageFileName) - return process_name != "conhost.exe" + return process_name.lower() != "conhost.exe" def run(self): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 57b8dcd3f..4a06ed0c9 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -13,7 +13,7 @@ from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import pslist, modules, dlllist +from volatility3.plugins.windows import pslist, modules vollog = logging.getLogger(__name__) diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json index 0a8cb5782..413b6f466 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json @@ -152,8 +152,8 @@ }, "ExeAliasList": { "type": { - "kind": "base", - "name": "unsigned short" + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" }, "offset": 1392 } @@ -562,7 +562,8 @@ } }, "offset": 88 - } }, + } + }, "kind": "struct", "size": 96 }, @@ -581,6 +582,70 @@ }, "kind": "struct", "size": 8 + }, + "_EXE_ALIAS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ExeLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 8 + }, + "ExeName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 24 + }, + "AliasList": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 48 + }, + "_ALIAS": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Source": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "Target": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json index 2b98945c9..b3bf9666c 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json @@ -152,8 +152,8 @@ }, "ExeAliasList": { "type": { - "kind": "base", - "name": "unsigned short" + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" }, "offset": 1392 } @@ -562,7 +562,8 @@ } }, "offset": 88 - } }, + } + }, "kind": "struct", "size": 96 }, @@ -581,6 +582,70 @@ }, "kind": "struct", "size": 8 + }, + "_EXE_ALIAS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ExeLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 16 + }, + "ExeName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 24 + }, + "AliasList": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 48 + }, + "_ALIAS": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Source": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "Target": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json index 3d76d1cb4..d962418d3 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json @@ -152,10 +152,10 @@ }, "ExeAliasList": { "type": { - "kind": "base", - "name": "unsigned short" + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" }, - "offset": 1436 + "offset": -856 } }, "kind": "struct", @@ -552,7 +552,8 @@ } }, "offset": 88 - } }, + } + }, "kind": "struct", "size": 96 }, @@ -571,6 +572,70 @@ }, "kind": "struct", "size": 8 + }, + "_EXE_ALIAS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ExeLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 8 + }, + "ExeName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 24 + }, + "AliasList": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 48 + }, + "_ALIAS": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Source": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "Target": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json index 681113b0f..0963afee3 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json @@ -1,5 +1,5 @@ { - "symbols": {}, + "symbols": {}, "enums": {}, "base_types": { "unsigned long": { @@ -152,8 +152,8 @@ }, "ExeAliasList": { "type": { - "kind": "base", - "name": "unsigned short" + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" }, "offset": 1436 } @@ -552,7 +552,8 @@ } }, "offset": 88 - } }, + } + }, "kind": "struct", "size": 96 }, @@ -571,6 +572,70 @@ }, "kind": "struct", "size": 8 + }, + "_EXE_ALIAS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ExeLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 8 + }, + "ExeName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 24 + }, + "AliasList": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 48 + }, + "_ALIAS": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Source": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "Target": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json index 7bc6ae9d1..cee78e2b5 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json @@ -152,8 +152,11 @@ }, "ExeAliasList": { "type": { - "kind": "base", - "name": "unsigned short" + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } }, "offset": 2410 } @@ -562,7 +565,8 @@ } }, "offset": 88 - } }, + } + }, "kind": "struct", "size": 96 }, @@ -581,6 +585,104 @@ }, "kind": "struct", "size": 8 + }, + "_EXE_ALIAS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ExeName": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "AliasList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 56 + } + }, + "kind": "struct", + "size": 64 + }, + "_ALIAS": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Source": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "Target": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 32 + }, + "_ALIAS_STRING": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json index f612c527a..59e9fd3b9 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json @@ -152,8 +152,11 @@ }, "ExeAliasList": { "type": { - "kind": "base", - "name": "unsigned short" + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } }, "offset": 9232 } @@ -562,7 +565,8 @@ } }, "offset": 88 - } }, + } + }, "kind": "struct", "size": 96 }, @@ -581,6 +585,104 @@ }, "kind": "struct", "size": 8 + }, + "_EXE_ALIAS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ExeName": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "AliasList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 56 + } + }, + "kind": "struct", + "size": 64 + }, + "_ALIAS": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Source": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "Target": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 32 + }, + "_ALIAS_STRING": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json index 083a966a0..df02169cb 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json @@ -152,8 +152,11 @@ }, "ExeAliasList": { "type": { - "kind": "base", - "name": "unsigned short" + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } }, "offset": -376 } @@ -562,7 +565,8 @@ } }, "offset": 88 - } }, + } + }, "kind": "struct", "size": 96 }, @@ -581,6 +585,104 @@ }, "kind": "struct", "size": 8 + }, + "_EXE_ALIAS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ExeName": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "AliasList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 56 + } + }, + "kind": "struct", + "size": 64 + }, + "_ALIAS": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Source": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "Target": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 32 + }, + "_ALIAS_STRING": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-22000-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-22000-x64.json new file mode 100644 index 000000000..0dfaf34af --- /dev/null +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-22000-x64.json @@ -0,0 +1,722 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "short": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_CONSOLE_INFORMATION": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 24 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 26 + }, + "CommandHistorySize": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 136 + }, + "HistoryBufferMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 140 + }, + "OriginalTitle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1648 + }, + "Title": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 1616 + }, + "GetScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1664 + }, + "CurrentScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 1296 + }, + "ConsoleProcessList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 1272 + }, + "ProcessCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 1280 + }, + "HistoryList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": -920 + }, + "HistoryBufferCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": -912 + }, + "ExeAliasList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": -1008 + } + }, + "kind": "struct", + "size": 140 + }, + "_VECTOR": { + "fields": { + "Begin": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 0 + }, + "End": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 8 + }, + "EndCapacity": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + }, + "_CONSOLE_PROCESS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ConsoleProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_CONSOLE_PROCESS_HANDLE": { + "fields": { + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 52 + }, + "_CONSOLE_PROCESS": { + "fields": { + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 28 + }, + "ThreadId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + }, + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND_HISTORY": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "CommandBucket": { + "type": { + "kind": "struct", + "name": "_VECTOR" + }, + "offset": 16 + }, + "CommandCountMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 40 + }, + "Application": { + "type": { + "kind": "struct", + "name": "_COMMAND" + }, + "offset": 48 + }, + "ConsoleProcessHandle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS_HANDLE" + } + }, + "offset": 80 + }, + "Flags": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 88 + }, + "LastDisplayed": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 92 + } + }, + "kind": "struct", + "size": 96 + }, + "_SCREEN_INFORMATION": { + "fields": { + "TextBufferInfo": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 56 + }, + "Next": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 64 + } + }, + "kind": "struct", + "size": 72 + }, + "_ROW_POINTER": { + "fields": { + "Row": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROWS_ARRAY": { + "fields": { + "Rows": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_TEXT_BUFFER_INFO": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 4 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 6 + }, + "BufferRows": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROWS_ARRAY" + } + }, + "offset": 8 + }, + "BufferCapacity": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 6 + }, + "ThisBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 32 + }, + "FirstRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 8 + }, + "LastRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 16 + }, + "BufferStart": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 80 + } + }, + "kind": "struct", + "size": 72 + }, + "_CHAR_ROW_CELL": { + "fields": { + "Text": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "DbcsAttribute": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 2 + } + }, + "kind": "struct", + "size": 3 + }, + "_CHAR_ROW_CELL_ARRAY": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROW": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": -88 + }, + "CharRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL_ARRAY" + } + }, + "offset": 0 + }, + "RowLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": -18 + }, + "Index": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": -20 + }, + "RowLength2": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 8 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 16 + }, + "TextBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": -8 + } + }, + "kind": "struct", + "size": 480 + }, + "_DEQUE": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_EXE_ALIAS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ExeName": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "AliasList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 56 + } + }, + "kind": "struct", + "size": 64 + }, + "_ALIAS": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Source": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "Target": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 32 + }, + "_ALIAS_STRING": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle", + "datetime": "2024-07-31T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-3672-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-3672-x64.json new file mode 100644 index 000000000..4c71ee32e --- /dev/null +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-3672-x64.json @@ -0,0 +1,722 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "short": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_CONSOLE_INFORMATION": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 2400 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 2402 + }, + "CommandHistorySize": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 2512 + }, + "HistoryBufferMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 2516 + }, + "OriginalTitle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 2944 + }, + "Title": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 2912 + }, + "GetScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 3008 + }, + "CurrentScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 2632 + }, + "ConsoleProcessList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 2608 + }, + "ProcessCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 2616 + }, + "HistoryList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 10640 + }, + "HistoryBufferCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 10648 + }, + "ExeAliasList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 10552 + } + }, + "kind": "struct", + "size": 140 + }, + "_VECTOR": { + "fields": { + "Begin": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 0 + }, + "End": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 8 + }, + "EndCapacity": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + }, + "_CONSOLE_PROCESS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ConsoleProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_CONSOLE_PROCESS_HANDLE": { + "fields": { + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 52 + }, + "_CONSOLE_PROCESS": { + "fields": { + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 28 + }, + "ThreadId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + }, + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND_HISTORY": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "CommandBucket": { + "type": { + "kind": "struct", + "name": "_VECTOR" + }, + "offset": 16 + }, + "CommandCountMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 40 + }, + "Application": { + "type": { + "kind": "struct", + "name": "_COMMAND" + }, + "offset": 48 + }, + "ConsoleProcessHandle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS_HANDLE" + } + }, + "offset": 80 + }, + "Flags": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 88 + }, + "LastDisplayed": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 92 + } + }, + "kind": "struct", + "size": 96 + }, + "_SCREEN_INFORMATION": { + "fields": { + "TextBufferInfo": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 56 + }, + "Next": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 64 + } + }, + "kind": "struct", + "size": 72 + }, + "_ROW_POINTER": { + "fields": { + "Row": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROWS_ARRAY": { + "fields": { + "Rows": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_TEXT_BUFFER_INFO": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 4 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 6 + }, + "BufferRows": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROWS_ARRAY" + } + }, + "offset": 8 + }, + "BufferCapacity": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 6 + }, + "ThisBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 32 + }, + "FirstRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 8 + }, + "LastRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 16 + }, + "BufferStart": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 72 + } + }, + "kind": "struct", + "size": 72 + }, + "_CHAR_ROW_CELL": { + "fields": { + "Text": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "DbcsAttribute": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 2 + } + }, + "kind": "struct", + "size": 3 + }, + "_CHAR_ROW_CELL_ARRAY": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROW": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": -80 + }, + "CharRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL_ARRAY" + } + }, + "offset": 0 + }, + "RowLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 8 + }, + "Index": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": -20 + }, + "RowLength2": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 8 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 16 + }, + "TextBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": -16 + } + }, + "kind": "struct", + "size": 472 + }, + "_DEQUE": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_EXE_ALIAS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ExeName": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "AliasList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 56 + } + }, + "kind": "struct", + "size": 64 + }, + "_ALIAS": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Source": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "Target": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 32 + }, + "_ALIAS_STRING": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle", + "datetime": "2024-07-31T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-x64.json new file mode 100644 index 000000000..d6da72ef2 --- /dev/null +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-x64.json @@ -0,0 +1,722 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "short": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_CONSOLE_INFORMATION": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 2400 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 2402 + }, + "CommandHistorySize": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 2512 + }, + "HistoryBufferMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 2516 + }, + "OriginalTitle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 2944 + }, + "Title": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 2912 + }, + "GetScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 3008 + }, + "CurrentScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 2632 + }, + "ConsoleProcessList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 2608 + }, + "ProcessCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 2616 + }, + "HistoryList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 10664 + }, + "HistoryBufferCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 10672 + }, + "ExeAliasList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 10576 + } + }, + "kind": "struct", + "size": 140 + }, + "_VECTOR": { + "fields": { + "Begin": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 0 + }, + "End": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 8 + }, + "EndCapacity": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + }, + "_CONSOLE_PROCESS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ConsoleProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_CONSOLE_PROCESS_HANDLE": { + "fields": { + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 52 + }, + "_CONSOLE_PROCESS": { + "fields": { + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 28 + }, + "ThreadId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + }, + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND_HISTORY": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "CommandBucket": { + "type": { + "kind": "struct", + "name": "_VECTOR" + }, + "offset": 16 + }, + "CommandCountMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 40 + }, + "Application": { + "type": { + "kind": "struct", + "name": "_COMMAND" + }, + "offset": 48 + }, + "ConsoleProcessHandle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS_HANDLE" + } + }, + "offset": 80 + }, + "Flags": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 88 + }, + "LastDisplayed": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 92 + } + }, + "kind": "struct", + "size": 96 + }, + "_SCREEN_INFORMATION": { + "fields": { + "TextBufferInfo": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 56 + }, + "Next": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 64 + } + }, + "kind": "struct", + "size": 72 + }, + "_ROW_POINTER": { + "fields": { + "Row": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROWS_ARRAY": { + "fields": { + "Rows": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_TEXT_BUFFER_INFO": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 4 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 6 + }, + "BufferRows": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROWS_ARRAY" + } + }, + "offset": 8 + }, + "BufferCapacity": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 6 + }, + "ThisBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 32 + }, + "FirstRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 8 + }, + "LastRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 16 + }, + "BufferStart": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 72 + } + }, + "kind": "struct", + "size": 72 + }, + "_CHAR_ROW_CELL": { + "fields": { + "Text": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "DbcsAttribute": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 2 + } + }, + "kind": "struct", + "size": 3 + }, + "_CHAR_ROW_CELL_ARRAY": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROW": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": -96 + }, + "CharRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL_ARRAY" + } + }, + "offset": 0 + }, + "RowLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 8 + }, + "Index": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": -20 + }, + "RowLength2": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 8 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 16 + }, + "TextBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": -8 + } + }, + "kind": "struct", + "size": 480 + }, + "_DEQUE": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_EXE_ALIAS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ExeName": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "AliasList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 56 + } + }, + "kind": "struct", + "size": 64 + }, + "_ALIAS": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Source": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "Target": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 32 + }, + "_ALIAS_STRING": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle", + "datetime": "2024-07-31T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-25398-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-25398-x64.json new file mode 100644 index 000000000..120ad85ec --- /dev/null +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-25398-x64.json @@ -0,0 +1,723 @@ +{ + "symbols": {}, + "enums": {}, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "short": { + "kind": "int", + "size": 2, + "signed": true, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_CONSOLE_INFORMATION": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 2592 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 2594 + }, + "CommandHistorySize": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 2704 + }, + "HistoryBufferMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 2708 + }, + "OriginalTitle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 3056 + }, + "Title": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 3120 + }, + "GetScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 3216 + }, + "CurrentScreenBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 2824 + }, + "ConsoleProcessList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 2800 + }, + "ProcessCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 2808 + }, + "HistoryList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": -360 + }, + "HistoryBufferCount": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": -352 + }, + "ExeAliasList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 3776 + } + }, + "kind": "struct", + "size": 140 + }, + "_VECTOR": { + "fields": { + "Begin": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 0 + }, + "End": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_COMMAND" + } + }, + "offset": 8 + }, + "EndCapacity": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + }, + "_CONSOLE_PROCESS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ConsoleProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 24 + }, + "_CONSOLE_PROCESS_HANDLE": { + "fields": { + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 52 + }, + "_CONSOLE_PROCESS": { + "fields": { + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 28 + }, + "ThreadId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + }, + "ProcessHandle": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 24 + }, + "_COMMAND_HISTORY": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "CommandBucket": { + "type": { + "kind": "struct", + "name": "_VECTOR" + }, + "offset": 16 + }, + "CommandCountMax": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 40 + }, + "Application": { + "type": { + "kind": "struct", + "name": "_COMMAND" + }, + "offset": 48 + }, + "ConsoleProcessHandle": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CONSOLE_PROCESS_HANDLE" + } + }, + "offset": 80 + }, + "Flags": { + "type": { + "kind": "base", + "name": "unsigned short" + }, + "offset": 88 + }, + "LastDisplayed": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 92 + } + }, + "kind": "struct", + "size": 96 + }, + "_SCREEN_INFORMATION": { + "fields": { + "TextBufferInfo": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 56 + }, + "Next": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SCREEN_INFORMATION" + } + }, + "offset": 64 + } + }, + "kind": "struct", + "size": 72 + }, + "_ROW_POINTER": { + "fields": { + "Row": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + + }, + "_ROWS_ARRAY": { + "fields": { + "Rows": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_ROW_POINTER" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_TEXT_BUFFER_INFO": { + "fields": { + "ScreenX": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 8 + }, + "ScreenY": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 10 + }, + "BufferRows": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROWS_ARRAY" + } + }, + "offset": 16 + }, + "BufferCapacity": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 10 + }, + "ThisBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": 40 + }, + "FirstRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 16 + }, + "LastRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": 24 + }, + "BufferStart": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 88 + } + }, + "kind": "struct", + "size": 72 + }, + "_CHAR_ROW_CELL": { + "fields": { + "Text": { + "type": { + "kind": "struct", + "name": "nt_symbols!_UNICODE_STRING" + }, + "offset": 0 + }, + "DbcsAttribute": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 2 + } + }, + "kind": "struct", + "size": 3 + }, + "_CHAR_ROW_CELL_ARRAY": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_ROW": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ROW" + } + }, + "offset": -88 + }, + "CharRow": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_CHAR_ROW_CELL_ARRAY" + } + }, + "offset": 0 + }, + "RowLength": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 8 + }, + "Index": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": -20 + }, + "RowLength2": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 8 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "short" + }, + "offset": 16 + }, + "TextBuffer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TEXT_BUFFER_INFO" + } + }, + "offset": -8 + } + }, + "kind": "struct", + "size": 464 + }, + "_DEQUE": { + "fields": { + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_EXE_ALIAS_LIST": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "ExeName": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "AliasList": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "offset": 56 + } + }, + "kind": "struct", + "size": 64 + }, + "_ALIAS": { + "fields": { + "ListEntry": { + "type": { + "kind": "struct", + "name": "nt_symbols!_LIST_ENTRY" + }, + "offset": 0 + }, + "Source": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 16 + }, + "Target": { + "type": { + "kind": "struct", + "name": "_ALIAS_STRING" + }, + "offset": 48 + } + }, + "kind": "struct", + "size": 32 + }, + "_ALIAS_STRING": { + "fields": { + "Chars": { + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + }, + "offset": 0 + }, + "Pointer": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "string" + } + }, + "offset": 0 + }, + "Length": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 16 + }, + "Allocated": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 24 + } + }, + "kind": "struct", + "size": 32 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "Dave Lassalle", + "datetime": "2024-07-31T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 23670bb9b..70ef15093 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -9,30 +9,42 @@ from volatility3.framework import constants class ROW(objects.StructType): """A Row Structure.""" - def _valid_dbcs(self, c): + def _valid_dbcs(self, c, n): # TODO this need more research and testing # https://github.com/search?q=repo%3Amicrosoft%2Fterminal+DbcsAttr&type=code - valid = c in ( + valid = n == 0 and c in ( 0x0, 0x1, 0x2, + 0x8, + 0x10, + 0x18, 0x20, 0x28, 0x30, 0x48, 0x50, + 0x58, 0x60, + 0x68, + 0x70, + 0x78, 0x80, + 0x88, 0xA8, + 0xB8, 0xC0, 0xC8, 0x98, + 0xD8, + 0xE0, + 0xE8, 0xF8, 0xF0, 0xA0, ) - # if not valid: - # print("Bad Dbcs Attribute {}".format(hex(c))) + if n == 0 and not valid: + print("Bad Dbcs Attribute {}".format(hex(c))) return valid def get_text(self, truncate=True): @@ -50,10 +62,9 @@ class ROW(objects.StructType): try: if char_row: line = "".join( - # "{} {} =".format(char_row[i:i + 2].decode('utf-16le', errors='replace'), char_row[i+2]) if self._valid_dbcs(char_row[i+2]) else "" ( char_row[i : i + 2].decode("utf-16le", errors="replace") - if self._valid_dbcs(char_row[i + 2]) + if self._valid_dbcs(char_row[i + 2], char_row[i+1]) else "" ) for i in range(0, len(char_row), 3) @@ -68,12 +79,78 @@ class ROW(objects.StructType): return line +class ALIAS(objects.StructType): + """An Alias Structure""" + + def get_source(self): + if self.Source.Length < 8: + return self.Source.Chars.cast( + "string", + encoding="utf-16", + errors="replace", + max_length=self.Source.Length * 2, + ) + elif self.Source.Length < 1024: + return self.Source.Pointer.dereference().cast( + "string", encoding="utf-16", errors="replace", max_length=512 + ) + + def get_target(self): + if self.Target.Length < 8: + return self.Target.Chars.cast( + "string", + encoding="utf-16", + errors="replace", + max_length=self.Target.Length * 2, + ) + elif self.Target.Length < 1024: + return self.Target.Pointer.dereference().cast( + "string", encoding="utf-16", errors="replace", max_length=512 + ) + +class EXE_ALIAS_LIST(objects.StructType): + """An Exe Alias List Structure""" + + def get_exename(self): + exe_name = self.ExeName + # Windows 10 22000 and Server 20348 removed the Pointer + if isinstance(exe_name, objects.Pointer): + exe_name = exe_name.dereference() + return exe_name.get_string() + + if self.ExeName.Length < 8: + return self.ExeName.Chars.cast( + "string", + encoding="utf-16", + errors="replace", + max_length=self.ExeName.Length * 2, + ) + elif self.ExeName.Length < 1024: + return self.ExeName.Pointer.dereference().cast( + "string", encoding="utf-16", errors="replace", max_length=512 + ) + + def get_aliases(self): + """Generator for the individual aliases for a + particular executable.""" + for alias in self.AliasList.to_list( + f"{self.get_symbol_table_name()}{constants.BANG}_ALIAS", + "ListEntry", + ): + yield alias + + class SCREEN_INFORMATION(objects.StructType): """A Screen Information Structure.""" @property def ScreenX(self): - return self.TextBufferInfo.BufferRows.Rows[0].Row.RowLength2 + # 22000 change from an array of pointers to _ROW to an array of _ROW + row = self.TextBufferInfo.BufferRows.Rows[0] + if hasattr(row, "Row"): + return row.Row.RowLength2 + else: + return row.RowLength2 @property def ScreenY(self): @@ -127,7 +204,9 @@ class SCREEN_INFORMATION(objects.StructType): for i in range(capacity): index = (start + i) % capacity - row = buffer_rows.Rows[index].Row + row = buffer_rows.Rows[index] + if hasattr(row, "Row"): + row = row.Row try: text = row.get_text(truncate_lines) rows.append(text) @@ -138,9 +217,9 @@ class SCREEN_INFORMATION(objects.StructType): rows = self._truncate_rows(rows) if rows: - rows = ["=== Start of buffer ==="] + rows + ["=== End of buffer ==="] + rows = ["=== START OF BUFFER ==="] + rows + ["=== END OF BUFFER ==="] else: - rows = ["=== No buffer data found ==="] + rows = ["=== NO BUFFER DATA FOUND ==="] return rows @@ -198,6 +277,17 @@ class CONSOLE_INFORMATION(objects.StructType): ): yield cmd_hist + def get_exe_aliases(self): + exe_alias_list = self.ExeAliasList + # Windows 10 22000 and Server 20348 made this a Pointer + if isinstance(exe_alias_list, objects.Pointer): + exe_alias_list = exe_alias_list.dereference() + for exe_alias_list_item in exe_alias_list.to_list( + f"{self.get_symbol_table_name()}{constants.BANG}_EXE_ALIAS_LIST", + "ListEntry" + ): + yield exe_alias_list_item + def get_processes(self): for proc in self.ConsoleProcessList.dereference().to_list( f"{self.get_symbol_table_name()}{constants.BANG}_CONSOLE_PROCESS_LIST", @@ -331,6 +421,8 @@ class COMMAND_HISTORY(objects.StructType): win10_x64_class_types = { + "_EXE_ALIAS_LIST": EXE_ALIAS_LIST, + "_ALIAS": ALIAS, "_ROW": ROW, "_SCREEN_INFORMATION": SCREEN_INFORMATION, "_CONSOLE_INFORMATION": CONSOLE_INFORMATION, From 9d98ab9b6548cfc5b36c3e6ade3425ea87498442 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 27 Sep 2024 10:39:57 -0500 Subject: [PATCH 413/979] #816 - black fixes --- .../framework/plugins/windows/consoles.py | 26 +++++++++++++------ .../symbols/windows/extensions/consoles.py | 5 ++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index 46699aa31..89f9fcc80 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -244,15 +244,19 @@ class Consoles(interfaces.plugins.PluginInterface): configuration.path_join(config_path, "conhost"), "windows", "pe", - class_types=pe.class_types + class_types=pe.class_types, ) try: - (major, minor, product, build) = verinfo.VerInfo.get_version_information( - context, pe_table_name, conhost_layer_name, conhost_base + (major, minor, product, build) = ( + verinfo.VerInfo.get_version_information( + context, pe_table_name, conhost_layer_name, conhost_base + ) ) conhost_mod_version = build - vollog.debug(f"Found conhost.exe version {major}.{minor}.{product}.{build} in {conhost_layer_name} at base {conhost_base:#x}") + vollog.debug( + f"Found conhost.exe version {major}.{minor}.{product}.{build} in {conhost_layer_name} at base {conhost_base:#x}" + ) except (exceptions.InvalidAddressException, TypeError, AttributeError): # the following is IntelLayer specific and might need to be adapted to other architectures. physical_layer_name = context.layers[layer_name].config.get( @@ -432,7 +436,7 @@ class Consoles(interfaces.plugins.PluginInterface): kernel_table_name, config_path, proc_layer_name, - conhostexe_base + conhostexe_base, ) conhost_module = context.module( @@ -620,7 +624,9 @@ class Consoles(interfaces.plugins.PluginInterface): "data": exe_alias_list.get_exename(), } ) - for alias_index, alias in enumerate(exe_alias_list.get_aliases()): + for alias_index, alias in enumerate( + exe_alias_list.get_aliases() + ): console_properties.append( { "level": 3, @@ -722,7 +728,9 @@ class Consoles(interfaces.plugins.PluginInterface): ) try: - vollog.debug(f"Getting ScreenBuffer entries for {console_info}") + vollog.debug( + f"Getting ScreenBuffer entries for {console_info}" + ) console_properties.append( { "level": 1, @@ -895,7 +903,9 @@ class Consoles(interfaces.plugins.PluginInterface): ), ) else: - vollog.warn(f"_CONSOLE_INFORMATION not found for {process_name} with pid {process_pid}.") + vollog.warn( + f"_CONSOLE_INFORMATION not found for {process_name} with pid {process_pid}." + ) if proc is None: vollog.warn("No conhost.exe processes found.") diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 70ef15093..555197fc4 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -64,7 +64,7 @@ class ROW(objects.StructType): line = "".join( ( char_row[i : i + 2].decode("utf-16le", errors="replace") - if self._valid_dbcs(char_row[i + 2], char_row[i+1]) + if self._valid_dbcs(char_row[i + 2], char_row[i + 1]) else "" ) for i in range(0, len(char_row), 3) @@ -108,6 +108,7 @@ class ALIAS(objects.StructType): "string", encoding="utf-16", errors="replace", max_length=512 ) + class EXE_ALIAS_LIST(objects.StructType): """An Exe Alias List Structure""" @@ -284,7 +285,7 @@ class CONSOLE_INFORMATION(objects.StructType): exe_alias_list = exe_alias_list.dereference() for exe_alias_list_item in exe_alias_list.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_EXE_ALIAS_LIST", - "ListEntry" + "ListEntry", ): yield exe_alias_list_item From bd678eaf80482d4285e3409288a7dfd116dccd82 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Fri, 27 Sep 2024 16:07:44 -0500 Subject: [PATCH 414/979] #816 - fix cmdscan --- .../framework/plugins/windows/cmdscan.py | 78 ++++++++++++------- .../framework/plugins/windows/consoles.py | 38 +++++---- 2 files changed, 75 insertions(+), 41 deletions(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index b92d7efab..09cb0f316 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -14,17 +14,9 @@ from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist, vadinfo, info, verinfo, consoles -from volatility3.plugins.windows.registry import hivelist +from volatility3.plugins.windows import pslist, consoles -try: - import capstone - - has_capstone = True -except ImportError: - has_capstone = False - vollog = logging.getLogger(__name__) @@ -32,6 +24,7 @@ class CmdScan(interfaces.plugins.PluginInterface): """Looks for Windows Command History lists""" _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls): @@ -46,7 +39,7 @@ class CmdScan(interfaces.plugins.PluginInterface): name="pslist", component=pslist.PsList, version=(2, 0, 0) ), requirements.PluginRequirement( - name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + name="consoles", plugin=consoles.Consoles, version=(1, 0, 0) ), requirements.BooleanRequirement( name="no_registry", @@ -70,7 +63,7 @@ class CmdScan(interfaces.plugins.PluginInterface): size_filter: Optional[int] = 0x40000000, ) -> List[Tuple[int, int]]: """ - Returns vads of a process with smaller than size_filter + Returns vads of a process with size smaller than size_filter Args: conhost_proc: the process object for conhost.exe @@ -114,12 +107,10 @@ class CmdScan(interfaces.plugins.PluginInterface): Returns: The conhost process object, the command history structure, a dictionary of properties for - that command historyn structure. + that command history structure. """ - conhost_symbol_table = consoles.Consoles.create_conhost_symbol_table( - context, kernel_layer_name, kernel_table_name, config_path - ) + conhost_symbol_table = None for conhost_proc, proc_layer_name in consoles.Consoles.find_conhost_proc(procs): if not conhost_proc: @@ -143,11 +134,22 @@ class CmdScan(interfaces.plugins.PluginInterface): proc_layer = context.layers[proc_layer_name] + if conhost_symbol_table is None: + conhost_symbol_table = consoles.Consoles.create_conhost_symbol_table( + context, + kernel_layer_name, + kernel_table_name, + config_path, + proc_layer_name, + conhostexe_base, + ) + conhost_module = context.module( conhost_symbol_table, proc_layer_name, offset=conhostexe_base ) sections = cls.get_filtered_vads(conhost_proc) + found_history_for_proc = False # scan for potential _COMMAND_HISTORY structures by using the CommandHistorySize for max_history_value in max_history: max_history_bytes = struct.pack("H", max_history_value) @@ -258,7 +260,12 @@ class CmdScan(interfaces.plugins.PluginInterface): f"reading {command_history} encountered exception {e}" ) - yield conhost_proc, command_history, command_history_properties + if command_history and command_history_properties: + found_history_for_proc = True + yield conhost_proc, command_history, command_history_properties + + if not found_history_for_proc: + yield conhost_proc, command_history or None, [] def _generator( self, procs: Generator[interfaces.objects.ObjectInterface, None, None] @@ -276,19 +283,18 @@ class CmdScan(interfaces.plugins.PluginInterface): no_registry = self.config.get("no_registry") if no_registry is False: - max_history, _max_buffers = ( - consoles.Consoles.get_console_settings_from_registry( - self.context, - self.config_path, - kernel.layer_name, - kernel.symbol_table_name, - max_history, - [], - ) + max_history, _ = consoles.Consoles.get_console_settings_from_registry( + self.context, + self.config_path, + kernel.layer_name, + kernel.symbol_table_name, + max_history, + [], ) vollog.debug(f"Possible CommandHistorySize values: {max_history}") + proc = None for ( proc, command_history, @@ -302,13 +308,14 @@ class CmdScan(interfaces.plugins.PluginInterface): max_history, ): process_name = utility.array_to_string(proc.ImageFileName) + process_pid = proc.UniqueProcessId if command_history and command_history_properties: for command_history_property in command_history_properties: yield ( command_history_property["level"], ( - proc.UniqueProcessId, + process_pid, process_name, format_hints.Hex(command_history.vol.offset), command_history_property["name"], @@ -322,6 +329,25 @@ class CmdScan(interfaces.plugins.PluginInterface): str(command_history_property["data"]), ), ) + else: + yield ( + 0, + ( + process_pid, + process_name, + ( + format_hints.Hex(command_history.vol.offset) + if command_history + else renderers.NotApplicableValue() + ), + "_COMMAND_HISTORY", + renderers.NotApplicableValue(), + "History Not Found", + ), + ) + + if proc is None: + vollog.warn("No conhost.exe processes found.") def _conhost_proc_filter(self, proc): """ diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index 89f9fcc80..a2eade321 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -17,19 +17,11 @@ from volatility3.framework.layers import scanners from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed -from volatility3.framework.symbols.windows import pdbutil from volatility3.framework.symbols.windows.extensions import pe, consoles -from volatility3.plugins.windows import pslist, vadinfo, info, verinfo +from volatility3.plugins.windows import pslist, info, verinfo from volatility3.plugins.windows.registry import hivelist -try: - import capstone - - has_capstone = True -except ImportError: - has_capstone = False - vollog = logging.getLogger(__name__) @@ -37,6 +29,7 @@ class Consoles(interfaces.plugins.PluginInterface): """Looks for Windows console buffers""" _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls): @@ -53,9 +46,6 @@ class Consoles(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="verinfo", component=verinfo.VerInfo, version=(1, 0, 0) ), - requirements.VersionRequirement( - name="pdbutil", component=pdbutil.PDBUtility, version=(1, 0, 0) - ), requirements.PluginRequirement( name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) ), @@ -443,6 +433,7 @@ class Consoles(interfaces.plugins.PluginInterface): conhost_symbol_table, proc_layer_name, offset=conhostexe_base ) + found_console_info_for_proc = False # scan for potential _CONSOLE_INFORMATION structures by using the CommandHistorySize for max_history_value in max_history: max_history_bytes = struct.pack("H", max_history_value) @@ -790,7 +781,12 @@ class Consoles(interfaces.plugins.PluginInterface): ) continue - yield conhost_proc, console_info, console_properties + if console_info and console_properties: + found_console_info_for_proc = True + yield conhost_proc, console_info, console_properties + + if not found_console_info_for_proc: + yield conhost_proc, console_info or None, [] @classmethod def get_console_settings_from_registry( @@ -903,8 +899,20 @@ class Consoles(interfaces.plugins.PluginInterface): ), ) else: - vollog.warn( - f"_CONSOLE_INFORMATION not found for {process_name} with pid {process_pid}." + yield ( + 0, + ( + process_pid, + process_name, + ( + format_hints.Hex(console_info.vol.offset) + if console_info + else renderers.NotApplicableValue() + ), + "_CONSOLE_INFORMATION", + renderers.NotApplicableValue(), + "Console Information Not Found", + ), ) if proc is None: From f1200c85b7e6cd3e1752523d478721c288e9d91b Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 27 Sep 2024 16:14:43 -0500 Subject: [PATCH 415/979] Windows: PsList type-hint cleanup It's more appropriate to use `Iterator` than `Iterable` as a return type here. Also updates the `Iterator` item type to be `EPROCESS` instead of just `ObjectInterface`. --- volatility3/framework/plugins/windows/pslist.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 8234b210f..3411c618c 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Iterable, List, Type +from typing import Callable, Iterator, List, Type, TYPE_CHECKING from volatility3.framework import renderers, interfaces, layers, exceptions, constants from volatility3.framework.configuration import requirements @@ -12,6 +12,7 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe +from volatility3.framework.symbols.windows import extensions from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) @@ -197,7 +198,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): filter_func: Callable[ [interfaces.objects.ObjectInterface], bool ] = lambda _: False, - ) -> Iterable[interfaces.objects.ObjectInterface]: + ) -> Iterator["extensions.EPROCESS"]: """Lists all the processes in the primary layer that are in the pid config option. From 4b0ece0fc992c27bea47b951b83219e19d633194 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 27 Sep 2024 16:17:02 -0500 Subject: [PATCH 416/979] Windows: psxview module type-hints and cleanup Adds type-hints to methods throughout the psxview module. --- .../framework/plugins/windows/psxview.py | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 71919c410..18ab3ca2f 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -1,9 +1,13 @@ -import datetime, logging, string +import datetime +import logging +import string +from typing import Dict, Iterable, List from volatility3.framework import constants, exceptions -from volatility3.framework.interfaces import plugins from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints, TreeGrid +from volatility3.framework.interfaces import plugins +from volatility3.framework.renderers import TreeGrid, format_hints +from volatility3.framework.symbols.windows import extensions from volatility3.plugins.windows import ( handles, info, @@ -77,14 +81,16 @@ class PsXView(plugins.PluginInterface): return False return True - def _filter_garbage_procs(self, proc_list): + def _filter_garbage_procs( + self, proc_list: Iterable[extensions.EPROCESS] + ) -> List[extensions.EPROCESS]: return [ p for p in proc_list if p.is_valid() and self._is_valid_proc_name(self._proc_name_to_string(p)) ] - def _translate_offset(self, offset): + def _translate_offset(self, offset: int) -> int: if not self.config["physical-offsets"]: return offset @@ -100,21 +106,25 @@ class PsXView(plugins.PluginInterface): return offset - def _proc_list_to_dict(self, tasks): + def _proc_list_to_dict( + self, tasks: Iterable[extensions.EPROCESS] + ) -> Dict[int, extensions.EPROCESS]: tasks = self._filter_garbage_procs(tasks) return {self._translate_offset(proc.vol.offset): proc for proc in tasks} def _check_pslist(self, tasks): return self._proc_list_to_dict(tasks) - def _check_psscan(self, layer_name, symbol_table): + def _check_psscan( + self, layer_name: str, symbol_table: str + ) -> Dict[int, extensions.EPROCESS]: res = psscan.PsScan.scan_processes( context=self.context, layer_name=layer_name, symbol_table=symbol_table ) return self._proc_list_to_dict(res) - def _check_thrdscan(self): + def _check_thrdscan(self) -> Dict[int, extensions.EPROCESS]: ret = [] for ethread in thrdscan.ThrdScan.scan_threads( @@ -135,8 +145,10 @@ class PsXView(plugins.PluginInterface): return self._proc_list_to_dict(ret) - def _check_csrss_handles(self, tasks, layer_name, symbol_table): - ret = [] + def _check_csrss_handles( + self, tasks: Iterable[extensions.EPROCESS], layer_name: str, symbol_table: str + ) -> Dict[int, extensions.EPROCESS]: + ret: List[extensions.EPROCESS] = [] for p in tasks: name = self._proc_name_to_string(p) From 3ebee83d05ca6d82361b85fa265a8de64bafff44 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 27 Sep 2024 16:18:27 -0500 Subject: [PATCH 417/979] Windows: psxview cleanup Replaced this series of statements with a single invocation of the `all` builtin for brevity and conciseness. --- volatility3/framework/plugins/windows/psxview.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 18ab3ca2f..b7a372a40 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -75,11 +75,8 @@ class PsXView(plugins.PluginInterface): "string", max_length=proc.ImageFileName.vol.count, errors="replace" ) - def _is_valid_proc_name(self, str): - for c in str: - if not c in self.valid_proc_name_chars: - return False - return True + def _is_valid_proc_name(self, string: str) -> bool: + return all(c in self.valid_proc_name_chars for c in string) def _filter_garbage_procs( self, proc_list: Iterable[extensions.EPROCESS] From d38a69fd5c4813abe1239783550798ba163c38a2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 Sep 2024 19:49:21 +0100 Subject: [PATCH 418/979] Ensure the class for Modules is stored in the configruation --- volatility3/framework/interfaces/context.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index a95f2b464..8b5e816e8 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -191,6 +191,9 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): self._native_layer_name ].build_configuration() + # Modules are constructable, and therefore require a class configuration variable + config["class"] = self.__class__.__module__ + "." + self.__class__.__name__ + for subconfig in subconfigs: for req in subconfigs[subconfig]: config[interfaces.configuration.path_join(subconfig, req)] = subconfigs[ From 49dc89c30cc005ec2a1fed7d697206c7aafc12d2 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 27 Sep 2024 16:19:27 -0500 Subject: [PATCH 419/979] Windows: psxview Win10+ fix The current implementation of this plugin does not incorporate the cookie on Win10+ systems, causing it to fail. This commit fixes that issue, and also contributes a performance improvement by extracting the call to `handles_plugin.get_type_map()` from the loop. Also replaces for-loop with list comprehension for clarity. --- .../framework/plugins/windows/pslist.py | 2 +- .../framework/plugins/windows/psxview.py | 71 +++++++++---------- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 3411c618c..478cc8b1b 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Iterator, List, Type, TYPE_CHECKING +from typing import Callable, Iterator, List, Type from volatility3.framework import renderers, interfaces, layers, exceptions, constants from volatility3.framework.configuration import requirements diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index b7a372a40..a8d185a2c 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -1,6 +1,7 @@ import datetime import logging import string +from itertools import chain from typing import Dict, Iterable, List from volatility3.framework import constants, exceptions @@ -147,30 +148,33 @@ class PsXView(plugins.PluginInterface): ) -> Dict[int, extensions.EPROCESS]: ret: List[extensions.EPROCESS] = [] + handles_plugin = handles.Handles( + context=self.context, config_path=self.config_path + ) + + type_map = handles_plugin.get_type_map(self.context, layer_name, symbol_table) + + cookie = handles_plugin.find_cookie( + context=self.context, + layer_name=layer_name, + symbol_table=symbol_table, + ) + for p in tasks: name = self._proc_name_to_string(p) - if name == "csrss.exe": - try: - if p.has_member("ObjectTable"): - handles_plugin = handles.Handles( - context=self.context, config_path=self.config_path - ) - hndls = list(handles_plugin.handles(p.ObjectTable)) - for h in hndls: - if ( - h.get_object_type( - handles_plugin.get_type_map( - self.context, layer_name, symbol_table - ) - ) - == "Process" - ): - ret.append(h.Body.cast("_EPROCESS")) + if name != "csrss.exe": + continue - except exceptions.InvalidAddressException: - vollog.log( - constants.LOGLEVEL_VVV, "Cannot access eprocess object table" - ) + try: + ret += [ + handle.Body.cast("_EPROCESS") + for handle in handles_plugin.handles(p.ObjectTable) + if handle.get_object_type(type_map, cookie) == "Process" + ] + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVV, "Cannot access eprocess object table" + ) return self._proc_list_to_dict(ret) @@ -187,7 +191,7 @@ class PsXView(plugins.PluginInterface): ) # get processes from each source - processes = {} + processes: Dict[str, Dict[int, extensions.EPROCESS]] = {} processes["pslist"] = self._check_pslist(kdbg_list_processes) processes["psscan"] = self._check_psscan(layer_name, symbol_table) @@ -196,27 +200,20 @@ class PsXView(plugins.PluginInterface): kdbg_list_processes, layer_name, symbol_table ) - # print results - - # list of lists of offsets - offsets = [list(processes[source].keys()) for source in processes] - - # flatten to one list - offsets = sum(offsets, []) - - # remove duplicates - offsets = set(offsets) + # Unique set of all offsets from all sources + offsets = set(chain(*(mapping.keys() for mapping in processes.values()))) for offset in offsets: - proc = None + # We know there will be at least one process mapped to each offset + proc: extensions.EPROCESS = next( + mapping[offset] for mapping in processes.values() if offset in mapping + ) in_sources = {src: False for src in processes} - for source in processes: - if offset in processes[source]: + for source, process_mapping in processes.items(): + if offset in process_mapping: in_sources[source] = True - if not proc: - proc = processes[source][offset] pid = proc.UniqueProcessId name = self._proc_name_to_string(proc) From d9dc28d0b5a05db8af852673ebefc5aab8257569 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Sep 2024 12:52:44 -0500 Subject: [PATCH 420/979] Windows: fix missing TCP connections This fixes some missing TCP connections in Windows 10 20348 by adding a scan constraint for `TTcb` pool tags. --- volatility3/framework/plugins/windows/netscan.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 868bd8bcd..66a24da5a 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -76,7 +76,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # ~ vollog.debug("Using pool size constraints: TcpL {}, TcpE {}, UdpA {}".format(tcpl_size, tcpe_size, udpa_size)) - return [ + constraints = [ # TCP listener poolscanner.PoolConstraint( b"TcpL", @@ -100,6 +100,19 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] + if symbol_table.startswith("netscan-win10-20348"): + vollog.debug("Adding additional pool constraint for `TTcb` tags") + constraints.append( + poolscanner.PoolConstraint( + b"TTcb", + type_name=symbol_table + constants.BANG + "_TCP_ENDPOINT", + size=(tcpe_size, None), + page_type=poolscanner.PoolType.NONPAGED | poolscanner.PoolType.FREE, + ) + ) + + return constraints + @classmethod def determine_tcpip_version( cls, From 8ad592a490f23b27e6f79e1cbff7b67a7c2f8aba Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Sep 2024 11:00:58 -0500 Subject: [PATCH 421/979] Windows: Fixes bad callback validity check This fixes a bug in the callbacks plugin which causes it to miss `IoRegisterShutdownNotification` callbacks on x86b samples. The `header.NameInfo.Name` field was being incorrectly treated as the device type. This fixes the issue by updating the `is_valid` method on the `_SHUTDOWN_PACKET` extension type to take a `type_map` parameter, and updates the method to correctly validate the object type. --- .../framework/plugins/windows/callbacks.py | 8 ++++++-- .../symbols/windows/extensions/callbacks.py | 19 +++++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index d5eeda1ea..562846def 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -248,8 +248,12 @@ class Callbacks(interfaces.plugins.PluginInterface): context, layer_name, nt_symbol_table, constraints ): try: - if hasattr(mem_object, "is_valid") and not mem_object.is_valid(): - continue + if isinstance(mem_object, callbacks._SHUTDOWN_PACKET): + if not mem_object.is_parseable(type_map): + continue + elif hasattr(mem_object, "is_valid"): + if not mem_object.is_valid(): + continue yield cls._process_scanned_callback(mem_object, type_map) except exceptions.InvalidAddressException: diff --git a/volatility3/framework/symbols/windows/extensions/callbacks.py b/volatility3/framework/symbols/windows/extensions/callbacks.py index f894644db..62e72803e 100644 --- a/volatility3/framework/symbols/windows/extensions/callbacks.py +++ b/volatility3/framework/symbols/windows/extensions/callbacks.py @@ -1,4 +1,5 @@ import logging +from typing import Dict from volatility3.framework import exceptions, objects from volatility3.framework.symbols.windows.extensions import pool @@ -14,7 +15,7 @@ class _SHUTDOWN_PACKET(objects.StructType, pool.ExecutiveObject): It exposes a function which sanity-checks structure members. """ - def is_valid(self) -> bool: + def is_parseable(self, type_map: Dict[int, str]) -> bool: """ Perform some checks. """ @@ -24,6 +25,9 @@ class _SHUTDOWN_PACKET(objects.StructType, pool.ExecutiveObject): and self.Entry.Blink.is_readable() and self.DeviceObject.is_readable() ): + vollog.debug( + f"Callback obj 0x{self.vol.offset:x} invalid due to unreadable structure members" + ) return False device = self.DeviceObject @@ -41,10 +45,17 @@ class _SHUTDOWN_PACKET(objects.StructType, pool.ExecutiveObject): try: header = device.get_object_header() - valid = header.NameInfo.Name == "Device" - return valid + object_type = header.get_object_type(type_map) + is_valid = object_type == "Device" + if not is_valid: + vollog.debug( + f"Callback obj 0x{self.vol.offset:x} invalid due to invalid device type: wanted 'Device', found '{object_type}'" + ) + return is_valid except ValueError: - vollog.debug(f"Could not get NameInfo for object at 0x{self.vol.offset:x}") + vollog.debug( + f"Could not get object type for object at 0x{self.vol.offset:x}" + ) return False From 806f78d3727b41152a2a70d4753f04012a5f8777 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Sep 2024 13:25:30 -0500 Subject: [PATCH 422/979] Linux: Sockstat - fix incorrect version bump I erred when I bumped the version as a part of !1271 - The `SockHandlers` class got the major version bump, not `Sockstat`. This reverts the `SockHandlers` version bump and applies it to `Sockstat` instead. --- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index d3efc78dd..652d039cf 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -438,7 +438,7 @@ class Sockstat(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -449,7 +449,7 @@ class Sockstat(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name="SockHandlers", component=SockHandlers, version=(1, 0, 0) + name="SockHandlers", component=SockHandlers, version=(2, 0, 0) ), requirements.PluginRequirement( name="lsof", plugin=lsof.Lsof, version=(1, 1, 0) From db55f23070b16d8c9df40e1ddfda5df8a6ad3282 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Sep 2024 16:55:27 -0500 Subject: [PATCH 423/979] Windows: Callbacks - fix breaking API change Moves as much of the `is_parseable` check as possible back into an `is_valid` method to avoid breaking API changes. --- .../symbols/windows/extensions/callbacks.py | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/callbacks.py b/volatility3/framework/symbols/windows/extensions/callbacks.py index 62e72803e..f54db39f2 100644 --- a/volatility3/framework/symbols/windows/extensions/callbacks.py +++ b/volatility3/framework/symbols/windows/extensions/callbacks.py @@ -15,7 +15,7 @@ class _SHUTDOWN_PACKET(objects.StructType, pool.ExecutiveObject): It exposes a function which sanity-checks structure members. """ - def is_parseable(self, type_map: Dict[int, str]) -> bool: + def is_valid(self) -> bool: """ Perform some checks. """ @@ -30,6 +30,25 @@ class _SHUTDOWN_PACKET(objects.StructType, pool.ExecutiveObject): ) return False + except exceptions.InvalidAddressException: + vollog.debug( + f"callback obj 0x{self.vol.offset:x} invalid due to invalid address access" + ) + return False + + return True + + def is_parseable(self, type_map: Dict[int, str]) -> bool: + """ + Determines whether or not this `_SHUTDOWN_PACKET` callback can be reliably parsed. + Requires a `type_map` that maps NT executive object type indices to string representations. + This type map can be acquired via the `handles.Handles.get_type_map` classmethod. + """ + if not self.is_valid(): + return False + + try: + device = self.DeviceObject if not device or not (device.DriverObject.DriverStart % 0x1000 == 0): vollog.debug( @@ -37,13 +56,6 @@ class _SHUTDOWN_PACKET(objects.StructType, pool.ExecutiveObject): ) return False - except exceptions.InvalidAddressException: - vollog.debug( - f"callback obj 0x{self.vol.offset:x} invalid due to invalid address access" - ) - return False - - try: header = device.get_object_header() object_type = header.get_object_type(type_map) is_valid = object_type == "Device" @@ -52,6 +64,11 @@ class _SHUTDOWN_PACKET(objects.StructType, pool.ExecutiveObject): f"Callback obj 0x{self.vol.offset:x} invalid due to invalid device type: wanted 'Device', found '{object_type}'" ) return is_valid + except exceptions.InvalidAddressException: + vollog.debug( + f"callback obj 0x{self.vol.offset:x} invalid due to invalid address access" + ) + return False except ValueError: vollog.debug( f"Could not get object type for object at 0x{self.vol.offset:x}" From 6cd39c04c14735057c747c933eeccdf364bb7466 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Sep 2024 17:04:17 +1000 Subject: [PATCH 424/979] Refactor of module object. Adding function helpers to simplify the control of exceptions and errors and consolidate everything on them. --- .../symbols/linux/extensions/__init__.py | 89 +++++++++---------- 1 file changed, 40 insertions(+), 49 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fdd34403a..0a879249b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,7 +13,6 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion -from volatility3.framework.configuration import requirements from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -56,88 +55,80 @@ class module(generic.GenericIntelProcess): self._mod_mem_type = {} return self._mod_mem_type + def _get_mem_type(self, mod_mem_type_name): + module_mem_index = self.mod_mem_type.get(mod_mem_type_name) + if module_mem_index is None: + raise AttributeError(f"Unknown module memory type '{mod_mem_type_name}'") + + if not (0 <= module_mem_index < self.mem.count): + raise AttributeError( + f"Invalid module memory type index '{module_mem_index}'" + ) + + return self.mem[module_mem_index] + + def _get_mem_size(self, mod_mem_type_name): + return self._get_mem_type(mod_mem_type_name).size + + def _get_mem_base(self, mod_mem_type_name): + return self._get_mem_type(mod_mem_type_name).base + def get_module_base(self): if self.has_member("mem"): # kernels 6.4+ - try: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base - except KeyError: - raise AttributeError( - "module -> get_module_base: Unable to get module base. Cannot read base from MOD_TEXT." - ) + return self._get_mem_base("MOD_TEXT") elif self.has_member("core_layout"): return self.core_layout.base elif self.has_member("module_core"): return self.module_core - raise AttributeError("module -> get_module_base: Unable to get module base") + + raise AttributeError("Unable to get module base") def get_init_size(self): if self.has_member("mem"): # kernels 6.4+ - try: - return ( - self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size - + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size - ) - except KeyError: - raise AttributeError( - "module -> get_init_size: Unable to determine .init section size of module. Cannot read size of MOD_INIT_TEXT, MOD_INIT_DATA, and MOD_INIT_RODATA" - ) + return ( + self._get_mem_size("MOD_INIT_TEXT") + + self._get_mem_size("MOD_INIT_DATA") + + self._get_mem_size("MOD_INIT_RODATA") + ) elif self.has_member("init_layout"): return self.init_layout.size elif self.has_member("init_size"): return self.init_size - raise AttributeError( - "module -> get_init_size: Unable to determine .init section size of module" - ) + + raise AttributeError("Unable to determine .init section size of module") def get_core_size(self): if self.has_member("mem"): # kernels 6.4+ - try: - return ( - self.mem[self.mod_mem_type["MOD_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_DATA"]].size - + self.mem[self.mod_mem_type["MOD_RODATA"]].size - + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size - ) - except KeyError: - raise AttributeError( - "module -> get_core_size: Unable to determine core size of module. Cannot read size of MOD_TEXT, MOD_DATA, MOD_RODATA, and MOD_RO_AFTER_INIT." - ) + return ( + self._get_mem_size("MOD_TEXT") + + self._get_mem_size("MOD_DATA") + + self._get_mem_size("MOD_RODATA") + + self._get_mem_size("MOD_RO_AFTER_INIT") + ) elif self.has_member("core_layout"): return self.core_layout.size elif self.has_member("core_size"): return self.core_size - raise AttributeError( - "module -> get_core_size: Unable to determine core size of module" - ) + + raise AttributeError("Unable to determine core size of module") def get_module_core(self): if self.has_member("mem"): # kernels 6.4+ - try: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base - except KeyError: - raise AttributeError( - "module -> get_module_core: Unable to get module core. Cannot read base from MOD_TEXT." - ) + return self._get_mem_base("MOD_TEXT") elif self.has_member("core_layout"): return self.core_layout.base elif self.has_member("module_core"): return self.module_core - raise AttributeError("module -> get_module_core: Unable to get module core") + raise AttributeError("Unable to get module core") def get_module_init(self): if self.has_member("mem"): # kernels 6.4+ - try: - return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base - except KeyError: - raise AttributeError( - "module -> get_module_core: Unable to get module init. Cannot read base from MOD_INIT_TEXT." - ) + return self._get_mem_base("MOD_INIT_TEXT") elif self.has_member("init_layout"): return self.init_layout.base elif self.has_member("module_init"): return self.module_init - raise AttributeError("module -> get_module_init: Unable to get module init") + raise AttributeError("Unable to get module init") def get_name(self): """Get the name of the module as a string""" From ba351f511d9e4427c3376bfa5d312e8253f85d9a Mon Sep 17 00:00:00 2001 From: eve Date: Tue, 1 Oct 2024 06:36:56 +0100 Subject: [PATCH 425/979] Linux: Update malfind plugin to use symbols.symbol_table_is_64bit when determining if a 32bit OS is detected in the sample --- volatility3/framework/plugins/linux/malfind.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index cf06ee0cc..18f3dcd56 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -5,7 +5,7 @@ from typing import List import logging from volatility3.framework import constants, interfaces -from volatility3.framework import renderers +from volatility3.framework import renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints @@ -63,15 +63,9 @@ class Malfind(interfaces.plugins.PluginInterface): def _generator(self, tasks): # determine if we're on a 32 or 64 bit kernel vmlinux = self.context.modules[self.config["kernel"]] - if ( - self.context.symbol_space.get_type( - vmlinux.symbol_table_name + constants.BANG + "pointer" - ).size - == 4 - ): - is_32bit_arch = True - else: - is_32bit_arch = False + is_32bit_arch = not symbols.symbol_table_is_64bit( + self.context, vmlinux.symbol_table_name + ) for task in tasks: process_name = utility.array_to_string(task.comm) From 5dee3aeeee5e6b08481c1e39d2f7cb37a17c086c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 1 Oct 2024 19:30:50 +1000 Subject: [PATCH 426/979] Add linux.hidden_modules plugin --- .../framework/plugins/linux/hidden_modules.py | 331 ++++++++++++++++++ .../symbols/linux/extensions/__init__.py | 38 ++ 2 files changed, 369 insertions(+) create mode 100644 volatility3/framework/plugins/linux/hidden_modules.py diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py new file mode 100644 index 000000000..1c2429449 --- /dev/null +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -0,0 +1,331 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import re +import functools +import logging +import contextlib +from typing import List, Iterable +from volatility3.framework import renderers, interfaces, exceptions, objects +from volatility3.framework.constants.architectures import LINUX_ARCHS +from volatility3.framework.renderers import format_hints +from volatility3.framework.configuration import requirements +from volatility3.plugins.linux import lsmod + +vollog = logging.getLogger(__name__) + + +class Hidden_modules(interfaces.plugins.PluginInterface): + """Carves memory to find hidden kernel modules""" + + _required_framework_version = (2, 10, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=LINUX_ARCHS, + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + requirements.BooleanRequirement( + name="fast", + description="Fast scan method. Recommended only for kernels 4.2 and above", + optional=True, + default=False, + ), + ] + + def _get_modules_memory_boundaries(self, vmlinux): + if vmlinux.has_symbol("mod_tree"): + mod_tree = vmlinux.object_from_symbol("mod_tree") + modules_addr_min = mod_tree.addr_min + modules_addr_max = mod_tree.addr_max + elif vmlinux.has_symbol("module_addr_min"): + modules_addr_min = vmlinux.object_from_symbol("module_addr_min") + modules_addr_max = vmlinux.object_from_symbol("module_addr_max") + + if isinstance(modules_addr_min, objects.Void): + # Crap ISF! Here's my best-effort workaround + vollog.warning( + "Your ISF symbols are missing type information. You may need to update " + "the ISF using the latest version of dwarf2json" + ) + # See issue #1041. In the Linux kernel these are "unsigned long" + for type_name in ("long unsigned int", "unsigned long"): + if vmlinux.has_type(type_name): + modules_addr_min = modules_addr_min.cast(type_name) + modules_addr_max = modules_addr_max.cast(type_name) + break + else: + raise exceptions.VolatilityException( + "Bad ISF! Please update the ISF using the latest version of dwarf2json" + ) + else: + raise exceptions.VolatilityException( + "Cannot find the module memory allocation area. Unsupported kernel" + ) + + return modules_addr_min, modules_addr_max + + def _get_module_state_live_bytes( + self, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> bytes: + """Retrieve the MODULE_STATE_LIVE value bytes by introspecting its enum type + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + + Returns: + The MODULE_STATE_LIVE value bytes + """ + vmlinux = context.modules[vmlinux_module_name] + module_state_type_template = vmlinux.get_type("module").vol.members["state"][1] + module_state_live_val = module_state_type_template.choices["MODULE_STATE_LIVE"] + data_format = module_state_type_template.base_type.vol.data_format + module_state_live_bytes = objects.convert_value_to_data( + module_state_live_val, int, data_format + ) + return module_state_live_bytes + + def get_hidden_modules_vol2( + self, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + known_module_addresses, + modules_memory_boundaries: tuple, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Enumerate hidden modules using the traditional implementation. + + This is a port of the Volatility2 plugin, with minor code improvements. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + Yields: + module objects + """ + vmlinux = context.modules[vmlinux_module_name] + vmlinux_layer = context.layers[vmlinux.layer_name] + + check_nums = ( + 3000, + 2800, + 2700, + 2500, + 2300, + 2100, + 2000, + 1500, + 1300, + 1200, + 1024, + 512, + 256, + 128, + 96, + 64, + 48, + 32, + 24, + ) + modules_addr_min, modules_addr_max = modules_memory_boundaries + modules_addr_min = modules_addr_min & ~0xFFF + modules_addr_max = (modules_addr_max & ~0xFFF) + vmlinux_layer.page_size + + check_bufs = [] + replace_bufs = [] + minus_size = vmlinux.get_type("pointer").size + null_pointer_bytes = b"\x00" * minus_size + for num in check_nums: + check_bufs.append(b"\x00" * num) + replace_bufs.append((b"\xff" * (num - minus_size)) + null_pointer_bytes) + + all_ffs = b"\xff" * 4096 + scan_list = [] + for page_addr in range( + modules_addr_min, modules_addr_max, vmlinux_layer.page_size + ): + content_fixed = all_ffs + with contextlib.suppress( + exceptions.InvalidAddressException, + exceptions.PagedInvalidAddressException, + ): + content = vmlinux_layer.read(page_addr, vmlinux_layer.page_size) + + all_nulls = all(x == 0 for x in content) + if content and not all_nulls: + content_fixed = content + for check_bytes, replace_bytes in zip(check_bufs, replace_bufs): + content_fixed = content_fixed.replace( + check_bytes, replace_bytes + ) + + scan_list.append(content_fixed) + + scan_buf = b"".join(scan_list) + del scan_list + + module_state_live_bytes = self._get_module_state_live_bytes( + context, vmlinux_module_name + ) + # f'strings cannot be combined with bytes literals + for cur_addr in re.finditer(b"(?=(%s))" % (module_state_live_bytes), scan_buf): + module_addr = modules_addr_min + cur_addr.start() + + if module_addr in known_module_addresses: + continue + + module = vmlinux.object("module", offset=module_addr, absolute=True) + if module and module.is_valid(): + yield module + + @functools.cached_property + def module_address_alignment(self) -> int: + """Obtain the module memory address alignment. This is only used with the fast scan method. + + struct module is aligned to the L1 cache line, which is typically 64 bytes for most + common i386/AMD64/ARM64 configurations. In some cases, it can be 128 bytes, but this + will still work. + + Returns: + The struct module alignment + """ + # FIXME: When dwarf2json/ISF supports type alignments. Read it directly from the type metadata + # The cached_property won't provide any benefits until then + return 64 + + def get_hidden_modules_fast( + self, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + known_module_addresses, + modules_memory_boundaries: tuple, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Enumerate hidden modules by taking advantage of memory address alignment patterns + + This technique is much faster and uses less memory than the traditional scan method + in Volatility2, but it doesn't work with older kernels. + + From kernels 4.2 struct module allocation are aligned to the L1 cache line size. + In i386/amd64/arm64 this is typically 64 bytes. However, this can be changed in + the Linux kernel configuration via CONFIG_X86_L1_CACHE_SHIFT. The alignment can + also be obtained from the DWARF info i.e. DW_AT_alignment<64>, but dwarf2json + doesn't support this feature yet. + In kernels < 4.2, alignment attributes are absent in the struct module, meaning + alignment cannot be guaranteed. Therefore, for older kernels, it's better to use + the traditional scan technique. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + Yields: + module objects + """ + vmlinux = context.modules[vmlinux_module_name] + vmlinux_layer = context.layers[vmlinux.layer_name] + + module_addr_min, module_addr_max = modules_memory_boundaries + + module_state_live_bytes = self._get_module_state_live_bytes( + context, vmlinux_module_name + ) + + for module_addr in range( + module_addr_min, module_addr_max, self.module_address_alignment + ): + if module_addr in known_module_addresses: + continue + + try: + # This is just a pre-filter. Module readability and consistency are verified in module.is_valid() + module_state_bytes = vmlinux_layer.read( + module_addr, len(module_state_live_bytes) + ) + if module_state_bytes != module_state_live_bytes: + continue + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ): + continue + + module = vmlinux.object("module", offset=module_addr, absolute=True) + if module and module.is_valid(): + yield module + + def _validate_alignment_patterns(self, addresses: Iterable[int]) -> bool: + """Check if the memory addresses meet our alignments patterns + + Args: + addresses: Iterable with the address values + + Returns: + True if all the addresses meet the alignment + """ + return all(addr % self.module_address_alignment == 0 for addr in addresses) + + def get_hidden_modules( + self, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Enumerate hidden modules + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + Yields: + module objects + """ + vmlinux = context.modules[vmlinux_module_name] + vmlinux_layer = context.layers[vmlinux.layer_name] + + known_module_addresses = { + vmlinux_layer.canonicalize(module.vol.offset) + for module in lsmod.Lsmod.list_modules(context, vmlinux_module_name) + } + + modules_memory_boundaries = self._get_modules_memory_boundaries(vmlinux) + + if self.config.get("fast"): + if self._validate_alignment_patterns(known_module_addresses): + scan_method = self.get_hidden_modules_fast + else: + vollog.warning( + f"Module addresses aren't aligned to {self.module_address_alignment} bytes. " + "Switching to the traditional scan method." + ) + scan_method = self.get_hidden_modules_vol2 + else: + scan_method = self.get_hidden_modules_vol2 + + yield from scan_method( + context, + vmlinux_module_name, + known_module_addresses, + modules_memory_boundaries, + ) + + def _generator(self): + vmlinux_module_name = self.config["kernel"] + for module in self.get_hidden_modules(self.context, vmlinux_module_name): + module_addr = module.vol.offset + module_name = module.get_name() or renderers.NotAvailableValue() + fields = (format_hints.Hex(module_addr), module_name) + yield (0, fields) + + def run(self): + headers = [ + ("Address", format_hints.Hex), + ("Name", str), + ] + return renderers.TreeGrid(headers, self._generator()) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0a879249b..b30634ca3 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -35,6 +35,34 @@ class module(generic.GenericIntelProcess): super().__init__(*args, **kwargs) self._mod_mem_type = None # Initialize _mod_mem_type to None for memoization + def is_valid(self): + layer = self._context.layers[self.vol.layer_name] + # Make sure the entire module content is readable + if not layer.is_valid(self.vol.offset, self.vol.size): + return False + + if not self.state.is_valid_choice: + return False + + core_size = self.get_core_size() + if not ( + 1 <= core_size <= 20000000 + and core_size + self.get_init_size() >= 4096 + and 1 <= self.get_core_text_size() <= 20000000 + ): + return False + + if self.has_member("mkobj") and self.mkobj.has_member("mod"): + if not ( + self.mkobj + and self.mkobj.mod + and self.mkobj.mod.is_readable() + and self.mkobj.mod == self.vol.offset + ): + return False + + return True + @property def mod_mem_type(self): """Return the mod_mem_type enum choices if available or an empty dict if not""" @@ -112,6 +140,16 @@ class module(generic.GenericIntelProcess): raise AttributeError("Unable to determine core size of module") + def get_core_text_size(self): + if self.has_member("mem"): # kernels 6.4+ + return self._get_mem_size("MOD_TEXT") + elif self.has_member("core_layout"): + return self.core_layout.text_size + elif self.has_member("core_text_size"): + return self.core_text_size + + raise AttributeError("Unable to determine core text size of module") + def get_module_core(self): if self.has_member("mem"): # kernels 6.4+ return self._get_mem_base("MOD_TEXT") From c0672f6c4b2769138da961f2d7c17f5c688b81ac Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Sep 2024 16:22:26 -0500 Subject: [PATCH 427/979] Linux: Sockstat - adds pytest This adds a pytest for the `linux.sockstat.Sockstat` plugin. --- test/test_volatility.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index 6e54aa053..847be88d9 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -341,6 +341,17 @@ def test_linux_tty_check(image, volatility, python): assert out.count(b"\n") >= 5 assert rc == 0 +def test_linux_sockstat(image, volatility, python): + rc, out, err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) + + assert out.count(b"AF_UNIX") >= 354 + assert out.count(b"AF_BLUETOOTH") >= 5 + assert out.count(b"AF_INET") >= 32 + assert out.count(b"AF_INET6") >= 20 + assert out.count(b"AF_PACKET") >= 1 + assert out.count(b"AF_NETLINK") >= 43 + assert rc == 0 + def test_linux_library_list(image, volatility, python): rc, out, err = runvol_plugin( From ece2dbe9a77947d31eaaf7da7cbb1a6cea5f330d Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Tue, 1 Oct 2024 13:00:15 -0500 Subject: [PATCH 428/979] #816 formatting fixes --- .../framework/plugins/windows/consoles.py | 4 ++-- .../symbols/windows/extensions/consoles.py | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index a2eade321..dfc14f2d6 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -168,7 +168,7 @@ class Consoles(interfaces.plugins.PluginInterface): raise NotImplementedError( "Kernel Debug Structure version format not supported!" ) - except: + except Exception: # unsure what to raise here. Also, it might be useful to add some kind of fallback, # either to a user-provided version or to another method to determine conhost.exe's version raise exceptions.VolatilityException( @@ -834,7 +834,7 @@ class Consoles(interfaces.plugins.PluginInterface): max_history.add(value.decode_data()) elif val_name == "NumberOfHistoryBuffers": max_buffers.add(value.decode_data()) - except: + except Exception: continue return max_history, max_buffers diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 555197fc4..390b6858d 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -95,6 +95,8 @@ class ALIAS(objects.StructType): "string", encoding="utf-16", errors="replace", max_length=512 ) + return None + def get_target(self): if self.Target.Length < 8: return self.Target.Chars.cast( @@ -108,6 +110,8 @@ class ALIAS(objects.StructType): "string", encoding="utf-16", errors="replace", max_length=512 ) + return None + class EXE_ALIAS_LIST(objects.StructType): """An Exe Alias List Structure""" @@ -131,6 +135,8 @@ class EXE_ALIAS_LIST(objects.StructType): "string", encoding="utf-16", errors="replace", max_length=512 ) + return None + def get_aliases(self): """Generator for the individual aliases for a particular executable.""" @@ -211,7 +217,7 @@ class SCREEN_INFORMATION(objects.StructType): try: text = row.get_text(truncate_lines) rows.append(text) - except: + except Exception: break if truncate_rows: @@ -301,7 +307,7 @@ class CONSOLE_INFORMATION(objects.StructType): return self.Title.dereference().cast( "string", encoding="utf-16", errors="replace", max_length=512 ) - except: + except Exception: return "" def get_original_title(self): @@ -309,7 +315,7 @@ class CONSOLE_INFORMATION(objects.StructType): return self.OriginalTitle.dereference().cast( "string", encoding="utf-16", errors="replace", max_length=512 ) - except: + except Exception: return "" @@ -340,6 +346,8 @@ class COMMAND(objects.StructType): "string", encoding="utf-16", errors="replace", max_length=512 ) + return None + class COMMAND_HISTORY(objects.StructType): """A Command History Structure.""" @@ -387,6 +395,8 @@ class COMMAND_HISTORY(objects.StructType): "string", encoding="utf-16", errors="replace", max_length=512 ) + return None + def scan_command_bucket(self, end=None): """Brute force print all strings pointed to by the CommandBucket entries by going to greater of EndCapacity or CommandCountMax*sizeof(_COMMAND)""" From 950ab3e201e0f31efeb496276e1d30169b091a2b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 1 Oct 2024 19:47:09 +0100 Subject: [PATCH 429/979] Add in additional microarchitectures for vmscan --- .../generic/vmcs/nehalem-architecture.json | 131 ++++++++++++++++++ .../vmcs/sandybridge-architecture.json | 131 ++++++++++++++++++ .../generic/vmcs/westmere-architecture.json | 131 ++++++++++++++++++ 3 files changed, 393 insertions(+) create mode 100644 volatility3/symbols/generic/vmcs/nehalem-architecture.json create mode 100644 volatility3/symbols/generic/vmcs/sandybridge-architecture.json create mode 100644 volatility3/symbols/generic/vmcs/westmere-architecture.json diff --git a/volatility3/symbols/generic/vmcs/nehalem-architecture.json b/volatility3/symbols/generic/vmcs/nehalem-architecture.json new file mode 100644 index 000000000..ae0ab2863 --- /dev/null +++ b/volatility3/symbols/generic/vmcs/nehalem-architecture.json @@ -0,0 +1,131 @@ +{ + "base_types": { + "pointer": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned char": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + } + }, + "enums": {}, + "metadata": { + "format": "6.1.0", + "producer": { + "datetime": "2021-07-31T17:37:28.302702", + "name": "vmextract-by-hand", + "version": "0.0.1" + } + }, + "symbols": { + "revision_id": { + "address": 0, + "constant_data": "MTQ=" + } + }, + "user_types": { + "_VMCS": { + "fields": { + "ept": { + "offset": 232, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "executive_vmcs_ptr": { + "offset": 208, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr3": { + "offset": 736, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr4": { + "offset": 744, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_pdpte": { + "offset": 928, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "unsigned long long" + } + } + }, + "guest_physical_addr": { + "offset": 240, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr3": { + "offset": 832, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr4": { + "offset": 840, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vmcs_link_ptr": { + "offset": 248, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vpid": { + "offset": 752, + "type": { + "kind": "struct", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 4096 + } + } +} \ No newline at end of file diff --git a/volatility3/symbols/generic/vmcs/sandybridge-architecture.json b/volatility3/symbols/generic/vmcs/sandybridge-architecture.json new file mode 100644 index 000000000..b2b3cfebf --- /dev/null +++ b/volatility3/symbols/generic/vmcs/sandybridge-architecture.json @@ -0,0 +1,131 @@ +{ + "base_types": { + "pointer": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned char": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + } + }, + "enums": {}, + "metadata": { + "format": "6.1.0", + "producer": { + "datetime": "2021-07-31T17:37:28.311608", + "name": "vmextract-by-hand", + "version": "0.0.1" + } + }, + "symbols": { + "revision_id": { + "address": 0, + "constant_data": "MTY=" + } + }, + "user_types": { + "_VMCS": { + "fields": { + "ept": { + "offset": 232, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "executive_vmcs_ptr": { + "offset": 208, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr3": { + "offset": 736, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr4": { + "offset": 744, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_pdpte": { + "offset": 928, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "unsigned long long" + } + } + }, + "guest_physical_addr": { + "offset": 240, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr3": { + "offset": 832, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr4": { + "offset": 840, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vmcs_link_ptr": { + "offset": 248, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vpid": { + "offset": 752, + "type": { + "kind": "struct", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 4096 + } + } +} \ No newline at end of file diff --git a/volatility3/symbols/generic/vmcs/westmere-architecture.json b/volatility3/symbols/generic/vmcs/westmere-architecture.json new file mode 100644 index 000000000..2769f7081 --- /dev/null +++ b/volatility3/symbols/generic/vmcs/westmere-architecture.json @@ -0,0 +1,131 @@ +{ + "base_types": { + "pointer": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned char": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + } + }, + "enums": {}, + "metadata": { + "format": "6.1.0", + "producer": { + "datetime": "2021-07-31T17:37:28.314801", + "name": "vmextract-by-hand", + "version": "0.0.1" + } + }, + "symbols": { + "revision_id": { + "address": 0, + "constant_data": "MTU=" + } + }, + "user_types": { + "_VMCS": { + "fields": { + "ept": { + "offset": 320, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "executive_vmcs_ptr": { + "offset": 208, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr3": { + "offset": 736, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_cr4": { + "offset": 744, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "guest_pdpte": { + "offset": 928, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "unsigned long long" + } + } + }, + "guest_physical_addr": { + "offset": 328, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr3": { + "offset": 832, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "host_cr4": { + "offset": 840, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vmcs_link_ptr": { + "offset": 248, + "type": { + "kind": "struct", + "name": "unsigned long long" + } + }, + "vpid": { + "offset": 220, + "type": { + "kind": "struct", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 4096 + } + } +} \ No newline at end of file From d5e6e7c203f4e17390f0891fa4b70138d1fb97a5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 2 Oct 2024 09:43:27 +1000 Subject: [PATCH 430/979] Allow any module state value in both traditional and fast scan methods --- .../framework/plugins/linux/hidden_modules.py | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index 1c2429449..6d04c08af 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -73,28 +73,29 @@ class Hidden_modules(interfaces.plugins.PluginInterface): return modules_addr_min, modules_addr_max - def _get_module_state_live_bytes( + def _get_module_state_values_bytes( self, context: interfaces.context.ContextInterface, vmlinux_module_name: str, - ) -> bytes: - """Retrieve the MODULE_STATE_LIVE value bytes by introspecting its enum type + ) -> List[bytes]: + """Retrieve the module state values bytes by introspecting its enum type Args: context: The context to retrieve required elements (layers, symbol tables) from vmlinux_module_name: The name of the kernel module on which to operate Returns: - The MODULE_STATE_LIVE value bytes + A list with the module state values bytes """ vmlinux = context.modules[vmlinux_module_name] module_state_type_template = vmlinux.get_type("module").vol.members["state"][1] - module_state_live_val = module_state_type_template.choices["MODULE_STATE_LIVE"] data_format = module_state_type_template.base_type.vol.data_format - module_state_live_bytes = objects.convert_value_to_data( - module_state_live_val, int, data_format - ) - return module_state_live_bytes + values = module_state_type_template.choices.values() + values_bytes = [ + objects.convert_value_to_data(value, int, data_format) + for value in sorted(values) + ] + return values_bytes def get_hidden_modules_vol2( self, @@ -174,11 +175,12 @@ class Hidden_modules(interfaces.plugins.PluginInterface): scan_buf = b"".join(scan_list) del scan_list - module_state_live_bytes = self._get_module_state_live_bytes( + module_state_values_bytes = self._get_module_state_values_bytes( context, vmlinux_module_name ) + values_bytes_pattern = b"|".join(module_state_values_bytes) # f'strings cannot be combined with bytes literals - for cur_addr in re.finditer(b"(?=(%s))" % (module_state_live_bytes), scan_buf): + for cur_addr in re.finditer(b"(?=(%s))" % values_bytes_pattern, scan_buf): module_addr = modules_addr_min + cur_addr.start() if module_addr in known_module_addresses: @@ -235,7 +237,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): module_addr_min, module_addr_max = modules_memory_boundaries - module_state_live_bytes = self._get_module_state_live_bytes( + module_state_values_bytes = self._get_module_state_values_bytes( context, vmlinux_module_name ) @@ -248,9 +250,9 @@ class Hidden_modules(interfaces.plugins.PluginInterface): try: # This is just a pre-filter. Module readability and consistency are verified in module.is_valid() module_state_bytes = vmlinux_layer.read( - module_addr, len(module_state_live_bytes) + module_addr, len(module_state_values_bytes[0]) ) - if module_state_bytes != module_state_live_bytes: + if module_state_bytes not in module_state_values_bytes: continue except ( exceptions.PagedInvalidAddressException, From c18818aa8d953f188e09dce7d5457488dbb6c4dc Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 2 Oct 2024 10:38:01 +1000 Subject: [PATCH 431/979] Fix misuse of all() and any() functions across the codebase --- volatility3/cli/text_filter.py | 4 +-- volatility3/cli/volshell/generic.py | 2 +- volatility3/framework/automagic/pdbscan.py | 6 ++-- .../framework/configuration/requirements.py | 28 ++++++++----------- volatility3/framework/interfaces/objects.py | 2 +- volatility3/framework/layers/intel.py | 6 ++-- volatility3/framework/layers/registry.py | 6 ++-- volatility3/framework/layers/segmented.py | 6 ++-- volatility3/framework/objects/__init__.py | 6 ++-- .../framework/plugins/linux/check_afinfo.py | 4 +-- volatility3/framework/plugins/timeliner.py | 8 ++---- volatility3/framework/symbols/metadata.py | 4 +-- 12 files changed, 31 insertions(+), 51 deletions(-) diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 948e969df..3d69934e9 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -53,7 +53,7 @@ class CLIFilter: """Filters the row based on each of the column_filters""" if not self._filters: return False - found = any([column_filter.found(row) for column_filter in self._filters]) + found = any(column_filter.found(row) for column_filter in self._filters) return not found @@ -86,7 +86,7 @@ class ColumnFilter: otherwise it is filtered. """ if self.column_num is None: - found = any([self.find(x) for x in row]) + found = any(self.find(x) for x in row) else: found = self.find(row[self.column_num]) if self.exclude: diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index b95129d19..c5eac36e2 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -529,7 +529,7 @@ class Volshell(interfaces.plugins.PluginInterface): val, interfaces.configuration.BasicTypes ) and not isinstance(val, list): if not isinstance(val, list) or all( - [isinstance(x, interfaces.configuration.BasicTypes) for x in val] + isinstance(x, interfaces.configuration.BasicTypes) for x in val ): raise TypeError( "Configurable values must be simple types (int, bool, str, bytes)" diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 06b2111b4..7f38a23e1 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -209,10 +209,8 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): try: kvp = vlayer.mapping(kvo, 0) if any( - [ - (p == kernel["mz_offset"] and layer_name == physical_layer_name) - for (_, _, p, _, layer_name) in kvp - ] + (p == kernel["mz_offset"] and layer_name == physical_layer_name) + for (_, _, p, _, layer_name) in kvp ): return (virtual_layer_name, kvo, kernel) else: diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index dcb7505ea..86e1aac52 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -161,7 +161,7 @@ class ListRequirement(interfaces.configuration.RequirementInterface): "TypeError - Too many values provided to list option.", ) return {config_path: self} - if not all([isinstance(element, self.element_type) for element in value]): + if not all(isinstance(element, self.element_type) for element in value): vollog.log( constants.LOGLEVEL_V, "TypeError - At least one element in the list is not of the correct type.", @@ -181,7 +181,7 @@ class ChoiceRequirement(interfaces.configuration.RequirementInterface): """ super().__init__(*args, **kwargs) if not isinstance(choices, list) or any( - [not isinstance(choice, str) for choice in choices] + not isinstance(choice, str) for choice in choices ): raise TypeError("ChoiceRequirement takes a list of strings as choices") self.choices = choices @@ -410,11 +410,9 @@ class TranslationLayerRequirement( args = {"context": context, "config_path": config_path, "name": name} if any( - [ - subreq.unsatisfied(context, config_path) - for subreq in self.requirements.values() - if not subreq.optional - ] + subreq.unsatisfied(context, config_path) + for subreq in self.requirements.values() + if not subreq.optional ): return None @@ -485,11 +483,9 @@ class SymbolTableRequirement( args = {"context": context, "config_path": config_path, "name": name} if any( - [ - subreq.unsatisfied(context, config_path) - for subreq in self.requirements.values() - if not subreq.optional - ] + subreq.unsatisfied(context, config_path) + for subreq in self.requirements.values() + if not subreq.optional ): return None @@ -710,11 +706,9 @@ class ModuleRequirement( args = {"context": context, "config_path": config_path, "name": name} if any( - [ - subreq.unsatisfied(context, config_path) - for subreq in self.requirements.values() - if not subreq.optional - ] + subreq.unsatisfied(context, config_path) + for subreq in self.requirements.values() + if not subreq.optional ): return None diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index ab568b927..51d25510d 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -216,7 +216,7 @@ class ObjectInterface(metaclass=abc.ABCMeta): Args: member_names: List of names to test as to members with those names validity """ - return all([self.has_valid_member(member_name) for member_name in member_names]) + return all(self.has_valid_member(member_name) for member_name in member_names) class VolTemplateProxy(metaclass=abc.ABCMeta): """A container for proxied methods that the ObjectTemplate of this diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index b969b952b..2b0df5372 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -270,10 +270,8 @@ class Intel(linear.LinearlyMappedLayer): try: # TODO: Consider reimplementing this, since calls to mapping can call is_valid return all( - [ - self._context.layers[layer].is_valid(mapped_offset) - for _, _, mapped_offset, _, layer in self.mapping(offset, length) - ] + self._context.layers[layer].is_valid(mapped_offset) + for _, _, mapped_offset, _, layer in self.mapping(offset, length) ) except exceptions.InvalidAddressException: return False diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 9841d2bb0..471d19f68 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -318,10 +318,8 @@ class RegistryHive(linear.LinearlyMappedLayer): with contextlib.suppress(exceptions.InvalidAddressException): # Pass this to the lower layers for now return all( - [ - self.context.layers[layer].is_valid(offset, length) - for (_, _, offset, length, layer) in self.mapping(offset, length) - ] + self.context.layers[layer].is_valid(offset, length) + for (_, _, offset, length, layer) in self.mapping(offset, length) ) return False diff --git a/volatility3/framework/layers/segmented.py b/volatility3/framework/layers/segmented.py index e8c067072..9825ae15c 100644 --- a/volatility3/framework/layers/segmented.py +++ b/volatility3/framework/layers/segmented.py @@ -51,10 +51,8 @@ class NonLinearlySegmentedLayer( try: base_layer = self._context.layers[self._base_layer] return all( - [ - base_layer.is_valid(mapped_offset) - for _i, _i, mapped_offset, _i, _s in self.mapping(offset, length) - ] + base_layer.is_valid(mapped_offset) + for _i, _i, mapped_offset, _i, _s in self.mapping(offset, length) ) except exceptions.InvalidAddressException: return False diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 316a30bec..b65277067 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -928,10 +928,8 @@ class AggregateType(interfaces.objects.ObjectInterface): members, collections.abc.Mapping ), f"{agg_name} members parameter must be a mapping: {type(members)}" assert all( - [ - (isinstance(member, tuple) and len(member) == 2) - for member in members.values() - ] + (isinstance(member, tuple) and len(member) == 2) + for member in members.values() ), f"{agg_name} members must be a tuple of relative_offsets and templates" def member(self, attr: str = "member") -> object: diff --git a/volatility3/framework/plugins/linux/check_afinfo.py b/volatility3/framework/plugins/linux/check_afinfo.py index 7fced6acd..201a443f7 100644 --- a/volatility3/framework/plugins/linux/check_afinfo.py +++ b/volatility3/framework/plugins/linux/check_afinfo.py @@ -53,9 +53,7 @@ class Check_afinfo(plugins.PluginInterface): def _check_afinfo(self, var_name, var, op_members, seq_members): # check if object has a least one of the members used for analysis by this function required_members = ["seq_fops", "seq_ops", "seq_show"] - has_required_member = any( - [var.has_member(member) for member in required_members] - ) + has_required_member = any(var.has_member(member) for member in required_members) if not has_required_member: vollog.debug( f"{var_name} object at {hex(var.vol.offset)} had none of the required members: {', '.join([member for member in required_members])}" diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index d1cb9f460..c754e43ef 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -66,7 +66,7 @@ class Timeliner(interfaces.plugins.PluginInterface): if selected_list: def filter_plugins(name: str, selected: List[str]) -> bool: - return any([s in name for s in selected]) + return any(s in name for s in selected) filter_func = filter_plugins else: @@ -252,10 +252,8 @@ class Timeliner(interfaces.plugins.PluginInterface): continue if filter_list and not any( - [ - filter in plugin_class.__module__ + "." + plugin_class.__name__ - for filter in filter_list - ] + filter in plugin_class.__module__ + "." + plugin_class.__name__ + for filter in filter_list ): continue diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 149829371..95f542f07 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -24,8 +24,8 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): version = self._json_data.get("version", None) if not version: return None - if all([x in "0123456789." for x in version]): - return tuple([int(x) for x in version.split(".")]) + if all(x in "0123456789." for x in version): + return tuple(int(x) for x in version.split(".")) vollog.log( constants.LOGLEVEL_VVVV, f"Metadata version contains unexpected characters: '{version}'", From a63e9f1699e4042d1372c0873b05f207e2a8a8d4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 2 Oct 2024 13:00:45 +1000 Subject: [PATCH 432/979] Add --remote-isf-url argument to be able mutually exclusive to --offline --- volatility3/cli/__init__.py | 11 ++++++++++- volatility3/cli/volshell/__init__.py | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 1209d7cdc..674fb4b4b 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -235,12 +235,19 @@ class CommandLine: default=constants.CACHE_PATH, type=str, ) - parser.add_argument( + isf_group = parser.add_mutually_exclusive_group() + isf_group.add_argument( "--offline", help="Do not search online for additional JSON files", default=False, action="store_true", ) + isf_group.add_argument( + "--remote-isf-url", + help="Search online for ISF json files", + default=constants.REMOTE_ISF_URL, + type=str, + ) parser.add_argument( "--filters", help="List of filters to apply to the output (in the form of [+-]columname,pattern[!])", @@ -313,6 +320,8 @@ class CommandLine: if partial_args.offline: constants.OFFLINE = partial_args.offline + elif partial_args.remote_isf_url: + constants.REMOTE_ISF_URL = partial_args.remote_isf_url # Do the initialization ctx = contexts.Context() # Construct a blank context diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 2bf1958e2..fc2f87bc2 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -159,12 +159,19 @@ class VolShell(cli.CommandLine): default=constants.CACHE_PATH, type=str, ) - parser.add_argument( + isf_group = parser.add_mutually_exclusive_group() + isf_group.add_argument( "--offline", help="Do not search online for additional JSON files", default=False, action="store_true", ) + isf_group.add_argument( + "--remote-isf-url", + help="Search online for ISF json files", + default=constants.REMOTE_ISF_URL, + type=str, + ) # Volshell specific flags os_specific = parser.add_mutually_exclusive_group(required=False) @@ -236,6 +243,8 @@ class VolShell(cli.CommandLine): if partial_args.offline: constants.OFFLINE = partial_args.offline + elif partial_args.remote_isf_url: + constants.REMOTE_ISF_URL = partial_args.remote_isf_url # Do the initialization ctx = contexts.Context() # Construct a blank context From a64813be43594d5171748c901e1c10e2c2271ed0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 2 Oct 2024 13:08:05 +1000 Subject: [PATCH 433/979] Added metavar and single letter argument to the --remote-isf-url --- volatility3/cli/__init__.py | 2 ++ volatility3/cli/volshell/__init__.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 674fb4b4b..50b5579af 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -243,7 +243,9 @@ class CommandLine: action="store_true", ) isf_group.add_argument( + "-u", "--remote-isf-url", + metavar="URL", help="Search online for ISF json files", default=constants.REMOTE_ISF_URL, type=str, diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index fc2f87bc2..5172c5363 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -167,7 +167,9 @@ class VolShell(cli.CommandLine): action="store_true", ) isf_group.add_argument( + "-u", "--remote-isf-url", + metavar="URL", help="Search online for ISF json files", default=constants.REMOTE_ISF_URL, type=str, From 57b5aabdfe885dd054ceb7d1cd1227b46f30ac22 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 3 Oct 2024 11:50:03 +1000 Subject: [PATCH 434/979] Add linux.ptrace plugin to enumerate tracer and tracee tasks --- .../framework/constants/linux/__init__.py | 39 +++++- volatility3/framework/plugins/linux/ptrace.py | 120 ++++++++++++++++++ 2 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 volatility3/framework/plugins/linux/ptrace.py diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 3eabc2341..0567b8574 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -5,7 +5,7 @@ Linux-specific values that aren't found in debug symbols """ -from enum import IntEnum +from enum import IntEnum, Flag KERNEL_NAME = "__kernel__" @@ -302,3 +302,40 @@ class ELF_CLASS(IntEnum): ELFCLASSNONE = 0 ELFCLASS32 = 1 ELFCLASS64 = 2 + + +PT_OPT_FLAG_SHIFT = 3 + +PTRACE_EVENT_FORK = 1 +PTRACE_EVENT_VFORK = 2 +PTRACE_EVENT_CLONE = 3 +PTRACE_EVENT_EXEC = 4 +PTRACE_EVENT_VFORK_DONE = 5 +PTRACE_EVENT_EXIT = 6 +PTRACE_EVENT_SECCOMP = 7 + +PTRACE_O_EXITKILL = 1 << 20 +PTRACE_O_SUSPEND_SECCOMP = 1 << 21 + + +class PT_FLAGS(Flag): + "PTrace flags" + PT_PTRACED = 0x00001 + PT_SEIZED = 0x10000 + + PT_TRACESYSGOOD = 1 << (PT_OPT_FLAG_SHIFT + 0) + PT_TRACE_FORK = 1 << (PT_OPT_FLAG_SHIFT + PTRACE_EVENT_FORK) + PT_TRACE_VFORK = 1 << (PT_OPT_FLAG_SHIFT + PTRACE_EVENT_VFORK) + PT_TRACE_CLONE = 1 << (PT_OPT_FLAG_SHIFT + PTRACE_EVENT_CLONE) + PT_TRACE_EXEC = 1 << (PT_OPT_FLAG_SHIFT + PTRACE_EVENT_EXEC) + PT_TRACE_VFORK_DONE = 1 << (PT_OPT_FLAG_SHIFT + PTRACE_EVENT_VFORK_DONE) + PT_TRACE_EXIT = 1 << (PT_OPT_FLAG_SHIFT + PTRACE_EVENT_EXIT) + PT_TRACE_SECCOMP = 1 << (PT_OPT_FLAG_SHIFT + PTRACE_EVENT_SECCOMP) + + PT_EXITKILL = PTRACE_O_EXITKILL << PT_OPT_FLAG_SHIFT + PT_SUSPEND_SECCOMP = PTRACE_O_SUSPEND_SECCOMP << PT_OPT_FLAG_SHIFT + + @property + def flags(self) -> str: + """Returns the ptrace flags string""" + return str(self).replace(self.__class__.__name__ + ".", "") diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py new file mode 100644 index 000000000..793a1b3a1 --- /dev/null +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -0,0 +1,120 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import List + +from volatility3.framework import renderers, interfaces, constants, objects +from volatility3.framework.constants.linux import PT_FLAGS +from volatility3.framework.constants.architectures import LINUX_ARCHS +from volatility3.framework.objects import utility +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + + +class Ptrace(plugins.PluginInterface): + """Enumerates tracer and tracee tasks""" + + _required_framework_version = (2, 10, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=LINUX_ARCHS, + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 2, 1) + ), + ] + + @classmethod + def enumerate_ptraced_tasks( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + ): + vmlinux = context.modules[symbol_table] + + tsk_struct_symname = vmlinux.symbol_table_name + constants.BANG + "task_struct" + + tasks = pslist.PsList.list_tasks( + context, + symbol_table, + filter_func=pslist.PsList.create_pid_filter(), + include_threads=True, + ) + + for task in tasks: + tracing_tid_list = [ + int(task_being_traced.pid) + for task_being_traced in task.ptraced.to_list( + tsk_struct_symname, "ptrace_entry" + ) + ] + + if task.ptrace == 0 and not tracing_tid_list: + continue + + flags = ( + PT_FLAGS(task.ptrace).flags + if task.ptrace != 0 + else renderers.NotAvailableValue() + ) + + traced_by_tid = ( + task.parent.pid + if task.real_parent != task.parent + else renderers.NotAvailableValue() + ) + + tracing_tids = ",".join(map(str, tracing_tid_list)) + + yield task.comm, task.tgid, task.pid, traced_by_tid, tracing_tids, flags + + def _generator(self, symbol_table): + for fields in self.enumerate_ptraced_tasks(self.context, symbol_table): + yield (0, fields) + + @staticmethod + def format_fields_with_headers(headers, generator): + """Uses the headers type to cast the fields obtained from the generator""" + for level, fields in generator: + formatted_fields = [] + for header, field in zip(headers, fields): + header_type = header[1] + + if isinstance( + field, (header_type, interfaces.renderers.BaseAbsentValue) + ): + formatted_field = field + elif isinstance(field, objects.Array) and header_type is str: + formatted_field = utility.array_to_string(field) + else: + formatted_field = header_type(field) + + formatted_fields.append(formatted_field) + yield level, formatted_fields + + def run(self): + symbol_table = self.config["kernel"] + + headers = [ + ("Process", str), + ("PID", int), + ("TID", int), + ("Traced by TID", int), + ("Tracing TIDs", str), + ("Flags", str), + ] + return renderers.TreeGrid( + headers, + self.format_fields_with_headers(headers, self._generator(symbol_table)), + ) From 590aa9c19ca56250e87c07ed49dbddc55a22b4a8 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 3 Oct 2024 14:50:39 +1000 Subject: [PATCH 435/979] Make it callable from other plugins. Additionally, classmethod helpers were added, and docstrings were enhanced for improved usability and clarity. --- .../framework/plugins/linux/hidden_modules.py | 154 +++++++++++++----- 1 file changed, 116 insertions(+), 38 deletions(-) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index 6d04c08af..4843f3037 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -2,10 +2,9 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import re -import functools import logging import contextlib -from typing import List, Iterable +from typing import List, Set, Tuple, Iterable from volatility3.framework import renderers, interfaces, exceptions, objects from volatility3.framework.constants.architectures import LINUX_ARCHS from volatility3.framework.renderers import format_hints @@ -41,7 +40,21 @@ class Hidden_modules(interfaces.plugins.PluginInterface): ), ] - def _get_modules_memory_boundaries(self, vmlinux): + @staticmethod + def get_modules_memory_boundaries( + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Tuple[int]: + """Determine the boundaries of the module allocation area + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + + Returns: + A tuple containing the minimum and maximum addresses for the module allocation area. + """ + vmlinux = context.modules[vmlinux_module_name] if vmlinux.has_symbol("mod_tree"): mod_tree = vmlinux.object_from_symbol("mod_tree") modules_addr_min = mod_tree.addr_min @@ -73,8 +86,8 @@ class Hidden_modules(interfaces.plugins.PluginInterface): return modules_addr_min, modules_addr_max + @staticmethod def _get_module_state_values_bytes( - self, context: interfaces.context.ContextInterface, vmlinux_module_name: str, ) -> List[bytes]: @@ -97,12 +110,13 @@ class Hidden_modules(interfaces.plugins.PluginInterface): ] return values_bytes - def get_hidden_modules_vol2( - self, + @classmethod + def _get_hidden_modules_vol2( + cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, - known_module_addresses, - modules_memory_boundaries: tuple, + known_module_addresses: Set[int], + modules_memory_boundaries: Tuple, ) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerate hidden modules using the traditional implementation. @@ -111,6 +125,9 @@ class Hidden_modules(interfaces.plugins.PluginInterface): Args: context: The context to retrieve required elements (layers, symbol tables) from vmlinux_module_name: The name of the kernel module on which to operate + known_module_addresses: Set with known module addresses + modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. + Yields: module objects """ @@ -175,7 +192,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): scan_buf = b"".join(scan_list) del scan_list - module_state_values_bytes = self._get_module_state_values_bytes( + module_state_values_bytes = cls._get_module_state_values_bytes( context, vmlinux_module_name ) values_bytes_pattern = b"|".join(module_state_values_bytes) @@ -190,27 +207,37 @@ class Hidden_modules(interfaces.plugins.PluginInterface): if module and module.is_valid(): yield module - @functools.cached_property - def module_address_alignment(self) -> int: + @classmethod + def _get_module_address_alignment( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> int: """Obtain the module memory address alignment. This is only used with the fast scan method. struct module is aligned to the L1 cache line, which is typically 64 bytes for most common i386/AMD64/ARM64 configurations. In some cases, it can be 128 bytes, but this will still work. + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + Returns: The struct module alignment """ # FIXME: When dwarf2json/ISF supports type alignments. Read it directly from the type metadata - # The cached_property won't provide any benefits until then + # Also, 'context' and 'vmlinux_module_name' are not used yet, but they will be needed to obtain + # the type metadata return 64 - def get_hidden_modules_fast( - self, + @classmethod + def _get_hidden_modules_fast( + cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, - known_module_addresses, - modules_memory_boundaries: tuple, + known_module_addresses: Set[int], + modules_memory_boundaries: Tuple, ) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerate hidden modules by taking advantage of memory address alignment patterns @@ -229,6 +256,9 @@ class Hidden_modules(interfaces.plugins.PluginInterface): Args: context: The context to retrieve required elements (layers, symbol tables) from vmlinux_module_name: The name of the kernel module on which to operate + known_module_addresses: Set with known module addresses + modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. + Yields: module objects """ @@ -237,12 +267,16 @@ class Hidden_modules(interfaces.plugins.PluginInterface): module_addr_min, module_addr_max = modules_memory_boundaries - module_state_values_bytes = self._get_module_state_values_bytes( + module_state_values_bytes = cls._get_module_state_values_bytes( + context, vmlinux_module_name + ) + + module_address_alignment = cls._get_module_address_alignment( context, vmlinux_module_name ) for module_addr in range( - module_addr_min, module_addr_max, self.module_address_alignment + module_addr_min, module_addr_max, module_address_alignment ): if module_addr in known_module_addresses: continue @@ -264,51 +298,59 @@ class Hidden_modules(interfaces.plugins.PluginInterface): if module and module.is_valid(): yield module - def _validate_alignment_patterns(self, addresses: Iterable[int]) -> bool: + @staticmethod + def _validate_alignment_patterns( + addresses: Iterable[int], + address_alignment: int, + ) -> bool: """Check if the memory addresses meet our alignments patterns Args: addresses: Iterable with the address values + address_alignment: Number of bytes for alignment validation Returns: True if all the addresses meet the alignment """ - return all(addr % self.module_address_alignment == 0 for addr in addresses) + return all(addr % address_alignment == 0 for addr in addresses) + @classmethod def get_hidden_modules( - self, + cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, + known_module_addresses: Set[int], + modules_memory_boundaries: Tuple, + fast_method: bool = False, ) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerate hidden modules Args: context: The context to retrieve required elements (layers, symbol tables) from vmlinux_module_name: The name of the kernel module on which to operate + known_module_addresses: Set with known module addresses + modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. + fast_method: If True, it uses the fast method. Otherwise, it uses the traditional one. + Yields: module objects """ - vmlinux = context.modules[vmlinux_module_name] - vmlinux_layer = context.layers[vmlinux.layer_name] - - known_module_addresses = { - vmlinux_layer.canonicalize(module.vol.offset) - for module in lsmod.Lsmod.list_modules(context, vmlinux_module_name) - } - - modules_memory_boundaries = self._get_modules_memory_boundaries(vmlinux) - - if self.config.get("fast"): - if self._validate_alignment_patterns(known_module_addresses): - scan_method = self.get_hidden_modules_fast + if fast_method: + module_address_alignment = cls._get_module_address_alignment( + context, vmlinux_module_name + ) + if cls._validate_alignment_patterns( + known_module_addresses, module_address_alignment + ): + scan_method = cls._get_hidden_modules_fast else: vollog.warning( - f"Module addresses aren't aligned to {self.module_address_alignment} bytes. " + f"Module addresses aren't aligned to {module_address_alignment} bytes. " "Switching to the traditional scan method." ) - scan_method = self.get_hidden_modules_vol2 + scan_method = cls._get_hidden_modules_vol2 else: - scan_method = self.get_hidden_modules_vol2 + scan_method = cls._get_hidden_modules_vol2 yield from scan_method( context, @@ -317,9 +359,45 @@ class Hidden_modules(interfaces.plugins.PluginInterface): modules_memory_boundaries, ) + @classmethod + def get_lsmod_module_addresses( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Set[int]: + """Obtain a set the known module addresses from linux.lsmod plugin + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + + Returns: + A set containing known kernel module addresses + """ + vmlinux = context.modules[vmlinux_module_name] + vmlinux_layer = context.layers[vmlinux.layer_name] + + known_module_addresses = { + vmlinux_layer.canonicalize(module.vol.offset) + for module in lsmod.Lsmod.list_modules(context, vmlinux_module_name) + } + return known_module_addresses + def _generator(self): vmlinux_module_name = self.config["kernel"] - for module in self.get_hidden_modules(self.context, vmlinux_module_name): + known_module_addresses = self.get_lsmod_module_addresses( + self.context, vmlinux_module_name + ) + modules_memory_boundaries = self.get_modules_memory_boundaries( + self.context, vmlinux_module_name + ) + for module in self.get_hidden_modules( + self.context, + vmlinux_module_name, + known_module_addresses, + modules_memory_boundaries, + fast_method=self.config.get("fast"), + ): module_addr = module.vol.offset module_name = module.get_name() or renderers.NotAvailableValue() fields = (format_hints.Hex(module_addr), module_name) From 8d925bd8ad6346370d6256ddacced9d0db36b23d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 3 Oct 2024 15:17:02 +1000 Subject: [PATCH 436/979] Added the --heuristic-mode option, which relaxes constraints to improve detection of more advanced threats --- .../framework/plugins/linux/hidden_modules.py | 46 ++++++++++++------- .../symbols/linux/extensions/__init__.py | 4 +- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index 4843f3037..1ec43aefd 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -38,6 +38,13 @@ class Hidden_modules(interfaces.plugins.PluginInterface): optional=True, default=False, ), + requirements.BooleanRequirement( + name="heuristic-mode", + description="Relaxed constraints. This may generate some false positives and" + "take a bit longer. This feature is available only when using the --fast option", + optional=True, + default=False, + ), ] @staticmethod @@ -117,6 +124,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_module_name: str, known_module_addresses: Set[int], modules_memory_boundaries: Tuple, + heuristic_mode: bool = False, ) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerate hidden modules using the traditional implementation. @@ -127,6 +135,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_module_name: The name of the kernel module on which to operate known_module_addresses: Set with known module addresses modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. + heuristic_mode: ignored for this scan method. Yields: module objects @@ -227,8 +236,8 @@ class Hidden_modules(interfaces.plugins.PluginInterface): The struct module alignment """ # FIXME: When dwarf2json/ISF supports type alignments. Read it directly from the type metadata - # Also, 'context' and 'vmlinux_module_name' are not used yet, but they will be needed to obtain - # the type metadata + # Additionally, while 'context' and 'vmlinux_module_name' are currently unused, they will be + # essential for retrieving type metadata in the future. return 64 @classmethod @@ -238,6 +247,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_module_name: str, known_module_addresses: Set[int], modules_memory_boundaries: Tuple, + heuristic_mode: bool = False, ) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerate hidden modules by taking advantage of memory address alignment patterns @@ -258,7 +268,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_module_name: The name of the kernel module on which to operate known_module_addresses: Set with known module addresses modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. - + heuristic_mode: If True, it loosens constraints to enhance the detection of advanced threats. Yields: module objects """ @@ -281,21 +291,22 @@ class Hidden_modules(interfaces.plugins.PluginInterface): if module_addr in known_module_addresses: continue - try: - # This is just a pre-filter. Module readability and consistency are verified in module.is_valid() - module_state_bytes = vmlinux_layer.read( - module_addr, len(module_state_values_bytes[0]) - ) - if module_state_bytes not in module_state_values_bytes: + if not heuristic_mode: + try: + # This is just a pre-filter. Module readability and consistency are verified in module.is_valid() + module_state_bytes = vmlinux_layer.read( + module_addr, len(module_state_values_bytes[0]) + ) + if module_state_bytes not in module_state_values_bytes: + continue + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ): continue - except ( - exceptions.PagedInvalidAddressException, - exceptions.InvalidAddressException, - ): - continue module = vmlinux.object("module", offset=module_addr, absolute=True) - if module and module.is_valid(): + if module and module.is_valid(strict_states=not heuristic_mode): yield module @staticmethod @@ -322,6 +333,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): known_module_addresses: Set[int], modules_memory_boundaries: Tuple, fast_method: bool = False, + heuristic_mode: bool = False, ) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerate hidden modules @@ -331,7 +343,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): known_module_addresses: Set with known module addresses modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. fast_method: If True, it uses the fast method. Otherwise, it uses the traditional one. - + heuristic_mode: If True, it loosens constraints to enhance the detection of advanced threats. Yields: module objects """ @@ -357,6 +369,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_module_name, known_module_addresses, modules_memory_boundaries, + heuristic_mode, ) @classmethod @@ -397,6 +410,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): known_module_addresses, modules_memory_boundaries, fast_method=self.config.get("fast"), + heuristic_mode=self.config.get("heuristic-mode"), ): module_addr = module.vol.offset module_name = module.get_name() or renderers.NotAvailableValue() diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b30634ca3..810df41f8 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -35,13 +35,13 @@ class module(generic.GenericIntelProcess): super().__init__(*args, **kwargs) self._mod_mem_type = None # Initialize _mod_mem_type to None for memoization - def is_valid(self): + def is_valid(self, strict_states=True): layer = self._context.layers[self.vol.layer_name] # Make sure the entire module content is readable if not layer.is_valid(self.vol.offset, self.vol.size): return False - if not self.state.is_valid_choice: + if strict_states and not self.state.is_valid_choice: return False core_size = self.get_core_size() From e8754fadbd93864ea2623e7df10bb33265db5073 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 3 Oct 2024 15:50:30 +1000 Subject: [PATCH 437/979] Fix typo in usage help --- volatility3/framework/plugins/linux/hidden_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index 1ec43aefd..86a4c3626 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -40,7 +40,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): ), requirements.BooleanRequirement( name="heuristic-mode", - description="Relaxed constraints. This may generate some false positives and" + description="Relaxed constraints. This may generate false positives and " "take a bit longer. This feature is available only when using the --fast option", optional=True, default=False, From 4ac9b0825a6a6b7548dfcb36cc2681c9b4753f3c Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 3 Oct 2024 10:49:38 -0500 Subject: [PATCH 438/979] Linux: Properly get paths in simple_dname paths Some dentry paths were not being recovered from `simple_dname` paths, this fixes the issue by properly dereferencing the string pointer. --- volatility3/framework/symbols/linux/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 2410d2627..1aceade24 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -206,7 +206,14 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): pre_name = "pipe" elif sym == "simple_dname": - pre_name = cls._get_path_file(task, filp) + name = dentry.d_name.name + if name: + pre_name = name.dereference().cast( + "string", max_length=255, errors="replace" + ) + return "/" + pre_name + " (deleted)" + else: + pre_name = "" elif sym == "ns_dname": # From Kernels 3.19 From e605bee4119e5c7da857ce9576520c36a7eb4b19 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 6 Sep 2024 14:07:24 -0500 Subject: [PATCH 439/979] Windows: unloadedmodules bugfix This fixes a bug in the `windows.unloadedmodules` plugin. An `InvalidAddressException` can be raised when parsing the module name; this adds a try/except block to replace the missing module name with a `renderers.UnreadableValue` when the exception occurs. --- volatility3/framework/plugins/windows/unloadedmodules.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/unloadedmodules.py b/volatility3/framework/plugins/windows/unloadedmodules.py index 0d88e96ff..01e575818 100644 --- a/volatility3/framework/plugins/windows/unloadedmodules.py +++ b/volatility3/framework/plugins/windows/unloadedmodules.py @@ -7,7 +7,7 @@ import datetime from typing import List, Iterable from volatility3.framework import constants -from volatility3.framework import interfaces, symbols +from volatility3.framework import interfaces, symbols, exceptions from volatility3.framework import renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import configuration @@ -132,10 +132,15 @@ class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInt kernel.symbol_table_name, unloadedmodule_table_name, ): + try: + name = mod.Name.String + except exceptions.InvalidAddressException: + name = renderers.UnreadableValue() + yield ( 0, ( - mod.Name.String, + name, format_hints.Hex(mod.StartAddress), format_hints.Hex(mod.EndAddress), conversion.wintime_to_datetime(mod.CurrentTime), From b5948d7ec0e9bb6dcc193f8c74b40bdb498ac3a8 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 4 Oct 2024 07:48:25 +1000 Subject: [PATCH 440/979] Linux: hidden_modules: Add @Abyss-W4tcher suggestion to optimize the fast scan method for even better performance, using the mkobj.mod self referential validation used in module.is_valid() as pre-filter Removed the --heuristic-mode and the module.states validation, since the self referential check is enough by itself --- .../framework/plugins/linux/hidden_modules.py | 53 ++++++++----------- .../symbols/linux/extensions/__init__.py | 20 +++---- 2 files changed, 30 insertions(+), 43 deletions(-) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index 86a4c3626..a76d1c69d 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -38,13 +38,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): optional=True, default=False, ), - requirements.BooleanRequirement( - name="heuristic-mode", - description="Relaxed constraints. This may generate false positives and " - "take a bit longer. This feature is available only when using the --fast option", - optional=True, - default=False, - ), ] @staticmethod @@ -124,7 +117,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_module_name: str, known_module_addresses: Set[int], modules_memory_boundaries: Tuple, - heuristic_mode: bool = False, ) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerate hidden modules using the traditional implementation. @@ -135,7 +127,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_module_name: The name of the kernel module on which to operate known_module_addresses: Set with known module addresses modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. - heuristic_mode: ignored for this scan method. Yields: module objects @@ -247,7 +238,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_module_name: str, known_module_addresses: Set[int], modules_memory_boundaries: Tuple, - heuristic_mode: bool = False, ) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerate hidden modules by taking advantage of memory address alignment patterns @@ -268,7 +258,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_module_name: The name of the kernel module on which to operate known_module_addresses: Set with known module addresses modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. - heuristic_mode: If True, it loosens constraints to enhance the detection of advanced threats. Yields: module objects """ @@ -276,37 +265,41 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_layer = context.layers[vmlinux.layer_name] module_addr_min, module_addr_max = modules_memory_boundaries - - module_state_values_bytes = cls._get_module_state_values_bytes( - context, vmlinux_module_name - ) - module_address_alignment = cls._get_module_address_alignment( context, vmlinux_module_name ) + mkobj_offset = vmlinux.get_type("module").relative_child_offset("mkobj") + mod_offset = vmlinux.get_type("module_kobject").relative_child_offset("mod") + offset_to_mkobj_mod = mkobj_offset + mod_offset + mod_member_template = vmlinux.get_type("module_kobject").vol.members["mod"][1] + mod_size = mod_member_template.size + mod_member_data_format = mod_member_template.data_format + for module_addr in range( module_addr_min, module_addr_max, module_address_alignment ): if module_addr in known_module_addresses: continue - if not heuristic_mode: - try: - # This is just a pre-filter. Module readability and consistency are verified in module.is_valid() - module_state_bytes = vmlinux_layer.read( - module_addr, len(module_state_values_bytes[0]) - ) - if module_state_bytes not in module_state_values_bytes: - continue - except ( - exceptions.PagedInvalidAddressException, - exceptions.InvalidAddressException, - ): + try: + # This is just a pre-filter. Module readability and consistency are verified in module.is_valid() + self_referential_bytes = vmlinux_layer.read( + module_addr + offset_to_mkobj_mod, mod_size + ) + self_referential = objects.convert_data_to_value( + self_referential_bytes, int, mod_member_data_format + ) + if self_referential != module_addr: continue + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ): + continue module = vmlinux.object("module", offset=module_addr, absolute=True) - if module and module.is_valid(strict_states=not heuristic_mode): + if module and module.is_valid(): yield module @staticmethod @@ -369,7 +362,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_module_name, known_module_addresses, modules_memory_boundaries, - heuristic_mode, ) @classmethod @@ -410,7 +402,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): known_module_addresses, modules_memory_boundaries, fast_method=self.config.get("fast"), - heuristic_mode=self.config.get("heuristic-mode"), ): module_addr = module.vol.offset module_name = module.get_name() or renderers.NotAvailableValue() diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 810df41f8..9f6622001 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -35,15 +35,12 @@ class module(generic.GenericIntelProcess): super().__init__(*args, **kwargs) self._mod_mem_type = None # Initialize _mod_mem_type to None for memoization - def is_valid(self, strict_states=True): + def is_valid(self): layer = self._context.layers[self.vol.layer_name] # Make sure the entire module content is readable if not layer.is_valid(self.vol.offset, self.vol.size): return False - if strict_states and not self.state.is_valid_choice: - return False - core_size = self.get_core_size() if not ( 1 <= core_size <= 20000000 @@ -52,14 +49,13 @@ class module(generic.GenericIntelProcess): ): return False - if self.has_member("mkobj") and self.mkobj.has_member("mod"): - if not ( - self.mkobj - and self.mkobj.mod - and self.mkobj.mod.is_readable() - and self.mkobj.mod == self.vol.offset - ): - return False + if not ( + self.mkobj + and self.mkobj.mod + and self.mkobj.mod.is_readable() + and self.mkobj.mod == self.vol.offset + ): + return False return True From f455c30e4962789e5cac595f57e9ea5090fac6b6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 4 Oct 2024 09:51:08 +1000 Subject: [PATCH 441/979] Linux: hidden_modules: remove missed optional heuristic_mode argument --- volatility3/framework/plugins/linux/hidden_modules.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index a76d1c69d..6542f58f3 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -326,7 +326,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): known_module_addresses: Set[int], modules_memory_boundaries: Tuple, fast_method: bool = False, - heuristic_mode: bool = False, ) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerate hidden modules @@ -336,7 +335,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): known_module_addresses: Set with known module addresses modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. fast_method: If True, it uses the fast method. Otherwise, it uses the traditional one. - heuristic_mode: If True, it loosens constraints to enhance the detection of advanced threats. Yields: module objects """ From cffad872bbae6fda31eb415de230e445603c451c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 4 Oct 2024 12:22:53 +1000 Subject: [PATCH 442/979] Linux: sockstat. Fix #1271 which unnecessarily extends the interface of the list_sockets() class method --- volatility3/framework/plugins/linux/sockstat.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index bb9c46fad..0ddd3e26d 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -22,7 +22,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (3, 0, 0) def __init__(self, vmlinux, task, *args, **kwargs): super().__init__(*args, **kwargs) @@ -439,7 +439,7 @@ class Sockstat(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (3, 0, 0) @classmethod def get_requirements(cls): @@ -450,7 +450,7 @@ class Sockstat(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name="SockHandlers", component=SockHandlers, version=(2, 0, 0) + name="SockHandlers", component=SockHandlers, version=(3, 0, 0) ), requirements.PluginRequirement( name="lsof", plugin=lsof.Lsof, version=(2, 0, 0) @@ -550,7 +550,7 @@ class Sockstat(plugins.PluginInterface): except AttributeError: netns_id = NotAvailableValue() - yield task_comm, task, netns_id, fd_num, family, sock_type, protocol, sock_fields + yield task, netns_id, fd_num, family, sock_type, protocol, sock_fields def _format_fields(self, sock_stat, protocol): """Prepare the socket fields to be rendered @@ -597,7 +597,6 @@ class Sockstat(plugins.PluginInterface): ) for ( - task_comm, task, netns_id, fd_num, @@ -618,6 +617,8 @@ class Sockstat(plugins.PluginInterface): else NotAvailableValue() ) + task_comm = utility.array_to_string(task.comm) + fields = ( netns_id, task_comm, From bb5a4475028c95af0e9d1e583d749ac9d01b82f3 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Sat, 5 Oct 2024 09:58:01 -0500 Subject: [PATCH 443/979] #1292 - use correct sector size for DataSectionObject file dumping --- .../framework/symbols/windows/extensions/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 80d3b48b7..ecfc2f163 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1247,6 +1247,15 @@ class CONTROL_AREA(objects.StructType): is_64bit = symbols.symbol_table_is_64bit(self._context, symbol_table_name) is_pae = self._context.layers[self.vol.layer_name].metadata.get("pae", False) + # the sector_size is used as a multiplier to the StartingSector + # within each _SUBSECTION. ImageSectionObjects use a multiplier + # of 0x200 corresponding to sector alignment on disk, + # while DataSectionObjects use a multiplier of 0x1000 corresponding + # to the size of a page + sector_size = 0x200 + if self.u.Flags.Image != 1: + sector_size = 0x1000 + # This is a null-terminated single-linked list. while subsection != 0: try: @@ -1257,7 +1266,7 @@ class CONTROL_AREA(objects.StructType): # The offset into the file is stored implicitly based on the PTE location within the Subsection. starting_sector = subsection.StartingSector - subsection_offset = starting_sector * 0x200 + subsection_offset = starting_sector * sector_size # Similar to the check in is_valid(), make sure the SubsectionBase is not page aligned. # if subsection.SubsectionBase & self.PAGE_MASK == 0: From 5d072864d726242e85e632edb99449cafaf187dd Mon Sep 17 00:00:00 2001 From: eve Date: Mon, 7 Oct 2024 07:00:05 +0100 Subject: [PATCH 444/979] Docs: Add warning around LiME raw format --- doc/source/getting-started-linux-tutorial.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst index 26ad2c2e4..d4b40d053 100644 --- a/doc/source/getting-started-linux-tutorial.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -11,6 +11,7 @@ Volatility3 does not provide the ability to acquire memory. Below are some exam * `AVML - Acquire Volatile Memory for Linux `_ * `LiME - Linux Memory Extract `_ +Be aware that LiME raw format is not supported by volatility3, the padded or lime option should be used instead. `This issue contains further information `_. Procedure to create symbol tables for linux -------------------------------------------- From bf027236950d549b2309159d8d7e7660bd84452e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 7 Oct 2024 17:59:55 +1100 Subject: [PATCH 445/979] Linux: lsof plugin: Fix module import to stick to the style guide --- volatility3/framework/plugins/linux/lsof.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 527815738..18e67cac9 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from datetime import datetime +import datetime from dataclasses import dataclass, astuple, field from typing import List, Callable, Tuple @@ -32,9 +32,9 @@ class FDUser: inode_num: int = field(default=renderers.NotAvailableValue()) inode_type: str = field(default=renderers.NotAvailableValue()) file_mode: str = field(default=renderers.NotAvailableValue()) - change_time: datetime = field(default=renderers.NotAvailableValue()) - modification_time: datetime = field(default=renderers.NotAvailableValue()) - access_time: datetime = field(default=renderers.NotAvailableValue()) + change_time: datetime.datetime = field(default=renderers.NotAvailableValue()) + modification_time: datetime.datetime = field(default=renderers.NotAvailableValue()) + access_time: datetime.datetime = field(default=renderers.NotAvailableValue()) inode_size: int = field(default=renderers.NotAvailableValue()) @@ -184,9 +184,9 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): ("Inode", int), ("Type", str), ("Mode", str), - ("Changed", datetime), - ("Modified", datetime), - ("Accessed", datetime), + ("Changed", datetime.datetime), + ("Modified", datetime.datetime), + ("Accessed", datetime.datetime), ("Size", int), ] return renderers.TreeGrid( From bf19b9f2e00de4bb01ea3156efb261b5619cac43 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 7 Oct 2024 18:46:18 +1100 Subject: [PATCH 446/979] Linux: lsof plugin: Fix dataclasses import to stick to the style guide --- volatility3/framework/plugins/linux/lsof.py | 30 ++++++++++++--------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 18e67cac9..8c3c4a299 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -3,7 +3,7 @@ # import logging import datetime -from dataclasses import dataclass, astuple, field +import dataclasses from typing import List, Callable, Tuple from volatility3.framework import renderers, interfaces, constants @@ -17,7 +17,7 @@ from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -@dataclass +@dataclasses.dataclass class FDUser: """FD user representation, featuring augmented information and formatted fields. This is the data the plugin will eventually display. @@ -28,17 +28,23 @@ class FDUser: task_comm: str fd_num: int full_path: str - device: str = field(default=renderers.NotAvailableValue()) - inode_num: int = field(default=renderers.NotAvailableValue()) - inode_type: str = field(default=renderers.NotAvailableValue()) - file_mode: str = field(default=renderers.NotAvailableValue()) - change_time: datetime.datetime = field(default=renderers.NotAvailableValue()) - modification_time: datetime.datetime = field(default=renderers.NotAvailableValue()) - access_time: datetime.datetime = field(default=renderers.NotAvailableValue()) - inode_size: int = field(default=renderers.NotAvailableValue()) + device: str = dataclasses.field(default=renderers.NotAvailableValue()) + inode_num: int = dataclasses.field(default=renderers.NotAvailableValue()) + inode_type: str = dataclasses.field(default=renderers.NotAvailableValue()) + file_mode: str = dataclasses.field(default=renderers.NotAvailableValue()) + change_time: datetime.datetime = dataclasses.field( + default=renderers.NotAvailableValue() + ) + modification_time: datetime.datetime = dataclasses.field( + default=renderers.NotAvailableValue() + ) + access_time: datetime.datetime = dataclasses.field( + default=renderers.NotAvailableValue() + ) + inode_size: int = dataclasses.field(default=renderers.NotAvailableValue()) -@dataclass +@dataclasses.dataclass class FDInternal: """FD internal representation containing only the core objects @@ -168,7 +174,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): self.context, vmlinux_module_name, filter_func=filter_func ): fd_user = fd_internal.to_user() - yield (0, astuple(fd_user)) + yield (0, dataclasses.astuple(fd_user)) def run(self): pids = self.config.get("pid", None) From 2859d7a641bee72f976e2e16b6dfec61bac107ce Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 7 Oct 2024 18:49:38 +1100 Subject: [PATCH 447/979] Linux: lsof plugin: Fix list_fds() return typing --- volatility3/framework/plugins/linux/lsof.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 8c3c4a299..075f2db84 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -4,7 +4,7 @@ import logging import datetime import dataclasses -from typing import List, Callable, Tuple +from typing import List, Callable, Tuple, Iterable from volatility3.framework import renderers, interfaces, constants from volatility3.framework.configuration import requirements @@ -140,7 +140,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): context: interfaces.context.ContextInterface, vmlinux_module_name: str, filter_func: Callable[[int], bool] = lambda _: False, - ) -> FDInternal: + ) -> Iterable[FDInternal]: """Enumerates open file descriptors in tasks Args: From ee759643456869c5d18cd093f5bb81c138a33eb5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 7 Oct 2024 20:46:44 +1100 Subject: [PATCH 448/979] Linux: lsof plugin: Fix typo in docstring --- volatility3/framework/plugins/linux/lsof.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 075f2db84..42b447dfb 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -49,7 +49,7 @@ class FDInternal: """FD internal representation containing only the core objects Fields: - task: 'task_truct' object + task: 'task_struct' object fd_fields: FD fields as obtained from LinuxUtilities.files_descriptors_for_process() """ From cb98a085d74531105a18b7974fc6250870955b5a Mon Sep 17 00:00:00 2001 From: MHL Date: Mon, 7 Oct 2024 09:11:49 -0500 Subject: [PATCH 449/979] refs #1296 sort volshell methods alphabetically in hh() output --- volatility3/cli/volshell/generic.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index c5eac36e2..c1ca7b5fa 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -112,7 +112,7 @@ class Volshell(interfaces.plugins.PluginInterface): variables = [] print("\nMethods:") - for aliases, item in self.construct_locals(): + for aliases, item in sorted(self.construct_locals()): name = ", ".join(aliases) if item.__doc__ and callable(item): print(f"* {name}") @@ -125,8 +125,7 @@ class Volshell(interfaces.plugins.PluginInterface): print(f" {var}") def construct_locals(self) -> List[Tuple[List[str], Any]]: - """Returns a dictionary listing the functions to be added to the - environment.""" + """Returns a listing of the functions to be added to the environment.""" return [ (["dt", "display_type"], self.display_type), (["db", "display_bytes"], self.display_bytes), From 0dd082b7fda2d762026a823b38ba004819b039be Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 7 Oct 2024 12:26:17 -0500 Subject: [PATCH 450/979] #816 initial PR comment fixes --- .../framework/plugins/windows/cmdscan.py | 43 +++--- .../framework/plugins/windows/consoles.py | 6 +- .../consoles-win10-17763-3232-x64.json | 4 +- .../consoles/consoles-win10-17763-x64.json | 4 +- .../consoles/consoles-win10-18362-x64.json | 4 +- .../consoles/consoles-win10-19041-x64.json | 4 +- .../consoles-win10-20348-1970-x64.json | 47 +----- .../consoles-win10-20348-2461-x64.json | 47 +----- .../consoles/consoles-win10-20348-x64.json | 47 +----- .../consoles/consoles-win10-22000-x64.json | 47 +----- ...son => consoles-win10-22621-3527-x64.json} | 47 +----- .../consoles/consoles-win10-22621-x64.json | 47 +----- .../consoles/consoles-win10-25398-x64.json | 47 +----- .../symbols/windows/extensions/consoles.py | 144 +++++++----------- 14 files changed, 111 insertions(+), 427 deletions(-) rename volatility3/framework/symbols/windows/consoles/{consoles-win10-22621-3672-x64.json => consoles-win10-22621-3527-x64.json} (93%) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 09cb0f316..9a3460a20 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -61,7 +61,7 @@ class CmdScan(interfaces.plugins.PluginInterface): cls, conhost_proc: interfaces.context.ContextInterface, size_filter: Optional[int] = 0x40000000, - ) -> List[Tuple[int, int]]: + ) -> Generator[Tuple[int, int], None, None]: """ Returns vads of a process with size smaller than size_filter @@ -73,20 +73,17 @@ class CmdScan(interfaces.plugins.PluginInterface): vad_base: the base address vad_size: the size of the VAD """ - vads = [] for vad in conhost_proc.get_vad_root().traverse(): base = vad.get_start() if vad.get_size() < size_filter: - vads.append((base, vad.get_size())) - - return vads + yield (base, vad.get_size()) @classmethod def get_command_history( cls, context: interfaces.context.ContextInterface, kernel_layer_name: str, - kernel_table_name: str, + kernel_symbol_table_name: str, config_path: str, procs: Generator[interfaces.objects.ObjectInterface, None, None], max_history: Set[int], @@ -100,7 +97,7 @@ class CmdScan(interfaces.plugins.PluginInterface): Args: context: The context to retrieve required elements (layers, symbol tables) from kernel_layer_name: The name of the layer on which to operate - kernel_table_name: The name of the table containing the kernel symbols + kernel_symbol_table_name: The name of the table containing the kernel symbols config_path: The config path where to find symbol files procs: list of process objects max_history: an initial set of CommandHistorySize values @@ -138,7 +135,7 @@ class CmdScan(interfaces.plugins.PluginInterface): conhost_symbol_table = consoles.Consoles.create_conhost_symbol_table( context, kernel_layer_name, - kernel_table_name, + kernel_symbol_table_name, config_path, proc_layer_name, conhostexe_base, @@ -147,6 +144,9 @@ class CmdScan(interfaces.plugins.PluginInterface): conhost_module = context.module( conhost_symbol_table, proc_layer_name, offset=conhostexe_base ) + command_count_max_offset = conhost_module.get_type( + "_COMMAND_HISTORY" + ).relative_child_offset("CommandCountMax") sections = cls.get_filtered_vads(conhost_proc) found_history_for_proc = False @@ -161,15 +161,13 @@ class CmdScan(interfaces.plugins.PluginInterface): scanners.BytesScanner(max_history_bytes), sections=sections, ): + command_history = None command_history_properties = [] try: command_history = conhost_module.object( "_COMMAND_HISTORY", - offset=address - - conhost_module.get_type( - "_COMMAND_HISTORY" - ).relative_child_offset("CommandCountMax"), + offset=address - command_count_max_offset, absolute=True, ) @@ -184,13 +182,13 @@ class CmdScan(interfaces.plugins.PluginInterface): "level": 0, "name": "_COMMAND_HISTORY", "address": command_history.vol.offset, - "data": "", + "data": None, } ) command_history_properties.append( { "level": 1, - "name": f"_COMMAND_HISTORY.Application", + "name": "_COMMAND_HISTORY.Application", "address": command_history.Application.vol.offset, "data": command_history.get_application(), } @@ -198,7 +196,7 @@ class CmdScan(interfaces.plugins.PluginInterface): command_history_properties.append( { "level": 1, - "name": f"_COMMAND_HISTORY.ProcessHandle", + "name": "_COMMAND_HISTORY.ProcessHandle", "address": command_history.ConsoleProcessHandle.ProcessHandle.vol.offset, "data": hex( command_history.ConsoleProcessHandle.ProcessHandle @@ -208,7 +206,7 @@ class CmdScan(interfaces.plugins.PluginInterface): command_history_properties.append( { "level": 1, - "name": f"_COMMAND_HISTORY.CommandCount", + "name": "_COMMAND_HISTORY.CommandCount", "address": None, "data": command_history.CommandCount, } @@ -216,7 +214,7 @@ class CmdScan(interfaces.plugins.PluginInterface): command_history_properties.append( { "level": 1, - "name": f"_COMMAND_HISTORY.LastDisplayed", + "name": "_COMMAND_HISTORY.LastDisplayed", "address": command_history.LastDisplayed.vol.offset, "data": command_history.LastDisplayed, } @@ -224,7 +222,7 @@ class CmdScan(interfaces.plugins.PluginInterface): command_history_properties.append( { "level": 1, - "name": f"_COMMAND_HISTORY.CommandCountMax", + "name": "_COMMAND_HISTORY.CommandCountMax", "address": command_history.CommandCountMax.vol.offset, "data": command_history.CommandCountMax, } @@ -233,7 +231,7 @@ class CmdScan(interfaces.plugins.PluginInterface): command_history_properties.append( { "level": 1, - "name": f"_COMMAND_HISTORY.CommandBucket", + "name": "_COMMAND_HISTORY.CommandBucket", "address": command_history.CommandBucket.vol.offset, "data": "", } @@ -248,7 +246,7 @@ class CmdScan(interfaces.plugins.PluginInterface): "level": 2, "name": f"_COMMAND_HISTORY.CommandBucket_Command_{cmd_index}", "address": bucket_cmd.vol.offset, - "data": bucket_cmd.get_command(), + "data": bucket_cmd.get_command_string(), } ) except Exception as e: @@ -264,6 +262,9 @@ class CmdScan(interfaces.plugins.PluginInterface): found_history_for_proc = True yield conhost_proc, command_history, command_history_properties + # if found_history_for_proc is still False, then none of the scanned locations found + # a valid _COMMAND_HISTORY for the process, so yield the process and some empty data + # so the process can at least be reported that it was found with no history if not found_history_for_proc: yield conhost_proc, command_history or None, [] @@ -349,7 +350,7 @@ class CmdScan(interfaces.plugins.PluginInterface): if proc is None: vollog.warn("No conhost.exe processes found.") - def _conhost_proc_filter(self, proc): + def _conhost_proc_filter(self, proc: interfaces.objects.ObjectInterface): """ Used to filter to only conhost.exe processes """ diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index dfc14f2d6..cef7cba46 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -209,7 +209,7 @@ class Consoles(interfaces.plugins.PluginInterface): (10, 0, 20348, 2520): "consoles-win10-20348-2461-x64", (10, 0, 22000, 0): "consoles-win10-22000-x64", (10, 0, 22621, 1): "consoles-win10-22621-x64", - (10, 0, 22621, 3672): "consoles-win10-22621-3672-x64", + (10, 0, 22621, 3527): "consoles-win10-22621-3527-x64", (10, 0, 25398, 0): "consoles-win10-22000-x64", } @@ -706,7 +706,7 @@ class Consoles(interfaces.plugins.PluginInterface): "level": 3, "name": f"_CONSOLE_INFORMATION.HistoryList.CommandHistory_{index}_Command_{cmd_index}", "address": bucket_cmd.vol.offset, - "data": bucket_cmd.get_command(), + "data": bucket_cmd.get_command_string(), } ) except Exception as e: @@ -918,7 +918,7 @@ class Consoles(interfaces.plugins.PluginInterface): if proc is None: vollog.warn("No conhost.exe processes found.") - def _conhost_proc_filter(self, proc): + def _conhost_proc_filter(self, proc: interfaces.objects.ObjectInterface) -> bool: """ Used to filter to only conhost.exe processes """ diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json index 413b6f466..13382c7ee 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-3232-x64.json @@ -632,14 +632,14 @@ "Source": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, "Target": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 48 } diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json index b3bf9666c..f813691bc 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-17763-x64.json @@ -632,14 +632,14 @@ "Source": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, "Target": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 48 } diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json index d962418d3..85ef9d718 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-18362-x64.json @@ -622,14 +622,14 @@ "Source": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, "Target": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 48 } diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json index 0963afee3..8f6dd8c6a 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-19041-x64.json @@ -622,14 +622,14 @@ "Source": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, "Target": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 48 } diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json index cee78e2b5..8357ece4a 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-1970-x64.json @@ -598,7 +598,7 @@ "ExeName": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, @@ -628,61 +628,20 @@ "Source": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, "Target": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 48 } }, "kind": "struct", "size": 32 - }, - "_ALIAS_STRING": { - "fields": { - "Chars": { - "type": { - "count": 1, - "kind": "array", - "subtype": { - "kind": "base", - "name": "unsigned char" - } - }, - "offset": 0 - }, - "Pointer": { - "type": { - "kind": "pointer", - "subtype": { - "kind": "base", - "name": "string" - } - }, - "offset": 0 - }, - "Length": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 16 - }, - "Allocated": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 24 - } - }, - "kind": "struct", - "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json index 59e9fd3b9..f80a99148 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-2461-x64.json @@ -598,7 +598,7 @@ "ExeName": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, @@ -628,61 +628,20 @@ "Source": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, "Target": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 48 } }, "kind": "struct", "size": 32 - }, - "_ALIAS_STRING": { - "fields": { - "Chars": { - "type": { - "count": 1, - "kind": "array", - "subtype": { - "kind": "base", - "name": "unsigned char" - } - }, - "offset": 0 - }, - "Pointer": { - "type": { - "kind": "pointer", - "subtype": { - "kind": "base", - "name": "string" - } - }, - "offset": 0 - }, - "Length": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 16 - }, - "Allocated": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 24 - } - }, - "kind": "struct", - "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json index df02169cb..5c7fc8473 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-20348-x64.json @@ -598,7 +598,7 @@ "ExeName": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, @@ -628,61 +628,20 @@ "Source": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, "Target": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 48 } }, "kind": "struct", "size": 32 - }, - "_ALIAS_STRING": { - "fields": { - "Chars": { - "type": { - "count": 1, - "kind": "array", - "subtype": { - "kind": "base", - "name": "unsigned char" - } - }, - "offset": 0 - }, - "Pointer": { - "type": { - "kind": "pointer", - "subtype": { - "kind": "base", - "name": "string" - } - }, - "offset": 0 - }, - "Length": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 16 - }, - "Allocated": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 24 - } - }, - "kind": "struct", - "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-22000-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-22000-x64.json index 0dfaf34af..cf38d3e14 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-22000-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-22000-x64.json @@ -624,7 +624,7 @@ "ExeName": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, @@ -654,61 +654,20 @@ "Source": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, "Target": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 48 } }, "kind": "struct", "size": 32 - }, - "_ALIAS_STRING": { - "fields": { - "Chars": { - "type": { - "count": 1, - "kind": "array", - "subtype": { - "kind": "base", - "name": "unsigned char" - } - }, - "offset": 0 - }, - "Pointer": { - "type": { - "kind": "pointer", - "subtype": { - "kind": "base", - "name": "string" - } - }, - "offset": 0 - }, - "Length": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 16 - }, - "Allocated": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 24 - } - }, - "kind": "struct", - "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-3672-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-3527-x64.json similarity index 93% rename from volatility3/framework/symbols/windows/consoles/consoles-win10-22621-3672-x64.json rename to volatility3/framework/symbols/windows/consoles/consoles-win10-22621-3527-x64.json index 4c71ee32e..a5bda0f35 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-3672-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-3527-x64.json @@ -624,7 +624,7 @@ "ExeName": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, @@ -654,61 +654,20 @@ "Source": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, "Target": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 48 } }, "kind": "struct", "size": 32 - }, - "_ALIAS_STRING": { - "fields": { - "Chars": { - "type": { - "count": 1, - "kind": "array", - "subtype": { - "kind": "base", - "name": "unsigned char" - } - }, - "offset": 0 - }, - "Pointer": { - "type": { - "kind": "pointer", - "subtype": { - "kind": "base", - "name": "string" - } - }, - "offset": 0 - }, - "Length": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 16 - }, - "Allocated": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 24 - } - }, - "kind": "struct", - "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-x64.json index d6da72ef2..44f926a87 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-22621-x64.json @@ -624,7 +624,7 @@ "ExeName": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, @@ -654,61 +654,20 @@ "Source": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, "Target": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 48 } }, "kind": "struct", "size": 32 - }, - "_ALIAS_STRING": { - "fields": { - "Chars": { - "type": { - "count": 1, - "kind": "array", - "subtype": { - "kind": "base", - "name": "unsigned char" - } - }, - "offset": 0 - }, - "Pointer": { - "type": { - "kind": "pointer", - "subtype": { - "kind": "base", - "name": "string" - } - }, - "offset": 0 - }, - "Length": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 16 - }, - "Allocated": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 24 - } - }, - "kind": "struct", - "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/consoles/consoles-win10-25398-x64.json b/volatility3/framework/symbols/windows/consoles/consoles-win10-25398-x64.json index 120ad85ec..5e91d8824 100644 --- a/volatility3/framework/symbols/windows/consoles/consoles-win10-25398-x64.json +++ b/volatility3/framework/symbols/windows/consoles/consoles-win10-25398-x64.json @@ -625,7 +625,7 @@ "ExeName": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, @@ -655,61 +655,20 @@ "Source": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 16 }, "Target": { "type": { "kind": "struct", - "name": "_ALIAS_STRING" + "name": "_COMMAND" }, "offset": 48 } }, "kind": "struct", "size": 32 - }, - "_ALIAS_STRING": { - "fields": { - "Chars": { - "type": { - "count": 1, - "kind": "array", - "subtype": { - "kind": "base", - "name": "unsigned char" - } - }, - "offset": 0 - }, - "Pointer": { - "type": { - "kind": "pointer", - "subtype": { - "kind": "base", - "name": "string" - } - }, - "offset": 0 - }, - "Length": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 16 - }, - "Allocated": { - "type": { - "kind": "base", - "name": "unsigned int" - }, - "offset": 24 - } - }, - "kind": "struct", - "size": 32 } }, "metadata": { diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 390b6858d..96f7892a5 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -2,17 +2,21 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import objects +import logging +from typing import Generator, List, Union, Tuple +from volatility3.framework import objects, interfaces from volatility3.framework import constants +vollog = logging.getLogger(__name__) + class ROW(objects.StructType): """A Row Structure.""" - def _valid_dbcs(self, c, n): + def _valid_dbcs(self, dbcs_attr: int, text_attr_msb: int) -> bool: # TODO this need more research and testing # https://github.com/search?q=repo%3Amicrosoft%2Fterminal+DbcsAttr&type=code - valid = n == 0 and c in ( + valid = text_attr_msb == 0 and dbcs_attr in ( 0x0, 0x1, 0x2, @@ -43,11 +47,11 @@ class ROW(objects.StructType): 0xF0, 0xA0, ) - if n == 0 and not valid: - print("Bad Dbcs Attribute {}".format(hex(c))) + if text_attr_msb == 0 and not valid: + vollog.debug(f"Bad Dbcs Attribute {dbcs_attr:#x}") return valid - def get_text(self, truncate=True): + def get_text(self, truncate: bool = True) -> str: """A convenience method to extract the text from the _ROW. The _ROW contains a pointer CharRow to an array of CharRowCell objects. Each CharRowCell contains the wide character and an attribute. Enumerating @@ -70,7 +74,6 @@ class ROW(objects.StructType): for i in range(0, len(char_row), 3) ) except Exception as e: - print(e) line = "" if truncate: @@ -82,62 +85,26 @@ class ROW(objects.StructType): class ALIAS(objects.StructType): """An Alias Structure""" - def get_source(self): - if self.Source.Length < 8: - return self.Source.Chars.cast( - "string", - encoding="utf-16", - errors="replace", - max_length=self.Source.Length * 2, - ) - elif self.Source.Length < 1024: - return self.Source.Pointer.dereference().cast( - "string", encoding="utf-16", errors="replace", max_length=512 - ) + def get_source(self) -> Union[str, None]: + return self.Source.get_command_string() - return None - - def get_target(self): - if self.Target.Length < 8: - return self.Target.Chars.cast( - "string", - encoding="utf-16", - errors="replace", - max_length=self.Target.Length * 2, - ) - elif self.Target.Length < 1024: - return self.Target.Pointer.dereference().cast( - "string", encoding="utf-16", errors="replace", max_length=512 - ) - - return None + def get_target(self) -> Union[str, None]: + return self.Target.get_command_string() class EXE_ALIAS_LIST(objects.StructType): """An Exe Alias List Structure""" - def get_exename(self): + def get_exename(self) -> Union[str, None]: exe_name = self.ExeName # Windows 10 22000 and Server 20348 removed the Pointer if isinstance(exe_name, objects.Pointer): exe_name = exe_name.dereference() return exe_name.get_string() - if self.ExeName.Length < 8: - return self.ExeName.Chars.cast( - "string", - encoding="utf-16", - errors="replace", - max_length=self.ExeName.Length * 2, - ) - elif self.ExeName.Length < 1024: - return self.ExeName.Pointer.dereference().cast( - "string", encoding="utf-16", errors="replace", max_length=512 - ) + return exe_name.get_command_string() - return None - - def get_aliases(self): + def get_aliases(self) -> Generator[interfaces.objects.ObjectInterface, None, None]: """Generator for the individual aliases for a particular executable.""" for alias in self.AliasList.to_list( @@ -151,7 +118,7 @@ class SCREEN_INFORMATION(objects.StructType): """A Screen Information Structure.""" @property - def ScreenX(self): + def ScreenX(self) -> int: # 22000 change from an array of pointers to _ROW to an array of _ROW row = self.TextBufferInfo.BufferRows.Rows[0] if hasattr(row, "Row"): @@ -160,10 +127,10 @@ class SCREEN_INFORMATION(objects.StructType): return row.RowLength2 @property - def ScreenY(self): + def ScreenY(self) -> int: return self.TextBufferInfo.BufferCapacity - def _truncate_rows(self, rows): + def _truncate_rows(self, rows: List[str]) -> List[str]: """To truncate empty rows at the end, walk the list backwards and get the last non-empty row. Use that row index to splice. Rows are created based on the @@ -187,7 +154,9 @@ class SCREEN_INFORMATION(objects.StructType): return rows - def get_buffer(self, truncate_rows=True, truncate_lines=True): + def get_buffer( + self, truncate_rows: bool = True, truncate_lines: bool = True + ) -> List[str]: """Get the screen buffer. The screen buffer is comprised of the screen's Y @@ -206,7 +175,7 @@ class SCREEN_INFORMATION(objects.StructType): capacity = self.TextBufferInfo.BufferCapacity start = self.TextBufferInfo.BufferStart - buffer_rows = self.TextBufferInfo.BufferRows.dereference() + buffer_rows = self.TextBufferInfo.BufferRows buffer_rows.Rows.count = self.TextBufferInfo.BufferCapacity for i in range(capacity): @@ -234,10 +203,10 @@ class CONSOLE_INFORMATION(objects.StructType): """A Console Information Structure.""" @property - def ScreenBuffer(self): + def ScreenBuffer(self) -> interfaces.objects.ObjectInterface: return self.GetScreenBuffer - def is_valid(self, max_buffers=4) -> bool: + def is_valid(self, max_buffers: int = 4) -> bool: """Determine if the structure is valid.""" # Last displayed must be between -1 and max @@ -249,7 +218,7 @@ class CONSOLE_INFORMATION(objects.StructType): return True - def get_screens(self): + def get_screens(self) -> Generator[interfaces.objects.ObjectInterface, None, None]: """Generator for screens in the console. A console can have multiple screen buffers at a time, @@ -277,14 +246,18 @@ class CONSOLE_INFORMATION(objects.StructType): seen.add(cur.vol.offset) cur = cur.Next - def get_histories(self): - for cmd_hist in self.HistoryList.dereference().to_list( + def get_histories( + self, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + for cmd_hist in self.HistoryList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_COMMAND_HISTORY", "ListEntry", ): yield cmd_hist - def get_exe_aliases(self): + def get_exe_aliases( + self, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: exe_alias_list = self.ExeAliasList # Windows 10 22000 and Server 20348 made this a Pointer if isinstance(exe_alias_list, objects.Pointer): @@ -295,14 +268,16 @@ class CONSOLE_INFORMATION(objects.StructType): ): yield exe_alias_list_item - def get_processes(self): - for proc in self.ConsoleProcessList.dereference().to_list( + def get_processes( + self, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + for proc in self.ConsoleProcessList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_CONSOLE_PROCESS_LIST", "ListEntry", ): yield proc - def get_title(self): + def get_title(self) -> Union[str, None]: try: return self.Title.dereference().cast( "string", encoding="utf-16", errors="replace", max_length=512 @@ -310,7 +285,7 @@ class CONSOLE_INFORMATION(objects.StructType): except Exception: return "" - def get_original_title(self): + def get_original_title(self) -> Union[str, None]: try: return self.OriginalTitle.dereference().cast( "string", encoding="utf-16", errors="replace", max_length=512 @@ -322,7 +297,7 @@ class CONSOLE_INFORMATION(objects.StructType): class COMMAND(objects.StructType): """A Command Structure""" - def is_valid(self): + def is_valid(self) -> bool: if ( self.Length < 1 or self.Allocated < 1 @@ -333,7 +308,7 @@ class COMMAND(objects.StructType): return True - def get_command(self): + def get_command_string(self) -> Union[str, None]: if self.Length < 8: return self.Chars.cast( "string", @@ -343,7 +318,10 @@ class COMMAND(objects.StructType): ) elif self.Length < 1024: return self.Pointer.dereference().cast( - "string", encoding="utf-16", errors="replace", max_length=512 + "string", + encoding="utf-16", + errors="replace", + max_length=self.Length * 2, ) return None @@ -353,17 +331,17 @@ class COMMAND_HISTORY(objects.StructType): """A Command History Structure.""" @property - def CommandCount(self): + def CommandCount(self) -> int: command_type = self.get_symbol_table_name() + constants.BANG + "_COMMAND" command_size = self._context.symbol_space.get_type(command_type).size return int((self.CommandBucket.End - self.CommandBucket.Begin) / command_size) @property - def ProcessHandle(self): + def ProcessHandle(self) -> int: """Allow ProcessHandle to be referenced regardless of OS version""" return self.ConsoleProcessHandle.ProcessHandle - def is_valid(self, max_history=50): + def is_valid(self, max_history: int = 50) -> bool: # The count must be between zero and max if self.CommandCount < 0 or self.CommandCount > max_history: return False @@ -382,22 +360,12 @@ class COMMAND_HISTORY(objects.StructType): return True - def get_application(self): - if self.Application.Length < 8: - return self.Application.Chars.cast( - "string", - encoding="utf-16", - errors="replace", - max_length=self.Application.Length * 2, - ) - elif self.Application.Length < 1024: - return self.Application.Pointer.dereference().cast( - "string", encoding="utf-16", errors="replace", max_length=512 - ) + def get_application(self) -> Union[str, None]: + return self.Application.get_command_string() - return None - - def scan_command_bucket(self, end=None): + def scan_command_bucket( + self, end: Union[int, None] = None + ) -> Generator[Tuple[int, interfaces.objects.ObjectInterface], None, None]: """Brute force print all strings pointed to by the CommandBucket entries by going to greater of EndCapacity or CommandCountMax*sizeof(_COMMAND)""" @@ -418,7 +386,9 @@ class COMMAND_HISTORY(objects.StructType): if cmd.is_valid(): yield i, cmd - def get_commands(self): + def get_commands( + self, + ) -> Generator[Tuple[int, interfaces.objects.ObjectInterface], None, None]: """Generator for commands in the history buffer. The CommandBucket is an array of pointers to _COMMAND From 95f56ad122151bf7654dca68f943b7ce5f969f94 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 7 Oct 2024 12:39:03 -0500 Subject: [PATCH 451/979] #816 - remove extra text from buffer output --- volatility3/framework/plugins/windows/consoles.py | 6 +++++- .../framework/symbols/windows/extensions/consoles.py | 4 ---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index cef7cba46..ad1c9d4bd 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -895,7 +895,11 @@ class Consoles(interfaces.plugins.PluginInterface): if console_property["address"] is None else format_hints.Hex(console_property["address"]) ), - str(console_property["data"]), + ( + str(console_property["data"]) + if console_property["data"] + else renderers.NotAvailableValue() + ), ), ) else: diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 96f7892a5..2312149c7 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -192,10 +192,6 @@ class SCREEN_INFORMATION(objects.StructType): if truncate_rows: rows = self._truncate_rows(rows) - if rows: - rows = ["=== START OF BUFFER ==="] + rows + ["=== END OF BUFFER ==="] - else: - rows = ["=== NO BUFFER DATA FOUND ==="] return rows From 1c3e5574f4dda317bfde0fdf72805ca488b342a6 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 7 Oct 2024 12:39:58 -0500 Subject: [PATCH 452/979] #816 - remove unused import --- volatility3/framework/plugins/windows/cmdscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 9a3460a20..9645ee507 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -6,7 +6,7 @@ import logging import struct -from typing import Tuple, Generator, Set, Dict, Any, List, Optional +from typing import Tuple, Generator, Set, Dict, Any, Optional from volatility3.framework import interfaces from volatility3.framework import renderers From 0bfac049defe05ea589dcc6d2b611df5e0eaacd4 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 18 Jul 2024 10:44:51 -0500 Subject: [PATCH 453/979] Windows: Adds Amcache plugin --- .../framework/plugins/windows/amcache.py | 646 ++++++++++++++++++ .../plugins/windows/registry/hivelist.py | 10 +- .../symbols/windows/extensions/registry.py | 62 +- 3 files changed, 700 insertions(+), 18 deletions(-) create mode 100644 volatility3/framework/plugins/windows/amcache.py diff --git a/volatility3/framework/plugins/windows/amcache.py b/volatility3/framework/plugins/windows/amcache.py new file mode 100644 index 000000000..5838c62a0 --- /dev/null +++ b/volatility3/framework/plugins/windows/amcache.py @@ -0,0 +1,646 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import dataclasses +import datetime +import enum +import logging +import itertools +from typing import Dict, Iterable, Iterator, List, Optional, Tuple, Union + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import registry +from volatility3.framework.renderers import conversion +from volatility3.framework.symbols.windows.extensions import registry as reg_extensions +from volatility3.plugins import timeliner +from volatility3.plugins.windows.registry import hivelist + +vollog = logging.getLogger(__name__) + +####################################################################### +# More information about the following enums can be found in the report +# 'Analysis of the AmCache` by Blanche Lagny, 2019 +####################################################################### + + +class Win8FileValName(enum.Enum): + """ + An enumeration that creates a helpful mapping of opaque Windows 8 Amcache + 'File' subkey value names to their human-readable equivalent. + """ + + ProgramID = "100" + SHA1Hash = "101" + Product = "0" + Company = "1" + Size = "6" + SizeOfImage = "7" + PEHeaderChecksum = "9" + LastModTime = "11" # REG_QWORD FILETIME + CreateTime = "12" # REG_QWORD FILETIME + Path = "15" + LastModTime2 = "17" # REG_QWORD FILETIME + Version = "d" + CompileTime = "f" # REG_QWORD UNIX EPOCH + + +class Win8ProgramValName(enum.Enum): + """ + An enumeration that creates a helpful mapping of opaque Windows 8 Amcache + 'Program' subkey value names to their human-readable equivalent. + """ + + Product = "0" + Version = "1" + Publisher = "2" + InstallTime = "a" + MSIProductCode = "11" + MSIPackageCode = "12" + ProductCode = "f" + PackageCode = "10" + + +class Win10InvAppFileValName(enum.Enum): + """ + An enumeration containing the most useful Windows 10 Amcache + 'InventoryApplicationFile' subkey value names. + """ + + FileId = "FileId" + LinkDate = "LinkDate" + LowerCaseLongPath = "LowerCaseLongPath" + ProductName = "ProductName" + ProductVersion = "ProductVersion" + ProgramID = "ProgramId" + Publisher = "Publisher" + + +class Win10InvAppValName(enum.Enum): + """ + An enumeration containing the most useful Windows 10 Amcache + 'InventoryApplication' subkey value names. + """ + + InstallDate = "InstallDate" + Name = "Name" + Publisher = "Publisher" + RootDirPath = "RootDirPath" + Version = "Version" + + +class Win10DriverBinaryValName(enum.Enum): + """ + An enumeration containing the most useful Windows 10 Amcache + 'InventoryDriverBinary' subkey value names. + """ + + DriverId = "DriverId" + DriverName = "DriverName" + DriverCompany = "DriverCompany" + Product = "Product" + Service = "Service" + DriverTimeStamp = "DriverTimeStamp" + + +class AmcacheEntryType(enum.IntEnum): + Driver = 1 + Program = 2 + File = 3 + + +NullableString = Union[str, None, interfaces.renderers.BaseAbsentValue] +NullableDatetime = Union[datetime.datetime, None, interfaces.renderers.BaseAbsentValue] + + +@dataclasses.dataclass +class _AmcacheEntry: + """ + A class containing all information about an entry from the Amcache registry hive. + Because all values could potentially be paged out of memory or malformed, they are all + a union between their expected value and `interfaces.renderers.BaseAbsentValue`. + """ + + entry_type: str + path: NullableString = renderers.NotApplicableValue() + company: NullableString = renderers.NotApplicableValue() + last_modify_time: NullableDatetime = renderers.NotApplicableValue() + last_modify_time_2: NullableDatetime = renderers.NotApplicableValue() + install_time: NullableDatetime = renderers.NotApplicableValue() + compile_time: NullableDatetime = renderers.NotApplicableValue() + sha1_hash: NullableString = renderers.NotApplicableValue() + service: NullableString = renderers.NotApplicableValue() + product_name: NullableString = renderers.NotApplicableValue() + + +def _entry_sort_key(entry_tuple: Tuple[NullableString, _AmcacheEntry]) -> str: + """Sorts entries by program ID. This is broken out as a function here + to ensure consistency in sorting between the `group_by` and `sorted` function + invocations. + """ + program_id, _ = entry_tuple + key = program_id if isinstance(program_id, str) else "" + return key + + +def _get_string_value( + values: Dict[str, reg_extensions.CM_KEY_VALUE], name: str +) -> NullableString: + try: + value = values[name] + except KeyError: + return renderers.NotAvailableValue() + + data = value.decode_data() + if not isinstance(data, bytes): + return renderers.UnparsableValue() + + return data.decode("utf-16le", errors="replace").rstrip("\u0000") + + +def _get_datetime_filetime_value( + values: Dict[str, reg_extensions.CM_KEY_VALUE], name: str +) -> NullableDatetime: + try: + value = values[name] + except KeyError: + return renderers.NotAvailableValue() + + data = value.decode_data() + if not isinstance(data, int): + return renderers.UnparsableValue() + + return conversion.wintime_to_datetime(data) + + +def _get_datetime_utc_epoch_value( + values: Dict[str, reg_extensions.CM_KEY_VALUE], name: str +) -> NullableDatetime: + try: + value = values[name] + except KeyError: + return renderers.NotAvailableValue() + + data = value.decode_data() + if not isinstance(data, (int, float)): + return renderers.UnparsableValue() + + try: + return datetime.datetime.fromtimestamp(float(data), datetime.timezone.utc) + except (ValueError, OverflowError, OSError): + return renderers.UnparsableValue() + + +def _get_datetime_str_value( + values: Dict[str, reg_extensions.CM_KEY_VALUE], name: str +) -> NullableDatetime: + try: + value = values[name] + except KeyError: + return renderers.NotAvailableValue() + + data = value.decode_data() + if not isinstance(data, int): + return renderers.UnparsableValue() + + if isinstance(data, str): + try: + return datetime.datetime.strptime(data, "%m/%d/%Y %H:%M:%S") + except ValueError: + return renderers.UnparsableValue() + else: + return renderers.UnparsableValue() + + +class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): + """Scans for windows services.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), + ] + + def generate_timeline( + self, + ) -> Iterator[Tuple[str, timeliner.TimeLinerType, datetime.datetime]]: + for _, entry in self._generator(): + if isinstance(entry.last_modify_time, datetime.datetime): + yield f"Amcache: {entry.entry_type} {entry.path} registry key modified", timeliner.TimeLinerType.MODIFIED, entry.last_modify_time + if isinstance(entry.last_modify_time_2, datetime.datetime): + yield f"Amcache: {entry.entry_type} {entry.path} STANDARD_INFORMATION create time", timeliner.TimeLinerType.CREATED, entry.last_modify_time_2 + if isinstance(entry.install_time, datetime.datetime): + yield f"Amcache: {entry.entry_type} {entry.path} installed", timeliner.TimeLinerType.CREATED, entry.install_time + if isinstance(entry.compile_time, datetime.datetime): + yield f"Amcache: {entry.entry_type} {entry.path} compiled (PE metadata)", timeliner.TimeLinerType.MODIFIED, entry.compile_time + + @classmethod + def get_amcache_hive( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + kernel: interfaces.context.ModuleInterface, + ) -> Optional[registry.RegistryHive]: + """Retrieves the `Amcache.hve` registry hive from the kernel module, if it can be located.""" + return next( + hivelist.HiveList.list_hives( + context=context, + base_config_path=interfaces.configuration.path_join( + config_path, "hivelist" + ), + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_string="amcache", + ), + None, + ) + + @classmethod + def parse_file_key( + cls, file_key: reg_extensions.CM_KEY_NODE + ) -> Iterator[Tuple[NullableString, _AmcacheEntry]]: + """Parses File entries from the Windows 8 `Root\\File` key. + + :param programs_key: The `Root\\File` registry key. + + :return: An iterator of tuples, where the first member is the program ID string for + correlating `Root\\Program` entries, and the second member is the `AmcacheEntry`. + """ + + for file_entry_key in itertools.chain( + *(key.get_subkeys() for key in file_key.get_subkeys()) + ): + vollog.debug(f"Checking Win8 File key {file_entry_key.get_name()}") + values = { + str(value.get_name()): value + for value in file_entry_key.get_values() + if value.get_name() in [key.value for key in Win8FileValName] + } + + program_id = _get_string_value(values, Win8FileValName.ProgramID.value) + path = _get_string_value(values, Win8FileValName.Path.value) + company = _get_string_value(values, Win8FileValName.Company.value) + last_mod_time = _get_datetime_filetime_value( + values, Win8FileValName.LastModTime.value + ) + last_mod_time_2 = _get_datetime_filetime_value( + values, Win8FileValName.LastModTime2.value + ) + install_time = _get_datetime_filetime_value( + values, Win8FileValName.CreateTime.value + ) + compile_time = _get_datetime_utc_epoch_value( + values, Win8FileValName.CompileTime.value + ) + sha1_hash = _get_string_value(values, Win8FileValName.SHA1Hash.value) + vollog.debug(f"Found sha1hash {sha1_hash}") + product_name = _get_string_value(values, Win8FileValName.Product.value) + + yield program_id, _AmcacheEntry( + AmcacheEntryType.File.name, + path=path, + company=company, + last_modify_time=last_mod_time, + last_modify_time_2=last_mod_time_2, + install_time=install_time, + compile_time=compile_time, + sha1_hash=( + sha1_hash.lstrip("0000") + if isinstance(sha1_hash, str) + else sha1_hash + ), + product_name=product_name, + ) + + @classmethod + def parse_programs_key( + cls, programs_key: reg_extensions.CM_KEY_NODE + ) -> Iterator[Tuple[str, _AmcacheEntry]]: + """Parses Program entries from the Windows 8 `Root\\Programs` key. + + :param programs_key: The `Root\\Programs` registry key. + + :return: An iterator of tuples, where the first member is the program ID string for + correlating `Root\\File` entries, and the second member is the `AmcacheEntry`. + """ + for program_key in programs_key.get_subkeys(): + values = { + str(value.get_name()): value + for value in program_key.get_values() + if value.get_name() in [key.value for key in Win8ProgramValName] + } + vollog.debug(f"Parsing Win8 Program key {program_key.get_name()}") + program_id = program_key.get_name().strip().strip("\u0000") + + product = _get_string_value(values, Win8ProgramValName.Product.value) + company = _get_string_value(values, Win8ProgramValName.Publisher.value) + install_time = _get_datetime_utc_epoch_value( + values, Win8ProgramValName.InstallTime.value + ) + _version = _get_string_value(values, Win8ProgramValName.Version.value) + + if isinstance(_version, str): + if isinstance(product, str): + product = f"{product} {_version}" + else: + product = f"UNKNOWN {_version}" + + yield program_id, _AmcacheEntry( + AmcacheEntryType.Program.name, + company=company, + last_modify_time=conversion.wintime_to_datetime( + program_key.LastWriteTime.QuadPart + ), + install_time=install_time, + product_name=product, + ) + + @classmethod + def parse_inventory_app_key( + cls, inv_app_key: reg_extensions.CM_KEY_NODE + ) -> Iterator[Tuple[str, _AmcacheEntry]]: + """Parses InventoryApplication entries from the Windows 10 `Root\\InventoryApplication` key. + + :param programs_key: The `Root\\InventoryApplication` registry key. + + :return: An iterator of tuples, where the first member is the program ID string for + correlating `Root\\InventoryApplicationFile` entries, and the second member is the `AmcacheEntry`. + """ + for program_key in inv_app_key.get_subkeys(): + program_id = program_key.get_name() + + values = { + str(value.get_name()): value + for value in program_key.get_values() + if value.get_name() in [key.value for key in Win10InvAppValName] + } + + name = _get_string_value(values, Win10InvAppValName.Name.value) + version = _get_string_value(values, Win10InvAppValName.Version.value) + publisher = _get_string_value(values, Win10InvAppValName.Publisher.value) + path = _get_string_value(values, Win10InvAppValName.RootDirPath.value) + install_date = _get_datetime_str_value( + values, Win10InvAppValName.InstallDate.value + ) + last_mod = conversion.wintime_to_datetime( + program_key.LastWriteTime.QuadPart + ) + + product: str = name if isinstance(name, str) else "UNKNOWN" # type: ignore + if isinstance(version, str): + product += " " + version + + yield program_id.strip().strip("\u0000"), _AmcacheEntry( + AmcacheEntryType.Program.name, + path=path, + last_modify_time=last_mod, + install_time=install_date, + product_name=product, + company=publisher, + ) + + @classmethod + def parse_inventory_app_file_key( + cls, inv_app_file_key: reg_extensions.CM_KEY_NODE + ) -> Iterator[Tuple[NullableString, _AmcacheEntry]]: + """Parses executable file entries from the `Root\\InventoryApplicationFile` registry key. + + :param inv_app_file_key: The `Root\\InventoryApplicationFile` registry key. + :return: An iterator of tuples, where the first member is the program ID string for correlating + with it's parent `InventoryApplication` program entry, and the second member is the `Amcache` entry. + """ + + valName = Win10InvAppFileValName + + for file_key in inv_app_file_key.get_subkeys(): + + vollog.debug( + f"Parsing Win10 InventoryApplicationFile key {file_key.get_name()}" + ) + + values = { + str(value.get_name()): value + for value in file_key.get_values() + if value.get_name() in [key.value for key in valName] + } + + last_mod = conversion.wintime_to_datetime(file_key.LastWriteTime.QuadPart) + path = _get_string_value(values, valName.LowerCaseLongPath.value) + linkdate = _get_datetime_str_value(values, valName.LinkDate.value) + sha1_hash = _get_string_value(values, valName.FileId.value) + publisher = _get_string_value(values, valName.Publisher.value) + prod_name = _get_string_value(values, valName.ProductName.value) + prod_ver = _get_string_value(values, valName.ProductVersion.value) + program_id = _get_string_value(values, valName.ProgramID.value) + + if isinstance(prod_ver, str): + if isinstance(prod_name, str): + prod_name = f"{prod_name} {prod_ver}" + else: + prod_name = f"UNKNOWN {prod_ver}" + + yield program_id, _AmcacheEntry( + AmcacheEntryType.File.name, + path=path, + company=publisher, + last_modify_time=last_mod, + compile_time=linkdate, + sha1_hash=( + sha1_hash.lstrip("0000") + if isinstance(sha1_hash, str) + else sha1_hash + ), + product_name=prod_name, + ) + + @classmethod + def parse_driver_binary_key( + cls, driver_binary_key: reg_extensions.CM_KEY_NODE + ) -> Iterator[_AmcacheEntry]: + """Parses information about installed drivers from the `Root\\InventoryDriverBinary` registry key. + + :param driver_binary_key: The `Root\\InventoryDriverBinary` registry key + :return: An iterator of `AmcacheEntry`s + """ + for binary_key in driver_binary_key.get_subkeys(): + + valName = Win10DriverBinaryValName + + values = { + str(value.get_name()): value + for value in binary_key.get_values() + if value.get_name() in [key.value for key in valName] + } + + # Depending on the Windows version, the key name will be either the name + # of the driver, or its SHA1 hash. + if "/" in binary_key.get_name(): + driver_name = binary_key.get_name() + sha1_hash = _get_string_value(values, valName.DriverId.name) + else: + sha1_hash = binary_key.get_name() + driver_name = _get_string_value(values, valName.DriverName.name) + + if isinstance(sha1_hash, str): + sha1_hash = sha1_hash[4:] if sha1_hash.startswith("0000") else sha1_hash + + company, product, service, last_write_time, driver_timestamp = ( + _get_string_value(values, valName.DriverCompany.name), + _get_string_value(values, valName.Product.name), + _get_string_value(values, valName.Service.name), + conversion.wintime_to_datetime(binary_key.LastWriteTime.QuadPart), + _get_datetime_utc_epoch_value(values, valName.DriverTimeStamp.name), + ) + + yield _AmcacheEntry( + entry_type=AmcacheEntryType.Driver.name, + path=driver_name, + company=company, + last_modify_time=last_write_time, + compile_time=driver_timestamp, + sha1_hash=( + sha1_hash.lstrip("0000") + if isinstance(sha1_hash, str) + else sha1_hash + ), + service=service, + product_name=product, + ) + + def _generator(self) -> Iterator[Tuple[int, _AmcacheEntry]]: + kernel = self.context.modules[self.config["kernel"]] + + def indented( + entry_gen: Iterable[_AmcacheEntry], indent: int = 0 + ) -> Iterator[Tuple[int, _AmcacheEntry]]: + for item in entry_gen: + yield indent, item + + # Building the dictionary ahead of time is much better for performance + # vs looking up each service's DLL individually. + amcache = self.get_amcache_hive(self.context, self.config_path, kernel) + if amcache is None: + return + + try: + yield from indented( + self.parse_driver_binary_key( + amcache.get_key("Root\\InventoryDriverBinary") # type: ignore + ) + ) + except KeyError: + # Registry key not found + pass + + try: + programs: Dict[str, _AmcacheEntry] = { + program_id: entry + for program_id, entry in self.parse_programs_key( + amcache.get_key("Root\\Programs") + ) # type: ignore + } + except KeyError: + programs = {} + + try: + files = sorted( + list( + self.parse_file_key(amcache.get_key("Root\\File")), # type: ignore + ), + key=_entry_sort_key, + ) + except KeyError: + files = [] + + for program_id, file_entries in itertools.groupby( + files, + key=_entry_sort_key, + ): + files_indent = 0 + if isinstance(program_id, str): + try: + program_entry = programs.pop(program_id.strip().strip("\u0000")) + yield (0, program_entry) + + files_indent = 1 + except KeyError: + # No parent program for this file entry + pass + for _, entry in file_entries: + yield files_indent, entry + + for empty_program in programs.values(): + yield 0, empty_program + + try: + programs: Dict[str, _AmcacheEntry] = dict( + self.parse_inventory_app_key( + amcache.get_key("Root\\InventoryApplication") # type: ignore + ) + ) + except KeyError: + programs = {} + + try: + files = sorted( + list( + self.parse_inventory_app_file_key(amcache.get_key("Root\\InventoryApplicationFile")), # type: ignore + ), + key=_entry_sort_key, + ) + except KeyError: + files = [] + + for program_id, file_entries in itertools.groupby( + files, + key=_entry_sort_key, + ): + files_indent = 0 + + if isinstance(program_id, str): + try: + program_entry = programs.pop(program_id.strip().strip("\u0000")) + yield (0, program_entry) + files_indent = 1 + except KeyError: + # No parent program for this file entry + pass + + for _, entry in file_entries: + yield files_indent, entry + + for empty_program in programs.values(): + yield 0, empty_program + + def run(self): + + return renderers.TreeGrid( + [ + ("EntryType", str), + ("Path", str), + ("Company", str), + ("LastModifyTime", datetime.datetime), + ("LastModifyTime2", datetime.datetime), + ("InstallTime", datetime.datetime), + ("CompileTime", datetime.datetime), + ("SHA1", str), + ("Service", str), + ("ProductName", str), + ], + ( + (indent, dataclasses.astuple(entry)) + for indent, entry in self._generator() + ), + ) diff --git a/volatility3/framework/plugins/windows/registry/hivelist.py b/volatility3/framework/plugins/windows/registry/hivelist.py index 1cc76dad6..ddc9c1855 100644 --- a/volatility3/framework/plugins/windows/registry/hivelist.py +++ b/volatility3/framework/plugins/windows/registry/hivelist.py @@ -2,9 +2,9 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import Iterator, List, Tuple, Iterable, Optional +from typing import Iterator, List, Optional, Tuple -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.framework.renderers import format_hints @@ -140,8 +140,8 @@ class HiveList(interfaces.plugins.PluginInterface): layer_name: str, symbol_table: str, filter_string: Optional[str] = None, - hive_offsets: List[int] = None, - ) -> Iterable[registry.RegistryHive]: + hive_offsets: Optional[List[int]] = None, + ) -> Iterator[registry.RegistryHive]: """Walks through a registry, hive by hive returning the constructed registry layer name. @@ -200,7 +200,7 @@ class HiveList(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - filter_string: str = None, + filter_string: Optional[str] = None, ) -> Iterator[interfaces.objects.ObjectInterface]: """Lists all the hives in the primary layer. diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 51be0841c..2e401caa3 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -5,7 +5,7 @@ import contextlib import enum import logging import struct -from typing import Iterable, Optional, Union +from typing import Iterator, Optional, Union, cast from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.layers.registry import ( @@ -39,6 +39,29 @@ class RegValueTypes(enum.Enum): return cls(RegValueTypes.REG_UNKNOWN) +INTEGER_TYPES = [ + RegValueTypes.REG_DWORD, + RegValueTypes.REG_QWORD, + RegValueTypes.REG_DWORD_BIG_ENDIAN, + RegValueTypes.REG_DWORD_BIG_ENDIAN, +] + +STRING_TYPES = [ + RegValueTypes.REG_SZ, + RegValueTypes.REG_MULTI_SZ, + RegValueTypes.REG_EXPAND_SZ, + RegValueTypes.REG_LINK, +] + +BINARY_TYPES = [ + RegValueTypes.REG_RESOURCE_LIST, + RegValueTypes.REG_BINARY, + RegValueTypes.REG_FULL_RESOURCE_DESCRIPTOR, + RegValueTypes.REG_RESOURCE_REQUIREMENTS_LIST, + RegValueTypes.REG_NONE, +] + + class RegKeyFlags(enum.IntEnum): KEY_IS_VOLATILE = 0x01 KEY_HIVE_EXIT = 0x02 @@ -142,7 +165,7 @@ class CM_KEY_NODE(objects.StructType): ) return bool(self.vol.offset & 0x80000000) - def get_subkeys(self) -> Iterable[interfaces.objects.ObjectInterface]: + def get_subkeys(self) -> Iterator["CM_KEY_NODE"]: """Returns a list of the key nodes.""" hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): @@ -152,9 +175,18 @@ class CM_KEY_NODE(objects.StructType): subkey_node = hive.get_cell(self.SubKeyLists[index]).u.KeyIndex yield from self._get_subkeys_recursive(hive, subkey_node) + def get_value(self, value_name) -> Optional["CM_KEY_VALUE"]: + for value in self.get_values(): + if value.name == value_name: + return value + return next( + (value for value in self.get_values() if value.get_name() == value_name), + None, + ) + def _get_subkeys_recursive( self, hive: RegistryHive, node: interfaces.objects.ObjectInterface - ) -> Iterable[interfaces.objects.ObjectInterface]: + ) -> Iterator["CM_KEY_NODE"]: """Recursively descend a node returning subkeys.""" # The keylist appears to include 4 bytes of key name after each value # We can either double the list and only use the even items, or @@ -170,7 +202,7 @@ class CM_KEY_NODE(objects.StructType): elif signature == "lh" or signature == "lf": listjump = 2 elif node.vol.type_name.endswith(constants.BANG + "_CM_KEY_NODE"): - yield node + yield cast("CM_KEY_NODE", node) else: vollog.debug( "Unexpected node type encountered when traversing subkeys: {}, signature: {}".format( @@ -200,7 +232,7 @@ class CM_KEY_NODE(objects.StructType): continue yield from self._get_subkeys_recursive(hive, subnode) - def get_values(self) -> Iterable[interfaces.objects.ObjectInterface]: + def get_values(self) -> Iterator["CM_KEY_VALUE"]: """Returns a list of the Value nodes for a key.""" hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): @@ -216,13 +248,14 @@ class CM_KEY_NODE(objects.StructType): except (RegistryInvalidIndex, RegistryFormatException) as excp: vollog.debug(f"Invalid address {excp}") continue - if node.vol.type_name.endswith(constants.BANG + "_CM_KEY_VALUE"): + if isinstance(node, CM_KEY_VALUE): yield node + except (exceptions.InvalidAddressException, RegistryFormatException) as excp: vollog.debug(f"Invalid address in get_values iteration: {excp}") return None - def get_name(self) -> interfaces.objects.ObjectInterface: + def get_name(self) -> str: """Gets the name for the current key node""" namelength = self.NameLength self.Name.count = namelength @@ -249,6 +282,10 @@ class CM_KEY_VALUE(objects.StructType): self.Name.count = namelength return self.Name.cast("string", max_length=namelength, encoding="latin-1") + def get_type(self) -> RegValueTypes: + """Get the type of the registry value""" + return RegValueTypes(self.Type) + def decode_data(self) -> Union[int, bytes]: """Properly decodes the data associated with the value node""" # Determine if the data is stored inline @@ -293,29 +330,28 @@ class CM_KEY_VALUE(objects.StructType): # but the length at the start could be negative so just adding 4 to jump past it data = layer.read(self.Data + 4, datalen) - self_type = RegValueTypes(self.Type) - if self_type == RegValueTypes.REG_DWORD: + if self.get_type() == RegValueTypes.REG_DWORD: if len(data) != struct.calcsize("L"): raise ValueError( f"Size of data does not match the type of registry value {self.get_name()}" ) (res,) = struct.unpack(">L", data) return res - if self_type == RegValueTypes.REG_QWORD: + if self.get_type() == RegValueTypes.REG_QWORD: if len(data) != struct.calcsize(" Date: Mon, 7 Oct 2024 13:14:32 -0500 Subject: [PATCH 454/979] Remove unused method --- .../framework/symbols/windows/extensions/registry.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 2e401caa3..9b09c7d98 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -175,15 +175,6 @@ class CM_KEY_NODE(objects.StructType): subkey_node = hive.get_cell(self.SubKeyLists[index]).u.KeyIndex yield from self._get_subkeys_recursive(hive, subkey_node) - def get_value(self, value_name) -> Optional["CM_KEY_VALUE"]: - for value in self.get_values(): - if value.name == value_name: - return value - return next( - (value for value in self.get_values() if value.get_name() == value_name), - None, - ) - def _get_subkeys_recursive( self, hive: RegistryHive, node: interfaces.objects.ObjectInterface ) -> Iterator["CM_KEY_NODE"]: From e46d24397db29f3eb61a872b3df62b4974f24041 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 7 Oct 2024 13:17:01 -0500 Subject: [PATCH 455/979] Code review fixes --- .../framework/plugins/windows/amcache.py | 108 ++++++++++-------- 1 file changed, 58 insertions(+), 50 deletions(-) diff --git a/volatility3/framework/plugins/windows/amcache.py b/volatility3/framework/plugins/windows/amcache.py index 5838c62a0..33bae718d 100644 --- a/volatility3/framework/plugins/windows/amcache.py +++ b/volatility3/framework/plugins/windows/amcache.py @@ -5,8 +5,8 @@ import dataclasses import datetime import enum -import logging import itertools +import logging from typing import Dict, Iterable, Iterator, List, Optional, Tuple, Union from volatility3.framework import interfaces, renderers @@ -132,6 +132,7 @@ class _AmcacheEntry: sha1_hash: NullableString = renderers.NotApplicableValue() service: NullableString = renderers.NotApplicableValue() product_name: NullableString = renderers.NotApplicableValue() + product_version: NullableString = renderers.NotApplicableValue() def _entry_sort_key(entry_tuple: Tuple[NullableString, _AmcacheEntry]) -> str: @@ -279,6 +280,10 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): correlating `Root\\Program` entries, and the second member is the `AmcacheEntry`. """ + val_enum = Win8FileValName + + wanted_values = [key.value for key in val_enum] + for file_entry_key in itertools.chain( *(key.get_subkeys() for key in file_key.get_subkeys()) ): @@ -286,27 +291,27 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): values = { str(value.get_name()): value for value in file_entry_key.get_values() - if value.get_name() in [key.value for key in Win8FileValName] + if value.get_name() in wanted_values } - program_id = _get_string_value(values, Win8FileValName.ProgramID.value) - path = _get_string_value(values, Win8FileValName.Path.value) - company = _get_string_value(values, Win8FileValName.Company.value) + program_id = _get_string_value(values, val_enum.ProgramID.value) + path = _get_string_value(values, val_enum.Path.value) + company = _get_string_value(values, val_enum.Company.value) last_mod_time = _get_datetime_filetime_value( - values, Win8FileValName.LastModTime.value + values, val_enum.LastModTime.value ) last_mod_time_2 = _get_datetime_filetime_value( - values, Win8FileValName.LastModTime2.value + values, val_enum.LastModTime2.value ) install_time = _get_datetime_filetime_value( - values, Win8FileValName.CreateTime.value + values, val_enum.CreateTime.value ) compile_time = _get_datetime_utc_epoch_value( - values, Win8FileValName.CompileTime.value + values, val_enum.CompileTime.value ) - sha1_hash = _get_string_value(values, Win8FileValName.SHA1Hash.value) + sha1_hash = _get_string_value(values, val_enum.SHA1Hash.value) vollog.debug(f"Found sha1hash {sha1_hash}") - product_name = _get_string_value(values, Win8FileValName.Product.value) + product_name = _get_string_value(values, val_enum.Product.value) yield program_id, _AmcacheEntry( AmcacheEntryType.File.name, @@ -335,21 +340,24 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): :return: An iterator of tuples, where the first member is the program ID string for correlating `Root\\File` entries, and the second member is the `AmcacheEntry`. """ + val_enum = Win8ProgramValName + + wanted_values = [key.value for key in val_enum] for program_key in programs_key.get_subkeys(): values = { str(value.get_name()): value for value in program_key.get_values() - if value.get_name() in [key.value for key in Win8ProgramValName] + if value.get_name() in wanted_values } vollog.debug(f"Parsing Win8 Program key {program_key.get_name()}") program_id = program_key.get_name().strip().strip("\u0000") - product = _get_string_value(values, Win8ProgramValName.Product.value) - company = _get_string_value(values, Win8ProgramValName.Publisher.value) + product = _get_string_value(values, val_enum.Product.value) + company = _get_string_value(values, val_enum.Publisher.value) install_time = _get_datetime_utc_epoch_value( - values, Win8ProgramValName.InstallTime.value + values, val_enum.InstallTime.value ) - _version = _get_string_value(values, Win8ProgramValName.Version.value) + _version = _get_string_value(values, val_enum.Version.value) if isinstance(_version, str): if isinstance(product, str): @@ -378,29 +386,29 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): :return: An iterator of tuples, where the first member is the program ID string for correlating `Root\\InventoryApplicationFile` entries, and the second member is the `AmcacheEntry`. """ + val_enum = Win10InvAppValName + + wanted_values = [key.value for key in val_enum] + for program_key in inv_app_key.get_subkeys(): program_id = program_key.get_name() values = { str(value.get_name()): value for value in program_key.get_values() - if value.get_name() in [key.value for key in Win10InvAppValName] + if value.get_name() in wanted_values } - name = _get_string_value(values, Win10InvAppValName.Name.value) - version = _get_string_value(values, Win10InvAppValName.Version.value) - publisher = _get_string_value(values, Win10InvAppValName.Publisher.value) - path = _get_string_value(values, Win10InvAppValName.RootDirPath.value) - install_date = _get_datetime_str_value( - values, Win10InvAppValName.InstallDate.value - ) + name = _get_string_value(values, val_enum.Name.value) + version = _get_string_value(values, val_enum.Version.value) + publisher = _get_string_value(values, val_enum.Publisher.value) + path = _get_string_value(values, val_enum.RootDirPath.value) + install_date = _get_datetime_str_value(values, val_enum.InstallDate.value) last_mod = conversion.wintime_to_datetime( program_key.LastWriteTime.QuadPart ) product: str = name if isinstance(name, str) else "UNKNOWN" # type: ignore - if isinstance(version, str): - product += " " + version yield program_id.strip().strip("\u0000"), _AmcacheEntry( AmcacheEntryType.Program.name, @@ -409,6 +417,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): install_time=install_date, product_name=product, company=publisher, + product_version=version, ) @classmethod @@ -422,7 +431,9 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): with it's parent `InventoryApplication` program entry, and the second member is the `Amcache` entry. """ - valName = Win10InvAppFileValName + val_enum = Win10InvAppFileValName + + wanted_values = [key.value for key in val_enum] for file_key in inv_app_file_key.get_subkeys(): @@ -433,23 +444,17 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): values = { str(value.get_name()): value for value in file_key.get_values() - if value.get_name() in [key.value for key in valName] + if value.get_name() in wanted_values } last_mod = conversion.wintime_to_datetime(file_key.LastWriteTime.QuadPart) - path = _get_string_value(values, valName.LowerCaseLongPath.value) - linkdate = _get_datetime_str_value(values, valName.LinkDate.value) - sha1_hash = _get_string_value(values, valName.FileId.value) - publisher = _get_string_value(values, valName.Publisher.value) - prod_name = _get_string_value(values, valName.ProductName.value) - prod_ver = _get_string_value(values, valName.ProductVersion.value) - program_id = _get_string_value(values, valName.ProgramID.value) - - if isinstance(prod_ver, str): - if isinstance(prod_name, str): - prod_name = f"{prod_name} {prod_ver}" - else: - prod_name = f"UNKNOWN {prod_ver}" + path = _get_string_value(values, val_enum.LowerCaseLongPath.value) + linkdate = _get_datetime_str_value(values, val_enum.LinkDate.value) + sha1_hash = _get_string_value(values, val_enum.FileId.value) + publisher = _get_string_value(values, val_enum.Publisher.value) + prod_name = _get_string_value(values, val_enum.ProductName.value) + prod_ver = _get_string_value(values, val_enum.ProductVersion.value) + program_id = _get_string_value(values, val_enum.ProgramID.value) yield program_id, _AmcacheEntry( AmcacheEntryType.File.name, @@ -463,6 +468,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): else sha1_hash ), product_name=prod_name, + product_version=prod_ver, ) @classmethod @@ -474,34 +480,36 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): :param driver_binary_key: The `Root\\InventoryDriverBinary` registry key :return: An iterator of `AmcacheEntry`s """ - for binary_key in driver_binary_key.get_subkeys(): + val_enum = Win10DriverBinaryValName - valName = Win10DriverBinaryValName + wanted_values = [key.value for key in val_enum] + + for binary_key in driver_binary_key.get_subkeys(): values = { str(value.get_name()): value for value in binary_key.get_values() - if value.get_name() in [key.value for key in valName] + if value.get_name() in wanted_values } # Depending on the Windows version, the key name will be either the name # of the driver, or its SHA1 hash. if "/" in binary_key.get_name(): driver_name = binary_key.get_name() - sha1_hash = _get_string_value(values, valName.DriverId.name) + sha1_hash = _get_string_value(values, val_enum.DriverId.name) else: sha1_hash = binary_key.get_name() - driver_name = _get_string_value(values, valName.DriverName.name) + driver_name = _get_string_value(values, val_enum.DriverName.name) if isinstance(sha1_hash, str): sha1_hash = sha1_hash[4:] if sha1_hash.startswith("0000") else sha1_hash company, product, service, last_write_time, driver_timestamp = ( - _get_string_value(values, valName.DriverCompany.name), - _get_string_value(values, valName.Product.name), - _get_string_value(values, valName.Service.name), + _get_string_value(values, val_enum.DriverCompany.name), + _get_string_value(values, val_enum.Product.name), + _get_string_value(values, val_enum.Service.name), conversion.wintime_to_datetime(binary_key.LastWriteTime.QuadPart), - _get_datetime_utc_epoch_value(values, valName.DriverTimeStamp.name), + _get_datetime_utc_epoch_value(values, val_enum.DriverTimeStamp.name), ) yield _AmcacheEntry( From e622acc7f8bd458b895d89a7a8922c1ee3f92e0e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 7 Oct 2024 13:17:44 -0500 Subject: [PATCH 456/979] Minor version bump: registry extensions API addition --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index a131609c9..ce803a687 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 10 # Number of changes that only add to the interface +VERSION_MINOR = 11 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From dec0d506f91c87f5ceced7a12766f746f0df68e3 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 7 Oct 2024 13:54:07 -0500 Subject: [PATCH 457/979] Fix type signature on get_name --- volatility3/framework/symbols/windows/extensions/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 9b09c7d98..bebfaea89 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -246,7 +246,7 @@ class CM_KEY_NODE(objects.StructType): vollog.debug(f"Invalid address in get_values iteration: {excp}") return None - def get_name(self) -> str: + def get_name(self) -> interfaces.objects.ObjectInterface: """Gets the name for the current key node""" namelength = self.NameLength self.Name.count = namelength From 27ea555b378b89b005bb030ecbdd6f94c872d966 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 7 Oct 2024 13:54:30 -0500 Subject: [PATCH 458/979] Add product version to output columns --- volatility3/framework/plugins/windows/amcache.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/amcache.py b/volatility3/framework/plugins/windows/amcache.py index 33bae718d..8d0a4769e 100644 --- a/volatility3/framework/plugins/windows/amcache.py +++ b/volatility3/framework/plugins/windows/amcache.py @@ -357,13 +357,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): install_time = _get_datetime_utc_epoch_value( values, val_enum.InstallTime.value ) - _version = _get_string_value(values, val_enum.Version.value) - - if isinstance(_version, str): - if isinstance(product, str): - product = f"{product} {_version}" - else: - product = f"UNKNOWN {_version}" + version = _get_string_value(values, val_enum.Version.value) yield program_id, _AmcacheEntry( AmcacheEntryType.Program.name, @@ -373,6 +367,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), install_time=install_time, product_name=product, + product_version=version, ) @classmethod @@ -494,11 +489,11 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Depending on the Windows version, the key name will be either the name # of the driver, or its SHA1 hash. - if "/" in binary_key.get_name(): - driver_name = binary_key.get_name() + if "/" in str(binary_key.get_name()): + driver_name = str(binary_key.get_name()) sha1_hash = _get_string_value(values, val_enum.DriverId.name) else: - sha1_hash = binary_key.get_name() + sha1_hash = str(binary_key.get_name()) driver_name = _get_string_value(values, val_enum.DriverName.name) if isinstance(sha1_hash, str): @@ -646,6 +641,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ("SHA1", str), ("Service", str), ("ProductName", str), + ("ProductVersion", str), ], ( (indent, dataclasses.astuple(entry)) From 996875c059bd8f817e98a40bd7e782070798275d Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 7 Oct 2024 16:13:12 -0500 Subject: [PATCH 459/979] #1298 - include hive name in key path --- volatility3/framework/plugins/windows/registry/printkey.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 180f8f9d9..4fe3f97fb 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -20,7 +20,7 @@ class PrintKey(interfaces.plugins.PluginInterface): """Lists the registry keys under a hive or specific key value.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -282,7 +282,7 @@ class PrintKey(interfaces.plugins.PluginInterface): renderers.UnreadableValue(), format_hints.Hex(hive.hive_offset), "Key", - "?\\" + (key or ""), + f"{hive.get_name()}\\" + (key or ""), renderers.UnreadableValue(), renderers.UnreadableValue(), renderers.UnreadableValue(), From 9addf6bee815bf6ad4a3333bb2ba155a5ad8ed06 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 7 Oct 2024 22:56:20 +0100 Subject: [PATCH 460/979] Remove the old .style.yapf This was used before we shifted to black permanently. It allowed additional spacing between keyword arguments and their default values, but it was inconsisnent and could get in a state where it would flip between two weights, so every "fix" would then be "fixed" back again. No longer feasible for a project of this size. --- .style.yapf | 261 ---------------------------------------------------- 1 file changed, 261 deletions(-) delete mode 100644 .style.yapf diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index 3f154e07b..000000000 --- a/.style.yapf +++ /dev/null @@ -1,261 +0,0 @@ -[style] -# Align closing bracket with visual indentation. -align_closing_bracket_with_visual_indent=True - -# Allow dictionary keys to exist on multiple lines. For example: -# -# x = { -# ('this is the first element of a tuple', -# 'this is the second element of a tuple'): -# value, -# } -allow_multiline_dictionary_keys=False - -# Allow lambdas to be formatted on more than one line. -allow_multiline_lambdas=False - -# Allow splits before the dictionary value. -allow_split_before_dict_value=True - -# Number of blank lines surrounding top-level function and class -# definitions. -blank_lines_around_top_level_definition=2 - -# Insert a blank line before a class-level docstring. -blank_line_before_class_docstring=False - -# Insert a blank line before a module docstring. -blank_line_before_module_docstring=False - -# Insert a blank line before a 'def' or 'class' immediately nested -# within another 'def' or 'class'. For example: -# -# class Foo: -# # <------ this blank line -# def method(): -# ... -blank_line_before_nested_class_or_def=True - -# Do not split consecutive brackets. Only relevant when -# dedent_closing_brackets is set. For example: -# -# call_func_that_takes_a_dict( -# { -# 'key1': 'value1', -# 'key2': 'value2', -# } -# ) -# -# would reformat to: -# -# call_func_that_takes_a_dict({ -# 'key1': 'value1', -# 'key2': 'value2', -# }) -coalesce_brackets=False - -# The column limit. -column_limit=120 - -# The style for continuation alignment. Possible values are: -# -# - SPACE: Use spaces for continuation alignment. This is default behavior. -# - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns -# (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs) for continuation -# alignment. -# - LESS: Slightly left if cannot vertically align continuation lines with -# indent characters. -# - VALIGN-RIGHT: Vertically align continuation lines with indent -# characters. Slightly right (one more indent character) if cannot -# vertically align continuation lines with indent characters. -# -# For options FIXED, and VALIGN-RIGHT are only available when USE_TABS is -# enabled. -continuation_align_style=SPACE - -# Indent width used for line continuations. -continuation_indent_width=4 - -# Put closing brackets on a separate line, dedented, if the bracketed -# expression can't fit in a single line. Applies to all kinds of brackets, -# including function definitions and calls. For example: -# -# config = { -# 'key1': 'value1', -# 'key2': 'value2', -# } # <--- this bracket is dedented and on a separate line -# -# time_series = self.remote_client.query_entity_counters( -# entity='dev3246.region1', -# key='dns.query_latency_tcp', -# transform=Transformation.AVERAGE(window=timedelta(seconds=60)), -# start_ts=now()-timedelta(days=3), -# end_ts=now(), -# ) # <--- this bracket is dedented and on a separate line -dedent_closing_brackets=False - -# Disable the heuristic which places each list element on a separate line -# if the list is comma-terminated. -disable_ending_comma_heuristic=False - -# Place each dictionary entry onto its own line. -each_dict_entry_on_separate_line=True - -# The regex for an i18n comment. The presence of this comment stops -# reformatting of that line, because the comments are required to be -# next to the string they translate. -i18n_comment= - -# The i18n function call names. The presence of this function stops -# reformatting on that line, because the string it has cannot be moved -# away from the i18n comment. -i18n_function_call= - -# Indent the dictionary value if it cannot fit on the same line as the -# dictionary key. For example: -# -# config = { -# 'key1': -# 'value1', -# 'key2': value1 + -# value2, -# } -indent_dictionary_value=False - -# The number of columns to use for indentation. -indent_width=4 - -# Join short lines into one line. E.g., single line 'if' statements. -join_multiple_lines=True - -# Do not include spaces around selected binary operators. For example: -# -# 1 + 2 * 3 - 4 / 5 -# -# will be formatted as follows when configured with "*,/": -# -# 1 + 2*3 - 4/5 -# -no_spaces_around_selected_binary_operators= - -# Use spaces around default or named assigns. -spaces_around_default_or_named_assign=True - -# Use spaces around the power operator. -spaces_around_power_operator=True - -# The number of spaces required before a trailing comment. -spaces_before_comment=2 - -# Insert a space between the ending comma and closing bracket of a list, -# etc. -space_between_ending_comma_and_closing_bracket=True - -# Split before arguments -split_all_comma_separated_values=False - -# Split before arguments if the argument list is terminated by a -# comma. -split_arguments_when_comma_terminated=False - -# Set to True to prefer splitting before '&', '|' or '^' rather than -# after. -split_before_bitwise_operator=True - -# Split before the closing bracket if a list or dict literal doesn't fit on -# a single line. -split_before_closing_bracket=True - -# Split before a dictionary or set generator (comp_for). For example, note -# the split before the 'for': -# -# foo = { -# variable: 'Hello world, have a nice day!' -# for variable in bar if variable != 42 -# } -split_before_dict_set_generator=True - -# Split before the '.' if we need to split a longer expression: -# -# foo = ('This is a really long string: {}, {}, {}, {}'.format(a, b, c, d)) -# -# would reformat to something like: -# -# foo = ('This is a really long string: {}, {}, {}, {}' -# .format(a, b, c, d)) -split_before_dot=False - -# Split after the opening paren which surrounds an expression if it doesn't -# fit on a single line. -split_before_expression_after_opening_paren=False - -# If an argument / parameter list is going to be split, then split before -# the first argument. -split_before_first_argument=False - -# Set to True to prefer splitting before 'and' or 'or' rather than -# after. -split_before_logical_operator=True - -# Split named assignments onto individual lines. -split_before_named_assigns=True - -# Set to True to split list comprehensions and generators that have -# non-trivial expressions and multiple clauses before each of these -# clauses. For example: -# -# result = [ -# a_long_var + 100 for a_long_var in xrange(1000) -# if a_long_var % 10] -# -# would reformat to something like: -# -# result = [ -# a_long_var + 100 -# for a_long_var in xrange(1000) -# if a_long_var % 10] -split_complex_comprehension=True - -# The penalty for splitting right after the opening bracket. -split_penalty_after_opening_bracket=200 - -# The penalty for splitting the line after a unary operator. -split_penalty_after_unary_operator=10000 - -# The penalty for splitting right before an if expression. -split_penalty_before_if_expr=0 - -# The penalty of splitting the line around the '&', '|', and '^' -# operators. -split_penalty_bitwise_operator=300 - -# The penalty for splitting a list comprehension or generator -# expression. -split_penalty_comprehension=80 - -# The penalty for characters over the column limit. -split_penalty_excess_character=7000 - -# The penalty incurred by adding a line split to the unwrapped line. The -# more line splits added the higher the penalty. -split_penalty_for_added_line_split=30 - -# The penalty of splitting a list of "import as" names. For example: -# -# from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, -# long_argument_2, -# long_argument_3) -# -# would reformat to something like: -# -# from a_very_long_or_indented_module_name_yada_yad import ( -# long_argument_1, long_argument_2, long_argument_3) -split_penalty_import_names=0 - -# The penalty of splitting the line around the 'and' and 'or' -# operators. -split_penalty_logical_operator=300 - -# Use the Tab character for indentation. -use_tabs=False - From e419b67581e6d31d7959480d5904e283fa2929a1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 8 Oct 2024 02:06:04 +0200 Subject: [PATCH 461/979] introduce dirty state ioc --- .../framework/plugins/windows/malfind.py | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 079274d69..5ce2279cc 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -122,10 +122,30 @@ class Malfind(interfaces.plugins.PluginInterface): vadinfo.winnt_protections, ) write_exec = "EXECUTE" in protection_string and "WRITE" in protection_string + dirty_page_check = False - # the write/exec check applies to everything if not write_exec: - continue + """ + # Inspect "PAGE_EXECUTE_READ" VAD pages to detect + # non writable memory regions having been injected + # using elevated WriteProcessMemory(). + """ + if "EXECUTE" in protection_string: + for page in range( + vad.get_start(), vad.get_end(), proc_layer.page_size + ): + try: + # If we have a dirty page in a non writable "EXECUTE" region, it is suspicious. + if proc_layer.is_dirty(page): + dirty_page_check = True + break + except exceptions.InvalidAddressException: + # Abort as it is likely that other addresses in the same range will also fail + break + if not dirty_page_check: + continue + else: + continue if (vad.get_private_memory() == 1 and vad.get_tag() == "VadS") or ( vad.get_private_memory() == 0 @@ -134,6 +154,11 @@ class Malfind(interfaces.plugins.PluginInterface): if cls.is_vad_empty(proc_layer, vad): continue + if dirty_page_check: + # Useful information to investigate the page content with volshell afterwards. + vollog.warning( + f"[proc_id {proc_id}] Found suspicious DIRTY + {protection_string} page at {hex(page)}", + ) data = proc_layer.read(vad.get_start(), 64, pad=True) yield vad, data From f92dfdd33b504e781d9ba660b8fba625618e33a9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 8 Oct 2024 15:53:07 +1100 Subject: [PATCH 462/979] Linux: ptrace: remove patch number in plugin requirement --- volatility3/framework/plugins/linux/ptrace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index 793a1b3a1..1a37b8782 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -31,7 +31,7 @@ class Ptrace(plugins.PluginInterface): architectures=LINUX_ARCHS, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 2, 1) + name="pslist", plugin=pslist.PsList, version=(2, 2, 0) ), ] From eb2d4b176518d8337a82ebc61b99ed04186904e3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 8 Oct 2024 15:54:48 +1100 Subject: [PATCH 463/979] Linux: ptrace: add ptrace functions to the task_struct object extension --- .../symbols/linux/extensions/__init__.py | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fdd34403a..7f6d44319 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,13 +13,12 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion -from volatility3.framework.configuration import requirements from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES -from volatility3.framework.constants.linux import CAPABILITIES +from volatility3.framework.constants.linux import CAPABILITIES, PT_FLAGS from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed @@ -374,6 +373,41 @@ class task_struct(generic.GenericIntelProcess): ): yield task + @property + def is_being_ptraced(self) -> bool: + """Returns True if this task is being traced using ptrace""" + return self.ptrace != 0 + + @property + def is_ptracing(self) -> bool: + """Returns True if this task is tracing other tasks using ptrace""" + is_tracing = ( + self.ptraced.next.is_readable() + and self.ptraced.next.dereference().vol.offset != self.ptraced.vol.offset + ) + return is_tracing + + def get_ptrace_tracer_tid(self) -> Optional[int]: + """Returns the tracer's TID tracing this task""" + return self.parent.pid if self.is_being_ptraced else None + + def get_ptrace_tracee_tids(self) -> List[int]: + """Returns the list of TIDs being traced by this task""" + task_symbol_table_name = self.get_symbol_table_name() + + task_struct_symname = f"{task_symbol_table_name}{constants.BANG}task_struct" + tracing_tid_list = [ + task_being_traced.pid + for task_being_traced in self.ptraced.to_list( + task_struct_symname, "ptrace_entry" + ) + ] + return tracing_tid_list + + def get_ptrace_tracee_flags(self) -> Optional[str]: + """Returns a string with the ptrace flags""" + return PT_FLAGS(self.ptrace).flags if self.is_being_ptraced else None + class fs_struct(objects.StructType): def get_root_dentry(self): From 3c4b9996ce78a00ef5121782fd75a45f90a557ae Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 8 Oct 2024 16:00:35 +1100 Subject: [PATCH 464/979] Linux: ptrace: move tracee TIDs to multiple rows --- volatility3/framework/plugins/linux/ptrace.py | 103 +++++++----------- 1 file changed, 40 insertions(+), 63 deletions(-) diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index 1a37b8782..a1964d960 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -3,10 +3,9 @@ # import logging -from typing import List +from typing import List, Iterator -from volatility3.framework import renderers, interfaces, constants, objects -from volatility3.framework.constants.linux import PT_FLAGS +from volatility3.framework import renderers, interfaces from volatility3.framework.constants.architectures import LINUX_ARCHS from volatility3.framework.objects import utility from volatility3.framework.configuration import requirements @@ -17,7 +16,7 @@ vollog = logging.getLogger(__name__) class Ptrace(plugins.PluginInterface): - """Enumerates tracer and tracee tasks""" + """Enumerates ptrace's tracer and tracee tasks""" _required_framework_version = (2, 10, 0) _version = (1, 0, 0) @@ -36,85 +35,63 @@ class Ptrace(plugins.PluginInterface): ] @classmethod - def enumerate_ptraced_tasks( + def enumerate_ptrace_tasks( cls, context: interfaces.context.ContextInterface, - symbol_table: str, - ): - vmlinux = context.modules[symbol_table] + vmlinux_module_name: str, + ) -> Iterator[interfaces.objects.ObjectInterface]: + """Enumerates ptrace's tracer and tracee tasks - tsk_struct_symname = vmlinux.symbol_table_name + constants.BANG + "task_struct" + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + + Yields: + A task_struct object + """ tasks = pslist.PsList.list_tasks( context, - symbol_table, + vmlinux_module_name, filter_func=pslist.PsList.create_pid_filter(), include_threads=True, ) for task in tasks: - tracing_tid_list = [ - int(task_being_traced.pid) - for task_being_traced in task.ptraced.to_list( - tsk_struct_symname, "ptrace_entry" - ) + if task.is_being_ptraced or task.is_ptracing: + yield task + + def _generator(self, vmlinux_module_name): + for task in self.enumerate_ptrace_tasks(self.context, vmlinux_module_name): + task_comm = utility.array_to_string(task.comm) + user_pid = task.tgid + user_tid = task.pid + tracer_tid = task.get_ptrace_tracer_tid() or renderers.NotAvailableValue() + tracee_tids = task.get_ptrace_tracee_tids() or [ + renderers.NotAvailableValue() ] + flags = task.get_ptrace_tracee_flags() or renderers.NotAvailableValue() - if task.ptrace == 0 and not tracing_tid_list: - continue - - flags = ( - PT_FLAGS(task.ptrace).flags - if task.ptrace != 0 - else renderers.NotAvailableValue() - ) - - traced_by_tid = ( - task.parent.pid - if task.real_parent != task.parent - else renderers.NotAvailableValue() - ) - - tracing_tids = ",".join(map(str, tracing_tid_list)) - - yield task.comm, task.tgid, task.pid, traced_by_tid, tracing_tids, flags - - def _generator(self, symbol_table): - for fields in self.enumerate_ptraced_tasks(self.context, symbol_table): - yield (0, fields) - - @staticmethod - def format_fields_with_headers(headers, generator): - """Uses the headers type to cast the fields obtained from the generator""" - for level, fields in generator: - formatted_fields = [] - for header, field in zip(headers, fields): - header_type = header[1] - - if isinstance( - field, (header_type, interfaces.renderers.BaseAbsentValue) - ): - formatted_field = field - elif isinstance(field, objects.Array) and header_type is str: - formatted_field = utility.array_to_string(field) - else: - formatted_field = header_type(field) - - formatted_fields.append(formatted_field) - yield level, formatted_fields + for tree, tracing_tid in enumerate(tracee_tids): + fields = [ + task_comm, + user_pid, + user_tid, + tracer_tid, + tracing_tid, + flags, + ] + yield (tree, fields) def run(self): - symbol_table = self.config["kernel"] + vmlinux_module_name = self.config["kernel"] headers = [ ("Process", str), ("PID", int), ("TID", int), ("Traced by TID", int), - ("Tracing TIDs", str), + ("Tracing TID", int), ("Flags", str), ] - return renderers.TreeGrid( - headers, - self.format_fields_with_headers(headers, self._generator(symbol_table)), - ) + return renderers.TreeGrid(headers, self._generator(vmlinux_module_name)) From aa959139409c22156cadb0b8fc91819f4d4f2c34 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 8 Oct 2024 16:03:36 +1100 Subject: [PATCH 465/979] Linux: ptrace: improve header names and variables --- volatility3/framework/plugins/linux/ptrace.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index a1964d960..e501f3294 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -72,16 +72,16 @@ class Ptrace(plugins.PluginInterface): ] flags = task.get_ptrace_tracee_flags() or renderers.NotAvailableValue() - for tree, tracing_tid in enumerate(tracee_tids): + for level, tracee_tid in enumerate(tracee_tids): fields = [ task_comm, user_pid, user_tid, tracer_tid, - tracing_tid, + tracee_tid, flags, ] - yield (tree, fields) + yield (level, fields) def run(self): vmlinux_module_name = self.config["kernel"] @@ -90,8 +90,8 @@ class Ptrace(plugins.PluginInterface): ("Process", str), ("PID", int), ("TID", int), - ("Traced by TID", int), - ("Tracing TID", int), + ("Tracer TID", int), + ("Tracee TID", int), ("Flags", str), ] return renderers.TreeGrid(headers, self._generator(vmlinux_module_name)) From 384042202e19fe5a4761daa74b76bed7fc70f55d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 8 Oct 2024 18:37:32 +1100 Subject: [PATCH 466/979] fix black stable version issue with Python 3.8 --- .github/workflows/black.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 5adab3259..e29ab6f29 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -11,3 +11,5 @@ jobs: with: options: "--check --diff --verbose" src: "./volatility3" + # FIXME: Remove when Volatility3 minimum Python version is >3.8 + version: "24.8.0" From 6b5bef41336b3fbd623aca85499aa39410188fb0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 8 Oct 2024 21:27:29 +1100 Subject: [PATCH 467/979] Linux: ptrace: bumping version patch number to re-run the black linter with #1301 --- volatility3/framework/plugins/linux/ptrace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index e501f3294..34185f912 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -19,7 +19,7 @@ class Ptrace(plugins.PluginInterface): """Enumerates ptrace's tracer and tracee tasks""" _required_framework_version = (2, 10, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 620dd4e67d971e97adab5daea523b60458bc5c35 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 8 Oct 2024 21:55:21 +1100 Subject: [PATCH 468/979] Linux: ptrace: version patch number back to zero --- volatility3/framework/plugins/linux/ptrace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index 34185f912..e501f3294 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -19,7 +19,7 @@ class Ptrace(plugins.PluginInterface): """Enumerates ptrace's tracer and tracee tasks""" _required_framework_version = (2, 10, 0) - _version = (1, 0, 1) + _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From e817e72bc2d82283ab2e942725f809ed31f5ca80 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 8 Oct 2024 22:52:44 +0100 Subject: [PATCH 469/979] Support hiding columns in the CLI --- volatility3/cli/__init__.py | 13 ++++ volatility3/cli/text_renderer.py | 92 +++++++++++++++++++++-------- volatility3/framework/exceptions.py | 4 ++ 3 files changed, 85 insertions(+), 24 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 50b5579af..b4e993dac 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -256,6 +256,14 @@ class CommandLine: default=[], action="append", ) + parser.add_argument( + "--hide-columns", + help="Case-insensitive space separated list of prefixes to determine which columns to hide in the output if provided", + default=None, + action="extend", + nargs="*", + type=str, + ) parser.set_defaults(**default_config) @@ -488,6 +496,7 @@ class CommandLine: grid = constructed.run() renderer = renderers[args.renderer]() renderer.filter = text_filter.CLIFilter(grid, args.filters) + renderer.column_hide_list = args.hide_columns renderer.render(grid) except exceptions.VolatilityException as excp: self.process_exceptions(excp) @@ -615,6 +624,10 @@ class CommandLine: caused_by = [ "A required python module is not installed (install the module and re-run)" ] + elif isinstance(excp, exceptions.RenderException): + general = "Volatility experienced an issue when rendering the output:" + detail = f"{excp}" + caused_by = ["An invalid renderer option, such as no visible columns"] else: general = "Volatility encountered an unexpected situation." detail = "" diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index da0cdf62a..96970d3ce 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -12,7 +12,7 @@ from functools import wraps from typing import Any, Callable, Dict, List, Tuple from volatility3.cli import text_filter -from volatility3.framework import interfaces, renderers +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.renderers import format_hints vollog = logging.getLogger(__name__) @@ -141,6 +141,30 @@ class CLIRenderer(interfaces.renderers.Renderer): name = "unnamed" structured_output = False filter: text_filter.CLIFilter = None + column_hide_list: list = None + + def ignored_columns( + self, + grid: interfaces.renderers.TreeGrid, + ) -> List[interfaces.renderers.Column]: + ignored_column_list = [] + if self.column_hide_list: + for column in grid.columns: + accept = True + for column_prefix in self.column_hide_list: + if column.name.lower().startswith(column_prefix.lower()): + accept = False + if not accept: + ignored_column_list.append(column) + elif self.column_hide_list is None: + return [] + + if len(ignored_column_list) == len(grid.columns): + raise exceptions.RenderException("No visible columns to render") + vollog.info( + f"Hiding columns: {[column.name for column in ignored_column_list]}" + ) + return ignored_column_list class QuickTextRenderer(CLIRenderer): @@ -173,9 +197,11 @@ class QuickTextRenderer(CLIRenderer): outfd = sys.stdout line = [] + ignore_columns = self.ignored_columns(grid) for column in grid.columns: # Ignore the type because namedtuples don't realize they have accessible attributes - line.append(f"{column.name}") + if column not in ignore_columns: + line.append(f"{column.name}") outfd.write("\n{}\n".format("\t".join(line))) def visitor(node: interfaces.renderers.TreeNode, accumulator): @@ -184,7 +210,8 @@ class QuickTextRenderer(CLIRenderer): renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) - line.append(renderer(node.values[column_index])) + if column not in ignore_columns: + line.append(renderer(node.values[column_index])) if self.filter and self.filter.filter(line): return accumulator @@ -245,11 +272,13 @@ class CSVRenderer(CLIRenderer): grid: The TreeGrid object to render """ outfd = sys.stdout + ignore_columns = self.ignored_columns(grid) header_list = ["TreeDepth"] for column in grid.columns: # Ignore the type because namedtuples don't realize they have accessible attributes - header_list.append(f"{column.name}") + if column not in ignore_columns: + header_list.append(f"{column.name}") writer = csv.DictWriter( outfd, header_list, lineterminator="\n", escapechar="\\" @@ -265,7 +294,10 @@ class CSVRenderer(CLIRenderer): column.type, self._type_renderers["default"] ) row[f"{column.name}"] = renderer(node.values[column_index]) - line.append(row[f"{column.name}"]) + if column not in ignore_columns: + line.append(row[f"{column.name}"]) + else: + del row[f"{column.name}"] if self.filter and self.filter.filter(line): return accumulator @@ -303,6 +335,7 @@ class PrettyTextRenderer(CLIRenderer): sys.stderr.write("Formatting...\n") + ignore_columns = self.ignored_columns(grid) display_alignment = ">" column_separator = " | " @@ -335,7 +368,8 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths[column.name] = max( max_column_widths.get(column.name, len(column.name)), field_width ) - line[column] = data.split("\n") + if column not in ignore_columns: + line[column] = data.split("\n") rendered_line.append(data) if self.filter and self.filter.filter(rendered_line): @@ -354,43 +388,49 @@ class PrettyTextRenderer(CLIRenderer): format_string_list = [ "{0:<" + str(max_column_widths.get(tree_indent_column, 0)) + "s}" ] + column_offset = 0 for column_index, column in enumerate(grid.columns): - format_string_list.append( - "{" - + str(column_index + 1) - + ":" - + display_alignment - + str(max_column_widths[column.name]) - + "s}" - ) + if column not in ignore_columns: + format_string_list.append( + "{" + + str(column_index - column_offset + 1) + + ":" + + display_alignment + + str(max_column_widths[column.name]) + + "s}" + ) + else: + column_offset += 1 format_string = column_separator.join(format_string_list) + "\n" - column_titles = [""] + [column.name for column in grid.columns] + column_titles = [""] + [ + column.name for column in grid.columns if column not in ignore_columns + ] + outfd.write(format_string.format(*column_titles)) for depth, line in final_output: nums_line = max([len(line[column]) for column in line]) for column in line: - line[column] = line[column] + ([""] * (nums_line - len(line[column]))) + if column in ignore_columns: + del line[column] + else: + line[column] = line[column] + ( + [""] * (nums_line - len(line[column])) + ) for index in range(nums_line): if index == 0: outfd.write( format_string.format( "*" * depth, - *[ - self.tab_stop(line[column][index]) - for column in grid.columns - ], + *[self.tab_stop(line[column][index]) for column in line], ) ) else: outfd.write( format_string.format( " " * depth, - *[ - self.tab_stop(line[column][index]) - for column in grid.columns - ], + *[self.tab_stop(line[column][index]) for column in line], ) ) @@ -436,6 +476,8 @@ class JsonRenderer(CLIRenderer): List[interfaces.renderers.TreeNode], ] = ({}, []) + ignore_columns = self.ignored_columns(grid) + def visitor( node: interfaces.renderers.TreeNode, accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]], @@ -445,6 +487,8 @@ class JsonRenderer(CLIRenderer): node_dict: Dict[str, Any] = {"__children": []} line = [] for column_index, column in enumerate(grid.columns): + if column in ignore_columns: + continue renderer = self._type_renderers.get( column.type, self._type_renderers["default"] ) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index f8701683b..c44fb4f2e 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -126,3 +126,7 @@ class OfflineException(VolatilityException): def __str__(self): return f"Volatility 3 is offline: unable to access {self._url}" + + +class RenderException(VolatilityException): + """Thrown if there is an error during rendering""" From 57ef3f587eae3bb0eb46e760e64ebdc9fbd36394 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Sep 2024 14:51:42 -0500 Subject: [PATCH 470/979] Windows: Adds scheduled tasks plugin This adds a plugin 'ScheduledTasks' that can decode binary-encoded scheduled tasks from the Windows registry's SOFTWARE hive using a custom reader that extends the `io.BytesIO` class. Decoding operations are intended to be as fault tolerant as possible, swallowing exceptions and returning `None` to account for smear or missing data. Because each task can have mulitple triggers and multiple actions, a single entry is generated for each trigger + action pair. In the event that the either the actions could not be parsed or the triggers could not be parsed due to missing or smeared data, an entry will still be generated using the available information from the other registry value, since trigger and action data is stored separately. Much more information is decoded than is rendered, this was done intentionally to avoid overpopulating the TreeGrid with less pertinent data and to avoid an explosion of trigger and action-specific fields that may not apply to most other entries. --- volatility3/framework/layers/registry.py | 3 + .../plugins/windows/scheduled_tasks.py | 1796 +++++++++++++++++ volatility3/framework/renderers/conversion.py | 21 + 3 files changed, 1820 insertions(+) create mode 100644 volatility3/framework/plugins/windows/scheduled_tasks.py diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 471d19f68..609832886 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -170,6 +170,9 @@ class RegistryHive(linear.LinearlyMappedLayer): return_list specifies whether the return result will be a single node (default) or a list of nodes from root to the current node (if return_list is true). + + Raises RegistryFormatException if an invalid structure is encountered + Raises KeyError if the key is not found """ root_node = self.get_node(self.root_cell_offset) if not root_node.vol.type_name.endswith(constants.BANG + "_CM_KEY_NODE"): diff --git a/volatility3/framework/plugins/windows/scheduled_tasks.py b/volatility3/framework/plugins/windows/scheduled_tasks.py new file mode 100644 index 000000000..6d3ee586d --- /dev/null +++ b/volatility3/framework/plugins/windows/scheduled_tasks.py @@ -0,0 +1,1796 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import base64 +import binascii +import dataclasses +import datetime +import enum +import io +import itertools +import logging +import struct +import traceback +import unittest +from typing import Dict, Iterator, List, Optional, Tuple, Union + +from volatility3.framework import exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import registry +from volatility3.framework.renderers import conversion +from volatility3.framework.symbols.windows.extensions import registry as reg_extensions +from volatility3.plugins import timeliner +from volatility3.plugins.windows.registry import hivelist + +vollog = logging.getLogger(__name__) + +# Reference: https://cyber.wtf/2022/06/01/windows-registry-analysis-todays-episode-tasks/ + + +class TimeMode(enum.Enum): + """ + Enumeration containing the different time modes that a 'Time' trigger can be configured to run in. + """ + + Once = "Once" + + # run at and repeat every days + Daily = "Daily" + + # run on days of week <(data2 as day_of_week bitmap)> every weeks starting at + Weekly = "Weekly" + + # run in months <(data3 as months bitmap> on days <(data2:data1 as day in month bitmap)> + # starting at + DaysInMonths = "Days In Months" + + # run in months <(data3 as months bitmap> in weeks <(data2 as week bitmap)> + # on days <(data1 as day_of_week bitmap)> starting at + DaysInWeeksInMonths = "Days In Weeks in Months" + + Unknown = "Unknown" + + +class ActionType(enum.Enum): + """ + Enumeration that maps action types to their magic number encodings + """ + + Exe = 0x6666 + ComHandler = 0x7777 + Email = 0x8888 + MessageBox = 0x9999 + + +class TriggerType(enum.Enum): + """ + Enumeration that maps trigger types to their magic number encodings + """ + + WindowsNotificationFacility = 0x6666 + Session = 0x7777 + Registration = 0x8888 + Logon = 0xAAAA + Event = 0xCCCC + Time = 0xDDDD + Idle = 0xEEEE + Boot = 0xFFFF + + +class Weekday(enum.Enum): + """ + Enumeration that contains bitwise values for days of the week. + """ + + Sunday = 0x1 + Monday = 0x2 + Tuesday = 0x4 + Wednesday = 0x8 + Thursday = 0x10 + Friday = 0x20 + Saturday = 0x40 + + +class Months(enum.Enum): + """ + Enumeration that contains bitwise values for months of the year. + """ + + January = 0x1 + February = 0x2 + March = 0x4 + April = 0x8 + May = 0x10 + June = 0x20 + July = 0x40 + August = 0x80 + September = 0x100 + October = 0x200 + November = 0x400 + December = 0x800 + + +class SidType(enum.Enum): + """ + Enumeration that maps SID types to their encoded integer values + """ + + User = 1 + Group = 2 + Domain = 3 + Alias = 4 + WellKnownGroup = 5 + DeletedAccount = 6 + Invalid = 7 + Unknown = 8 + Computer = 9 + Label = 10 + LogonSession = 11 + + +@dataclasses.dataclass +class TaskSchedulerTime: + """ + A class containing datetime information about when a task will run + """ + + is_localized: bool + filetime: Optional[datetime.datetime] + + +@dataclasses.dataclass +class TaskSchedulerTimePeriod: + """ + Class containing information delimiting time periods within scheduled tasks. + """ + + years: int + months: int + weeks: int + days: int + hours: int + minutes: int + seconds: int + + +JOB_BUCKET_FLAGS = { + 0x2: "Run only if idle", + 0x4: "Restart on idle", + 0x8: "Stop on idle end", + 0x10: "Disallow start if on batteries", + 0x20: "Stop if going on batteries", + 0x40: "Start when available", + 0x80: "Run only if network available", + 0x100: "Allow start on demand", + 0x200: "Wake to run", + 0x400: "Execute parallel", + 0x800: "Execute stop existing", + 0x1000: "Execute queue", + 0x2000: "Execute ignore new", + 0x4000: "Logon type s4u", + 0x10000: "Logon type InteractiveToken", + 0x40000: "Logon type Password", + 0x80000: "Logon type InteractiveTokenOrPassword", + 0x400000: "Enabled", + 0x800000: "Hidden", + 0x1000000: "Runlevel highest available", + 0x2000000: "Task", + 0x4000000: "Version", + 0x8000000: "Token SID type none", + 0x10000000: "Token SID type unrestricted", + 0x20000000: "Interval", + 0x40000000: "Allow hard terminate", +} + +NULL = "\u0000" + + +class _ScheduledTasksReader(io.BytesIO): + + def read_task_scheduler_time(self) -> Optional[Tuple[bool, datetime.datetime]]: + is_localized = bool(self.read_aligned_u1()) + filetime = self.decode_filetime() + if filetime is None: + return None + return is_localized, filetime + + def read_filetime(self) -> Optional[datetime.datetime]: + return datetime.datetime.now() + + def seek_relative(self, offset) -> int: + return self.seek(offset, io.SEEK_CUR) + + def read_bool(self, aligned=False) -> Optional[bool]: + try: + val = struct.unpack("?", self.read(1))[0] + if aligned: + self.seek(7) + return val + except struct.error: + return None + + def decode_filetime(self) -> Optional[datetime.datetime]: + filetime = self.read_u8() + if filetime is None: + return None + + if filetime == 0 or filetime == 0xFFFFFFFFFFFFFFFF: + return None + filetime = conversion.wintime_to_datetime(filetime) + if isinstance(filetime, datetime.datetime): + return filetime + else: + return None + + def _read_uint( + self, size: int, format: str, aligned: bool = False + ) -> Optional[int]: + try: + val = struct.unpack(format, self.read(size))[0] + if aligned: + self.seek(8 - size, io.SEEK_CUR) + return val + except struct.error: + return None + + def read_aligned_u1(self) -> Optional[int]: + return self._read_uint(1, "B", True) + + def read_u2(self) -> Optional[int]: + return self._read_uint(2, " Optional[int]: + return self._read_uint(2, " Optional[int]: + return self._read_uint(4, " Optional[int]: + return self._read_uint(8, " Optional[int]: + return self._read_uint(4, " Optional[bytes]: + count = self.read_u4() if not aligned else self.read_aligned_u4() + if count is None: + return None + data = self.read(count) + if aligned: + self.seek((8 - (count % 8)) % 8, io.SEEK_CUR) + return data + + def read_bstring(self, aligned=False) -> Optional[str]: + size = self.read_u4() if not aligned else self.read_aligned_u4() + if size is None: + return None + try: + raw = self.read(size) + val = raw.decode("utf-16le", errors="replace").rstrip(NULL) or None + except UnicodeDecodeError: + val = None + + if aligned: + self.seek((8 - (size % 8)) % 8, io.SEEK_CUR) + + return val + + def read_aligned_bstring_expand_sz(self) -> Optional[str]: + # type: () -> Optional[str] + sz = self.read_aligned_u4() + if sz is None: + return None + byte_count = sz * 2 + 2 + + if sz == 0: + return None + + try: + content = self.read(byte_count).decode("utf-16le") + except UnicodeDecodeError: + content = None + + self.seek((8 - (byte_count % 8)) % 8, io.SEEK_CUR) + return content.rstrip("\x00") if content is not None else None + + def read_tstimeperiod(self) -> Optional[TaskSchedulerTimePeriod]: + values = ( + self.read_u2(), + self.read_u2(), + self.read_u2(), + self.read_u2(), + self.read_u2(), + self.read_u2(), + self.read_u2(), + ) + + if any(value is None for value in values): + return None + + return TaskSchedulerTimePeriod(*values) + + +def _build_guid_name_map(key: reg_extensions.CM_KEY_NODE) -> Dict[str, str]: + mapping = {} + task_id_value = None + for value in key.get_values(): + try: + if value.get_name() == "Id": + task_id_value = value + break + except exceptions.InvalidAddressException: + continue + + if ( + task_id_value is not None + and task_id_value.get_type() == reg_extensions.RegValueTypes.REG_SZ + ): + try: + id_str = task_id_value.decode_data() + except exceptions.InvalidAddressException: + id_str = None + + if isinstance(id_str, bytes): + mapping[id_str.decode("utf-16le", errors="replace").rstrip(NULL)] = str( + key.get_name() + ) + + for subkey in key.get_subkeys(): + mapping.update(_build_guid_name_map(subkey)) + return mapping + + +@dataclasses.dataclass +class TaskAction: + action_type: ActionType + action: str + action_args: Optional[str] + working_directory: Optional[str] + + @classmethod + def decode_messagebox_action( + cls, reader: _ScheduledTasksReader + ) -> Optional["TaskAction"]: + caption, content = reader.read_bstring(), reader.read_bstring() + return cls( + ActionType.MessageBox, + f'"{caption or ""}": {content or ""}', + None, + None, + ) + + @classmethod + def _decode_exe_action( + cls, reader: _ScheduledTasksReader, version: int + ) -> Optional["TaskAction"]: + command = reader.read_bstring() + args = reader.read_bstring() + if command is None or args is None: + return None + + workdir = reader.read_bstring() + if version == 3: + _flags = reader.read_u2() + + return cls(ActionType.Exe, command, args, workdir) + + @classmethod + def _decode_email_action( + cls, reader: _ScheduledTasksReader + ) -> Optional["TaskAction"]: + props = { + "From": reader.read_bstring(), + "To": reader.read_bstring(), + "Cc": reader.read_bstring(), + "Bcc": reader.read_bstring(), + "Reply_to": reader.read_bstring(), + "Server": reader.read_bstring(), + "Subject": reader.read_bstring(), + "Body": reader.read_bstring(), + } + + num_attachment_filenames = reader.read_u4() + if num_attachment_filenames is not None: + + attachment_filenames = [ + reader.read_bstring() for _ in range(num_attachment_filenames) + ] + + props["Attachments"] = ( + "<" + + ", ".join( + filename + for filename in attachment_filenames + if filename is not None + ) + + ">" + ) + + num_headers = reader.read_u4() + if num_headers is not None: + headers = [ + (reader.read_bstring(), reader.read_bstring()) + for _ in range(num_headers) + ] + + props["Headers"] = ( + "<" + + ", ".join( + f"{field}: {value}" + for field, value in headers + if field is not None and value is not None + ) + + ">" + ) + + cls( + ActionType.Email, + ", ".join( + f"{key}: {value}" for key, value in props.items() if value is not None + ), + None, + None, + ) + + @classmethod + def _decode_comhandler_action( + cls, reader: _ScheduledTasksReader + ) -> Optional["TaskAction"]: + guid_raw = reader.read(16) + if not guid_raw and len(guid_raw) == 16: + return None + clsid = conversion.windows_bytes_to_guid(guid_raw) + args = reader.read_bstring() + + return cls(ActionType.ComHandler, clsid, args, None) + + +@dataclasses.dataclass +class _ScheduledTaskEntry: + name: Union[str, interfaces.renderers.BaseAbsentValue] + principal_id: Union[str, interfaces.renderers.BaseAbsentValue] + display_name: Union[str, interfaces.renderers.BaseAbsentValue] + enabled: Union[bool, interfaces.renderers.BaseAbsentValue] + creation_time: Union[datetime.datetime, interfaces.renderers.BaseAbsentValue] + last_run_time: Union[datetime.datetime, interfaces.renderers.BaseAbsentValue] + last_successful_run_time: Union[ + datetime.datetime, interfaces.renderers.BaseAbsentValue + ] + trigger_type: Union[str, interfaces.renderers.BaseAbsentValue] + trigger_description: Union[str, interfaces.renderers.BaseAbsentValue] + action_type: Union[str, interfaces.renderers.BaseAbsentValue] + action_description: Union[str, interfaces.renderers.BaseAbsentValue] + action_args: Union[str, interfaces.renderers.BaseAbsentValue] + action_context: Union[str, interfaces.renderers.BaseAbsentValue] + working_directory: Union[str, interfaces.renderers.BaseAbsentValue] + guid: str + + +@dataclasses.dataclass +class _JobSchedule: + start_boundary: Optional[datetime.datetime] + end_boundary: Optional[datetime.datetime] + repetition_interval_secs: Optional[int] + repetition_duration_secs: Optional[int] + execution_time_limit_secs: Optional[int] + mode: Optional[TimeMode] + data1: Optional[int] + data2: Optional[int] + data3: Optional[int] + stop_tasks_at_duration_end: Optional[int] + is_enabled: Optional[bool] + max_delay_seconds: Optional[int] + + def get_description(self) -> Optional[str]: + if self.mode == TimeMode.Once: + return "Run one time starting at {}".format( + self.start_boundary.isoformat() + if self.start_boundary is not None + else "" + ) + + elif self.mode == TimeMode.Daily: + if self.data1 is None: + return None + return "Run at {} and repeat every {} days".format( + ( + self.start_boundary.isoformat() + if self.start_boundary is not None + else "" + ), + self.data1, + ) + + elif self.mode == TimeMode.Weekly: + if self.data2 is None: + return None + + days = [k.name for k in Weekday if k.value & self.data2] + return "Run on {} every {} weeks starting at {}".format( + ", ".join(days), + self.data1, + ( + self.start_boundary.isoformat() + if self.start_boundary is not None + else "" + ), + ) + elif self.mode == TimeMode.DaysInMonths: + if self.data2 is None or self.data1 is None or self.data3 is None: + return None + months = [month.name for month in Months if month.value & self.data3] + days_bitmap = (self.data2 << 16) + self.data1 + days = [str(v + 1) for v in range(31) if (1 << v) & days_bitmap] + return "Run in months {} on days {} starting at {}".format( + ", ".join(months), + ", ".join(days), + ( + self.start_boundary.isoformat() + if self.start_boundary is not None + else "" + ), + ) + elif self.mode == TimeMode.DaysInWeeksInMonths: + if self.data1 is None or self.data2 is None or self.data3 is None: + return None + + months = [month.name for month in Months if month.value & self.data3] + weeks = [str(v + 1) for v in range(5) if (v << 1) & self.data2] + days = [day.name for day in Weekday if day.value & self.data1] + return "Run in months {} in weeks {} on days {} starting at {}".format( + ", ".join(months), + ", ".join(weeks), + ", ".join(days), + ( + self.start_boundary.isoformat() + if self.start_boundary is not None + else "" + ), + ) + else: + return None + + @classmethod + def decode(cls, reader: _ScheduledTasksReader) -> Optional["_JobSchedule"]: + start_boundary = reader.read_task_scheduler_time() + end_boundary = reader.read_task_scheduler_time() + + _ = reader.read_task_scheduler_time() + repetition_interval_secs = reader.read_u4() + repetition_duration_secs = reader.read_u4() + execution_time_limit_secs = reader.read_u4() + mode_index = reader.read_u4() + try: + if mode_index is not None: + mode = list(TimeMode)[mode_index] + else: + mode = None + except IndexError: + mode = TimeMode.Unknown + data1 = reader.read_u2() + data2 = reader.read_u2() + data3 = reader.read_u2() + + reader.seek(2, io.SEEK_CUR) # pad + stop_tasks_at_duration_end = reader.read_bool() + is_enabled = reader.read_bool() + reader.seek(6, io.SEEK_CUR) # pad (2) + unknown (4) + max_delay_seconds = reader.read_u4() + reader.seek(4, io.SEEK_CUR) # pad + + return cls( + start_boundary[1] if start_boundary is not None else None, + end_boundary[1] if end_boundary is not None else None, + repetition_interval_secs, + repetition_duration_secs, + execution_time_limit_secs, + mode, + data1, + data2, + data3, + stop_tasks_at_duration_end, + is_enabled, + max_delay_seconds, + ) + + +def decode_sid(data: bytes) -> Optional[str]: + """ + Decodes a windows SID from variable-length raw bytes + + Returns the string representation of the SID if decoding was successful, or None + if the data could not be parsed due to an insufficent number of bytes. + """ + try: + revision, subid_count, id_authority = struct.unpack( + ">BBQ", data[:2] + b"\x00\x00" + data[2:8] + ) + subauthorities = struct.unpack( + "<" + "I" * subid_count, data[8 : 8 + subid_count * 4] + ) + sid_string = "S-" + "-".join( + [str(item) for item in [revision, id_authority] + list(subauthorities)] + ) + except struct.error: + return None + + return sid_string + + +@dataclasses.dataclass +class UserInfo: + sid_type: Optional[SidType] + sid: Optional[str] + username: Optional[str] + + @classmethod + def _decode(cls, reader: _ScheduledTasksReader) -> Optional["UserInfo"]: + skip_user = reader.read_aligned_u1() != 0 + if not skip_user: + skip_sid = reader.read_aligned_u1() != 0 + else: + skip_sid = None + + sid_type = None + sid = None + if not skip_user and not skip_sid: + try: + sid_type = SidType(reader.read_aligned_u4()) + except ValueError: + sid_type = SidType.Unknown + + sid_raw = reader.read_buffer(aligned=True) + if sid_raw is None: + return None + sid = decode_sid(sid_raw) + + username = reader.read_bstring(aligned=True) if not skip_user else None + + return UserInfo(sid_type, sid, username) + + +@dataclasses.dataclass +class OptionalSettings: + IdleDurationSeconds: int + idleWaitTimeoutSeconds: int + ExecutionTimeLimitSeconds: int + DeleteExpiredTaskAfter: int + Priority: int + RestartOnFailureDelay: int + RestartOnFailureRetries: int + NetworkId: bytes + Privileges: Optional[List[str]] + Periodicity: Optional[TaskSchedulerTimePeriod] + Deadline: Optional[TaskSchedulerTimePeriod] + Exclusive: Optional[bool] + + @classmethod + def _decode(cls, reader: _ScheduledTasksReader) -> Optional["OptionalSettings"]: + LEN_WITH_PRIVILEGES = 0x38 + LEN_WITH_TIME_PERIODS = 0x58 + length = reader.read_aligned_u4() + if length == 0: + return None + + base_values = ( + reader.read_u4(), + reader.read_u4(), + reader.read_u4(), + reader.read_u4(), + reader.read_u4(), + reader.read_u4(), + reader.read_u4(), + binascii.hexlify(reader.read(16)), + ) + + if any(value is None for value in base_values): + return None + + reader.seek(4, io.SEEK_CUR) # padding + + privileges = None + periodicity = None + deadline = None + exclusive = None + if length == LEN_WITH_PRIVILEGES or length == LEN_WITH_TIME_PERIODS: + privileges_raw = reader.read_u8() + if privileges_raw is None: + return None + privileges = [ + priv.name for priv in Privileges if priv.value & privileges_raw + ] + if length == LEN_WITH_TIME_PERIODS: + periodicity = reader.read_tstimeperiod() + deadline = reader.read_tstimeperiod() + exclusive = reader.read_bool() + reader.seek(3, io.SEEK_CUR) # padding + + return OptionalSettings( + *base_values, privileges, periodicity, deadline, exclusive + ) + + +@dataclasses.dataclass +class JobBucket: + flags: List[str] + crc32: int + principal_id: Optional[str] + display_name: Optional[str] + user_info: Optional[UserInfo] + optional_settings: Optional[OptionalSettings] + + @classmethod + def _decode( + cls, reader: _ScheduledTasksReader, version: int + ) -> Optional["JobBucket"]: + flags_raw = reader.read_aligned_u4() + if flags_raw is None: + return None + flags = [y for x, y in JOB_BUCKET_FLAGS.items() if x & flags_raw] + crc32 = reader.read_aligned_u4() + if crc32 is None: + return None + + principal_id = None + display_name = None + if version >= 0x16: + principal_id = reader.read_bstring(aligned=True) + if version >= 0x17: + display_name = reader.read_bstring(aligned=True) + + user_info = UserInfo._decode(reader) + optional_settings = OptionalSettings._decode(reader) + + return JobBucket( + flags, crc32, principal_id, display_name, user_info, optional_settings + ) + + +class Privileges(enum.Enum): + SeCreateTokenPrivilege = 0x4 + SeAssignPrimaryTokenPrivilege = 0x8 + SeLockMemoryPrivilege = 0x10 + SeIncreaseQuotaPrivilege = 0x20 + SeMachineAccountPrivilege = 0x40 + SeTcbPrivilege = 0x80 + SeSecurityPrivilege = 0x100 + SeTakeOwnershipPrivilege = 0x200 + SeLoadDriverPrivilege = 0x400 + SeSystemProfilePrivilege = 0x800 + SeSystemtimePrivilege = 0x1000 + SeProfileSingleProcessPrivilege = 0x2000 + SeIncreaseBasePriorityPrivilege = 0x4000 + SeCreatePagefilePrivilege = 0x8000 + SeCreatePermanentPrivilege = 0x10000 + SeBackupPrivilege = 0x20000 + SeRestorePrivilege = 0x40000 + SeShutdownPrivilege = 0x80000 + SeDebugPrivilege = 0x100000 + SeAuditPrivilege = 0x200000 + SeSystemEnvironmentPrivilege = 0x400000 + SeChangeNotifyPrivilege = 0x800000 + SeRemoteShutdownPrivilege = 0x1000000 + SeUndockPrivilege = 0x2000000 + SeSyncAgentPrivilege = 0x4000000 + SeEnableDelegationPrivilege = 0x8000000 + SeManageVolumePrivilege = 0x10000000 + SeImpersonatePrivilege = 0x20000000 + SeCreateGlobalPrivilege = 0x40000000 + SeTrustedCredManAccessPrivilege = 0x80000000 + SeRelabelPrivilege = 0x100000000 + SeIncreaseWorkingSetPrivilege = 0x200000000 + SeTimeZonePrivilege = 0x400000000 + SeCreateSymbolicLinkPrivilege = 0x800000000 + SeDelegateSessionUserImpersonatePrivilege = 0x1000000000 + + +class SessionState(enum.Enum): + ConsoleConnect = 1 + ConsoleDisconnect = 2 + RemoteConnect = 3 + RemoteDisconnect = 4 + SessionLock = 5 + SessionUnlock = 6 + Unknown = "Unknown" + + +@dataclasses.dataclass +class TaskTrigger: + start_boundary: Optional[datetime.datetime] + end_boundary: Optional[datetime.datetime] + repetition_interval_seconds: Optional[int] + enabled: Optional[bool] + trigger_type: TriggerType + description: Optional[str] + + @classmethod + def _decode_generic_trigger( + cls, reader: _ScheduledTasksReader, version: int, trigger_type: TriggerType + ) -> Optional["TaskTrigger"]: + start_boundary = reader.read_task_scheduler_time() + end_boundary = reader.read_task_scheduler_time() + + _ = reader.read_u4() # delay seconds + _ = reader.read_u4() # timeout seconds + + repetition_interval_secs = reader.read_u4() + _ = reader.read_u4() # reptition duration seconds + _ = reader.read_u4() # repetition duration seconds 2 + + _ = reader.read_bool() # stop at duration end + reader.seek(3, io.SEEK_CUR) + trigger_enabled = bool(reader.read_aligned_u1()) + reader.seek(8, io.SEEK_CUR) # unknown field + + if version >= 0x16: + cur = reader.tell() + _ = reader.read_bstring() # trigger id + reader.seek((8 - (reader.tell() - cur)) % 8, io.SEEK_CUR) # pad to block + + return cls( + start_boundary[1] if start_boundary is not None else None, + end_boundary[1] if end_boundary is not None else None, + repetition_interval_secs, + trigger_enabled, + trigger_type, + f"{trigger_type.name} trigger", + ) + + @classmethod + def _decode_logon_trigger( + cls, reader: _ScheduledTasksReader, version: int + ) -> Optional["TaskTrigger"]: + base = cls._decode_generic_trigger(reader, version, TriggerType.Logon) + if base is None: + return None + + user = UserInfo._decode(reader) + if user is not None and user.username is not None: + base.description = f"{user.username}: {user.sid} ({user.sid_type})" + + return base + + @classmethod + def _decode_session_trigger( + cls, reader: _ScheduledTasksReader, version: int + ) -> Optional["TaskTrigger"]: + base = cls._decode_generic_trigger(reader, version, TriggerType.Session) + if base is None: + return None + session_type_raw = reader.read_u4() + reader.seek(4, io.SEEK_CUR) + + try: + session_type = SessionState(session_type_raw) + except ValueError: + session_type = SessionState.Unknown + + user_info = UserInfo._decode(reader) + if user_info is not None and user_info.username is not None: + base.description = f"{session_type.name} for user {user_info.username}" + else: + base.description = session_type.name + + return base + + @classmethod + def _decode_time_trigger( + cls, reader: _ScheduledTasksReader, version: int + ) -> Optional["TaskTrigger"]: + job_schedule = _JobSchedule.decode(reader) + if job_schedule is None: + return None + + if version >= 0x16: + cur = reader.tell() + _ = reader.read_bstring() # trigger id + reader.seek((8 - (reader.tell() - cur)) % 8, io.SEEK_CUR) # pad to block + + return cls( + job_schedule.start_boundary, + job_schedule.end_boundary, + job_schedule.repetition_interval_secs, + job_schedule.is_enabled, + TriggerType.Time, + job_schedule.get_description() or None, + ) + + @classmethod + def _decode_event_trigger( + cls, reader: _ScheduledTasksReader, version: int + ) -> Optional["TaskTrigger"]: + base = cls._decode_generic_trigger(reader, version, TriggerType.Event) + if base is None: + return base + + subscription = reader.read_aligned_bstring_expand_sz() + reader.seek(8, io.SEEK_CUR) # 2 4-byte unknown fields + reader.read_aligned_bstring_expand_sz() # another unknown field + len_value_queries = reader.read_aligned_u4() + + if len_value_queries is None: + return base + + queries = [ + ( + reader.read_aligned_bstring_expand_sz(), + reader.read_aligned_bstring_expand_sz(), + ) + for _ in range(len_value_queries) + ] + valid = [(k, v) for (k, v) in queries if k is not None and v is not None] + if base.description is None: + base.description = "Event Trigger" + base.description += f": Subscription: {subscription}, Queries: {str(valid)}" + return base + + @classmethod + def _decode_boot_trigger( + cls, reader: _ScheduledTasksReader, version: int + ) -> Optional["TaskTrigger"]: + return cls._decode_generic_trigger(reader, version, TriggerType.Boot) + + @classmethod + def _decode_wnf_trigger( + cls, reader: _ScheduledTasksReader, version: int + ) -> Optional["TaskTrigger"]: + base = cls._decode_generic_trigger( + reader, version, TriggerType.WindowsNotificationFacility + ) + if base is None: + return None + + state_name = binascii.hexlify(reader.read(8)).decode("ascii") + datalen = reader.read_aligned_u4() + _ = base64.b64encode(reader.read(datalen)) # state binary data + base.description = f"WNF state {state_name}" + return base + + @classmethod + def _decode_idle_trigger( + cls, reader: _ScheduledTasksReader, version: int + ) -> Optional["TaskTrigger"]: + return cls._decode_generic_trigger(reader, version, TriggerType.Logon) + + @classmethod + def _decode_registration_trigger( + cls, reader: _ScheduledTasksReader, version: int + ) -> Optional["TaskTrigger"]: + return cls._decode_generic_trigger(reader, version, TriggerType.Logon) + + +@dataclasses.dataclass +class TriggerSet: + job_bucket: JobBucket + triggers: List[TaskTrigger] + + @classmethod + def decode(cls, data) -> Optional["TriggerSet"]: + reader = _ScheduledTasksReader(data) + + version = reader.read_aligned_u1() + _ = reader.read_task_scheduler_time() # start boundary + _ = reader.read_task_scheduler_time() # end_boundary + + if version is None: + return None + + job_bucket = JobBucket._decode(reader, version) + if job_bucket is None: + return None + + triggers = [] + + while True: + magic = reader.read_aligned_u4() + if magic is None: + break + try: + trigger_type = TriggerType(magic) + except ValueError: + vollog.warning(f"Invalid trigger magic {hex(magic)}") + break + + if trigger_type == TriggerType.Logon: + trigger = TaskTrigger._decode_logon_trigger(reader, version) + elif trigger_type == TriggerType.Session: + trigger = TaskTrigger._decode_session_trigger(reader, version) + elif trigger_type == TriggerType.WindowsNotificationFacility: + trigger = TaskTrigger._decode_wnf_trigger(reader, version) + elif trigger_type == TriggerType.Boot: + trigger = TaskTrigger._decode_boot_trigger(reader, version) + elif trigger_type == TriggerType.Registration: + trigger = TaskTrigger._decode_registration_trigger(reader, version) + elif trigger_type == TriggerType.Event: + trigger = TaskTrigger._decode_event_trigger(reader, version) + elif trigger_type == TriggerType.Idle: + trigger = TaskTrigger._decode_idle_trigger(reader, version) + elif trigger_type == TriggerType.Time: + trigger = TaskTrigger._decode_time_trigger(reader, version) + else: + vollog.warning( + f"Invalid trigger magic {hex(magic)} encountered at offset {hex(reader.tell() - 8)}, stopping parsing" + ) + break + triggers.append(trigger) + + return cls(job_bucket, triggers) + + +@dataclasses.dataclass +class ActionSet: + actions: List[TaskAction] + context: Optional[str] + + @classmethod + def decode(cls, data: bytes) -> Optional["ActionSet"]: + reader = _ScheduledTasksReader(data) + actions = [] + + version = reader.read_u2() + if version is None: + return None + + if version in [2, 3]: + action_context = reader.read_bstring() + else: + action_context = None + + while True: + magic = reader.read_u2() + if magic is None: + break + + _ = ( + reader.read_bstring() + ) # action identifier, usually (but not always) empty + + if magic == ActionType.Email.value: + action = TaskAction._decode_email_action(reader) + elif magic == ActionType.Exe.value: + action = TaskAction._decode_exe_action(reader, version) + elif magic == ActionType.ComHandler.value: + action = TaskAction._decode_comhandler_action(reader) + elif magic == ActionType.MessageBox.value: + action = TaskAction.decode_messagebox_action(reader) + else: + break + actions.append(action) + + return cls(actions, action_context) + + +@dataclasses.dataclass +class DynamicInfo: + """ + Contains information about execution history for this task, + including timestamps and the last error code + """ + + creation_time: Optional[datetime.datetime] + last_run_time: Optional[datetime.datetime] + last_successful_run_time: Optional[datetime.datetime] + last_error_code: Optional[int] + + @classmethod + def decode(cls, data: bytes) -> Optional["DynamicInfo"]: + """ + Decodes a DynamicInfo structure from RegBin value data. + Raises a `ScheduledTaskDecodingError` if the magic bytes are invalid, but otherwise + attempts to decode as much as possible without returning an error. + """ + DYNAMICINFO_MAGIC = 3 + + reader = _ScheduledTasksReader(data) + magic = reader.read_u4() + if magic != DYNAMICINFO_MAGIC: + return None + + creation_time = reader.decode_filetime() + last_run_time = reader.decode_filetime() + + reader.seek(4, io.SEEK_CUR) # deprecated field 'TaskState' + + last_error_code = reader.read_u4() + last_success_time = reader.decode_filetime() + + vollog.debug((creation_time, last_run_time, last_success_time)) + + return cls( + last_run_time, + creation_time, + last_success_time, + last_error_code, + ) + + +class ScheduledTasks(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): + """Decodes scheduled task information from the Windows registry, including + information about triggers, actions, run times, and creation times. + """ + + _required_framework_version = (2, 11, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel33", "Intel64"], + ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), + ] + + def generate_timeline( + self, + ) -> Iterator[Tuple[str, timeliner.TimeLinerType, datetime.datetime]]: + for _, task in self._generator(): + if isinstance(task.last_run_time, datetime.datetime): + yield f"ScheduledTasks: task action {task.action_description} with trigger {task.trigger_description} ran", timeliner.TimeLinerType.ACCESSED, task.last_run_time + if isinstance(task.last_successful_run_time, datetime.datetime): + yield f"ScheduledTasks: task action {task.action_description} with trigger {task.trigger_description} ran successfully", timeliner.TimeLinerType.ACCESSED, task.last_successful_run_time + if isinstance(task.creation_time, datetime.datetime): + yield f"ScheduledTasks: Creation Time for task {task.guid} with trigger {task.trigger_description or ''}", timeliner.TimeLinerType.CREATED, task.creation_time + + @classmethod + def get_software_hive( + cls, + context: interfaces.context.ContextInterface, + config_path: str, + kernel: interfaces.context.ModuleInterface, + ) -> Optional[registry.RegistryHive]: + """Retrieves the `Amcache.hve` registry hive from the kernel module, if it can be located.""" + return next( + hivelist.HiveList.list_hives( + context=context, + base_config_path=interfaces.configuration.path_join( + config_path, "hivelist" + ), + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_string="SOFTWARE", + ), + None, + ) + + @classmethod + def parse_actions_value( + cls, actions_value: reg_extensions.CM_KEY_VALUE + ) -> Optional[ActionSet]: + """Parses File entries from the Windows 8 `Root\\File` key. + + :param programs_key: The `Root\\File` registry key. + + :return: An iterator of tuples, where the first member is the program ID string for + correlating `Root\\Program` entries, and the second member is the `AmcacheEntry`. + """ + try: + data = actions_value.decode_data() + except exceptions.InvalidAddressException: + data = None + + if not isinstance(data, bytes): + return None + + return ActionSet.decode(data) + + @classmethod + def parse_triggers_value( + cls, triggers_value: reg_extensions.CM_KEY_VALUE + ) -> Optional[TriggerSet]: + try: + data = triggers_value.decode_data() + except exceptions.InvalidAddressException: + data = None + + if not isinstance(data, bytes): + return None + + return TriggerSet.decode(data) + + @classmethod + def parse_dynamic_info_value( + cls, dyn_info_value: reg_extensions.CM_KEY_VALUE + ) -> Optional[DynamicInfo]: + + try: + data = dyn_info_value.decode_data() + except exceptions.InvalidAddressException: + data = None + + if not isinstance(data, bytes): + return None + + return DynamicInfo.decode(data) + + @classmethod + def _get_task_keys( + cls, software_hive: reg_extensions.RegistryHive + ) -> Tuple[ + Optional[reg_extensions.CM_KEY_NODE], Optional[reg_extensions.CM_KEY_NODE] + ]: + try: + task_key = software_hive.get_key( + "Microsoft\\Windows NT\\CurrentVersion\\Schedule\\TaskCache\\Tasks" + ) + except (KeyError, registry.RegistryFormatException): + task_key = None + + try: + task_tree = software_hive.get_key( + "Microsoft\\Windows NT\\CurrentVersion\\Schedule\\TaskCache\\Tree" + ) + except (KeyError, registry.RegistryFormatException): + task_tree = None + + return (task_key, task_tree) # type: ignore + + @classmethod + def _parse_task_key( + cls, key: reg_extensions.CM_KEY_NODE, guid_mapping: Dict[str, str] + ) -> Iterator[_ScheduledTaskEntry]: + values = {} + for value in key.get_values(): + try: + name = str(value.get_name()) + except exceptions.InvalidAddressException: + continue + + if name in ["Actions", "Triggers", "DynamicInfo"]: + values[name] = value + + task_name = guid_mapping.get(str(key.get_name()), renderers.NotAvailableValue()) + + try: + action_set = cls.parse_actions_value(values["Actions"]) + except KeyError: + vollog.debug("Failed to get Actions value") + action_set = None + + try: + triggers_value = values["Triggers"] + trigger_set = cls.parse_triggers_value(triggers_value) + except KeyError: + vollog.debug("Failed to get Triggers value") + trigger_set = None + + if trigger_set is not None: + vollog.debug("Parsed triggers successfully") + + principal_id = ( + trigger_set.job_bucket.principal_id or renderers.NotAvailableValue() + ) + display_name = ( + trigger_set.job_bucket.display_name or renderers.NotAvailableValue() + ) + else: + vollog.debug("Failed to parse triggers") + + principal_id = renderers.NotAvailableValue() + display_name = renderers.NotAvailableValue() + + try: + dynamic_info = cls.parse_dynamic_info_value(values["DynamicInfo"]) + except KeyError: + vollog.debug("DynamicInfo value not found") + dynamic_info = None + + vollog.debug(dynamic_info) + + creation_time = dynamic_info.creation_time if dynamic_info is not None else None + last_run_time = dynamic_info.last_run_time if dynamic_info is not None else None + last_successful_run_time = ( + dynamic_info.last_successful_run_time if dynamic_info is not None else None + ) + + all_triggers = ( + trigger_set.triggers or [None] if trigger_set is not None else [None] + ) + + all_actions = action_set.actions or [None] if action_set is not None else [None] + + for action, trigger in itertools.product(all_actions, all_triggers): + + if action is not None: + if action.action_type in ( + ActionType.Exe, + ActionType.ComHandler, + ): + if action.action_args is None: + args = renderers.NotAvailableValue() + else: + args = action.action_args + else: + args = renderers.NotApplicableValue() + + if action.action_type == ActionType.Exe: + working_directory = ( + action.working_directory or renderers.NotAvailableValue() + ) + else: + working_directory = renderers.NotApplicableValue() + + else: + args = renderers.NotAvailableValue() + working_directory = renderers.NotAvailableValue() + + if trigger is not None and trigger.enabled is not None: + enabled = trigger.enabled + else: + enabled = renderers.NotAvailableValue() + + yield _ScheduledTaskEntry( + task_name, + principal_id, + display_name, + enabled, + creation_time or renderers.NotAvailableValue(), + last_run_time or renderers.NotAvailableValue(), + last_successful_run_time or renderers.NotAvailableValue(), + ( + trigger.trigger_type.name + if trigger is not None + else renderers.NotAvailableValue() + ), + ( + trigger.description or renderers.NotAvailableValue() + if trigger is not None + else renderers.NotAvailableValue() + ), + ( + action.action_type.name + if action is not None + else renderers.NotAvailableValue() + ), + ( + action.action + if action is not None + else renderers.NotAvailableValue() + ), + args, + ( + action_set.context + if action_set is not None + else renderers.NotAvailableValue() + ), + working_directory, + str(key.get_name()), + ) + + def _generator(self) -> Iterator[Tuple[int, _ScheduledTaskEntry]]: + kernel = self.context.modules[self.config["kernel"]] + + # Building the dictionary ahead of time is much better for performance + # vs looking up each service's DLL individually. + software_hive = self.get_software_hive(self.context, self.config_path, kernel) + if software_hive is None: + vollog.warning("Failed to get SOFTWARE hive") + return + + task_key_root, task_tree = self._get_task_keys(software_hive) + if task_key_root is None: + vollog.warning("Failed to get 'Tasks' key") + return + + if task_tree is not None: + task_name_map = _build_guid_name_map(task_tree) + else: + vollog.info("'Tree' key not found, can't map GUIDs to task names") + task_name_map = {} + + for key in task_key_root.get_subkeys(): + for task in self._parse_task_key(key, task_name_map): + yield 0, task + + def run(self): + return renderers.TreeGrid( + [ + ("Task Name", str), + ("Principal ID", str), + ("Display Name", str), + ("Enabled", bool), + ("Creation Time", datetime.datetime), + ("Last Run Time", datetime.datetime), + ("Last Successful Run Time", datetime.datetime), + ("Trigger Type", str), + ("Trigger Description", str), + ("Action Type", str), + ("Action", str), + ("Action Arguments", str), + ("Action Context", str), + ("Working Directory", str), + ("Key Name", str), + ], + ( + (indent, dataclasses.astuple(entry)) + for indent, entry in self._generator() + ), + ) + + +class TestActionsDecoding(unittest.TestCase): + def test_decode_exe_action(self): + # fmt: off + buf = struct.pack( + "512B", + *[ + 0x03, 0x00, 0x16, 0x00, 0x00, 0x00, 0x4c, 0x00, + 0x6f, 0x00, 0x63, 0x00, 0x61, 0x00, 0x6c, 0x00, + 0x53, 0x00, 0x79, 0x00, 0x73, 0x00, 0x74, 0x00, + 0x65, 0x00, 0x6d, 0x00, 0x66, 0x66, 0x00, 0x00, + 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, 0x25, 0x00, + 0x77, 0x00, 0x69, 0x00, 0x6e, 0x00, 0x64, 0x00, + 0x69, 0x00, 0x72, 0x00, 0x25, 0x00, 0x5c, 0x00, + 0x73, 0x00, 0x79, 0x00, 0x73, 0x00, 0x74, 0x00, + 0x65, 0x00, 0x6d, 0x00, 0x33, 0x00, 0x32, 0x00, + 0x5c, 0x00, 0x57, 0x00, 0x69, 0x00, 0x6e, 0x00, + 0x64, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x73, 0x00, + 0x50, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x53, 0x00, 0x68, 0x00, 0x65, 0x00, + 0x6c, 0x00, 0x6c, 0x00, 0x5c, 0x00, 0x76, 0x00, + 0x31, 0x00, 0x2e, 0x00, 0x30, 0x00, 0x5c, 0x00, + 0x70, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x73, 0x00, 0x68, 0x00, 0x65, 0x00, + 0x6c, 0x00, 0x6c, 0x00, 0x2e, 0x00, 0x65, 0x00, + 0x78, 0x00, 0x65, 0x00, 0x62, 0x01, 0x00, 0x00, + 0x2d, 0x00, 0x45, 0x00, 0x78, 0x00, 0x65, 0x00, + 0x63, 0x00, 0x75, 0x00, 0x74, 0x00, 0x69, 0x00, + 0x6f, 0x00, 0x6e, 0x00, 0x50, 0x00, 0x6f, 0x00, + 0x6c, 0x00, 0x69, 0x00, 0x63, 0x00, 0x79, 0x00, + 0x20, 0x00, 0x55, 0x00, 0x6e, 0x00, 0x72, 0x00, + 0x65, 0x00, 0x73, 0x00, 0x74, 0x00, 0x72, 0x00, + 0x69, 0x00, 0x63, 0x00, 0x74, 0x00, 0x65, 0x00, + 0x64, 0x00, 0x20, 0x00, 0x2d, 0x00, 0x4e, 0x00, + 0x6f, 0x00, 0x6e, 0x00, 0x49, 0x00, 0x6e, 0x00, + 0x74, 0x00, 0x65, 0x00, 0x72, 0x00, 0x61, 0x00, + 0x63, 0x00, 0x74, 0x00, 0x69, 0x00, 0x76, 0x00, + 0x65, 0x00, 0x20, 0x00, 0x2d, 0x00, 0x4e, 0x00, + 0x6f, 0x00, 0x50, 0x00, 0x72, 0x00, 0x6f, 0x00, + 0x66, 0x00, 0x69, 0x00, 0x6c, 0x00, 0x65, 0x00, + 0x20, 0x00, 0x2d, 0x00, 0x57, 0x00, 0x69, 0x00, + 0x6e, 0x00, 0x64, 0x00, 0x6f, 0x00, 0x77, 0x00, + 0x53, 0x00, 0x74, 0x00, 0x79, 0x00, 0x6c, 0x00, + 0x65, 0x00, 0x20, 0x00, 0x48, 0x00, 0x69, 0x00, + 0x64, 0x00, 0x64, 0x00, 0x65, 0x00, 0x6e, 0x00, + 0x20, 0x00, 0x22, 0x00, 0x26, 0x00, 0x20, 0x00, + 0x25, 0x00, 0x77, 0x00, 0x69, 0x00, 0x6e, 0x00, + 0x64, 0x00, 0x69, 0x00, 0x72, 0x00, 0x25, 0x00, + 0x5c, 0x00, 0x73, 0x00, 0x79, 0x00, 0x73, 0x00, + 0x74, 0x00, 0x65, 0x00, 0x6d, 0x00, 0x33, 0x00, + 0x32, 0x00, 0x5c, 0x00, 0x57, 0x00, 0x69, 0x00, + 0x6e, 0x00, 0x64, 0x00, 0x6f, 0x00, 0x77, 0x00, + 0x73, 0x00, 0x50, 0x00, 0x6f, 0x00, 0x77, 0x00, + 0x65, 0x00, 0x72, 0x00, 0x53, 0x00, 0x68, 0x00, + 0x65, 0x00, 0x6c, 0x00, 0x6c, 0x00, 0x5c, 0x00, + 0x76, 0x00, 0x31, 0x00, 0x2e, 0x00, 0x30, 0x00, + 0x5c, 0x00, 0x4d, 0x00, 0x6f, 0x00, 0x64, 0x00, + 0x75, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x73, 0x00, + 0x5c, 0x00, 0x53, 0x00, 0x6d, 0x00, 0x62, 0x00, + 0x53, 0x00, 0x68, 0x00, 0x61, 0x00, 0x72, 0x00, + 0x65, 0x00, 0x5c, 0x00, 0x44, 0x00, 0x69, 0x00, + 0x73, 0x00, 0x61, 0x00, 0x62, 0x00, 0x6c, 0x00, + 0x65, 0x00, 0x55, 0x00, 0x6e, 0x00, 0x75, 0x00, + 0x73, 0x00, 0x65, 0x00, 0x64, 0x00, 0x53, 0x00, + 0x6d, 0x00, 0x62, 0x00, 0x31, 0x00, 0x2e, 0x00, + 0x70, 0x00, 0x73, 0x00, 0x31, 0x00, 0x20, 0x00, + 0x2d, 0x00, 0x53, 0x00, 0x63, 0x00, 0x65, 0x00, + 0x6e, 0x00, 0x61, 0x00, 0x72, 0x00, 0x69, 0x00, + 0x6f, 0x00, 0x20, 0x00, 0x43, 0x00, 0x6c, 0x00, + 0x69, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74, 0x00, + 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ] + ) + + try: + actions = ActionSet.decode(buf) # type: ignore + self.assertEqual(len(actions), 1) + self.assertEqual(actions[0].action_type, ActionType.Exe) + except Exception: + self.fail( + "ActionDecoder.decode should not raise exception:\n%s" + % traceback.format_exc() + ) + + +class TestTriggersDecoding(unittest.TestCase): + def test_decode_all_triggers(self): + """ + Tests decoding a set of all triggers that can be constructed via the + Task Scheduler GUI interface. Ensures that the correct number of bytes + is being consumed for each trigger structure. + """ + buf = struct.pack( + "1808B", + # fmt: off + *[ + 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x38, 0x21, 0x41, 0x42, 0x48, 0x48, 0x48, 0x48, + 0xa0, 0x12, 0xa0, 0xa4, 0x48, 0x48, 0x48, 0x48, + 0x0e, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x41, 0x00, 0x75, 0x00, 0x74, 0x00, 0x68, 0x00, + 0x6f, 0x00, 0x72, 0x00, 0x00, 0x00, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x1c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x15, 0x00, 0x00, 0x00, 0x69, 0xce, 0x28, 0x2a, + 0xce, 0xd8, 0x1f, 0x77, 0x37, 0x9c, 0xe2, 0x44, + 0xf4, 0x01, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x40, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x44, 0x00, 0x45, 0x00, 0x53, 0x00, 0x4b, 0x00, + 0x54, 0x00, 0x4f, 0x00, 0x50, 0x00, 0x2d, 0x00, + 0x45, 0x00, 0x33, 0x00, 0x38, 0x00, 0x38, 0x00, + 0x44, 0x00, 0x38, 0x00, 0x50, 0x00, 0x5c, 0x00, + 0x41, 0x00, 0x64, 0x00, 0x6d, 0x00, 0x69, 0x00, + 0x6e, 0x00, 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, + 0x72, 0x00, 0x61, 0x00, 0x74, 0x00, 0x6f, 0x00, + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x2c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x80, 0xf4, 0x03, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0xdd, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x07, 0x0a, 0x00, 0x00, 0x00, 0x09, 0x00, + 0x80, 0x48, 0x11, 0xf8, 0x36, 0x1a, 0xdb, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x2e, 0xe2, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x31, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0xaa, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0xee, 0xee, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0xcc, 0xcc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x65, 0x00, 0x78, 0x00, 0x65, 0x00, + 0x22, 0x00, 0x20, 0x00, 0x53, 0x00, 0x74, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x3c, 0x00, 0x51, 0x00, 0x75, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x79, 0x00, 0x4c, 0x00, 0x69, 0x00, + 0x73, 0x00, 0x74, 0x00, 0x3e, 0x00, 0x3c, 0x00, + 0x51, 0x00, 0x75, 0x00, 0x65, 0x00, 0x72, 0x00, + 0x79, 0x00, 0x20, 0x00, 0x49, 0x00, 0x64, 0x00, + 0x3d, 0x00, 0x22, 0x00, 0x30, 0x00, 0x22, 0x00, + 0x20, 0x00, 0x50, 0x00, 0x61, 0x00, 0x74, 0x00, + 0x68, 0x00, 0x3d, 0x00, 0x22, 0x00, 0x49, 0x00, + 0x6e, 0x00, 0x74, 0x00, 0x65, 0x00, 0x72, 0x00, + 0x6e, 0x00, 0x65, 0x00, 0x74, 0x00, 0x20, 0x00, + 0x45, 0x00, 0x78, 0x00, 0x70, 0x00, 0x6c, 0x00, + 0x6f, 0x00, 0x72, 0x00, 0x65, 0x00, 0x72, 0x00, + 0x22, 0x00, 0x3e, 0x00, 0x3c, 0x00, 0x53, 0x00, + 0x65, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x63, 0x00, + 0x74, 0x00, 0x20, 0x00, 0x50, 0x00, 0x61, 0x00, + 0x74, 0x00, 0x68, 0x00, 0x3d, 0x00, 0x22, 0x00, + 0x49, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x6e, 0x00, 0x65, 0x00, 0x74, 0x00, + 0x20, 0x00, 0x45, 0x00, 0x78, 0x00, 0x70, 0x00, + 0x6c, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x22, 0x00, 0x3e, 0x00, 0x2a, 0x00, + 0x5b, 0x00, 0x53, 0x00, 0x79, 0x00, 0x73, 0x00, + 0x74, 0x00, 0x65, 0x00, 0x6d, 0x00, 0x5b, 0x00, + 0x45, 0x00, 0x76, 0x00, 0x65, 0x00, 0x6e, 0x00, + 0x74, 0x00, 0x49, 0x00, 0x44, 0x00, 0x3d, 0x00, + 0x32, 0x00, 0x5d, 0x00, 0x5d, 0x00, 0x3c, 0x00, + 0x2f, 0x00, 0x53, 0x00, 0x65, 0x00, 0x6c, 0x00, + 0x65, 0x00, 0x63, 0x00, 0x74, 0x00, 0x3e, 0x00, + 0x3c, 0x00, 0x2f, 0x00, 0x51, 0x00, 0x75, 0x00, + 0x65, 0x00, 0x72, 0x00, 0x79, 0x00, 0x3e, 0x00, + 0x3c, 0x00, 0x2f, 0x00, 0x51, 0x00, 0x75, 0x00, + 0x65, 0x00, 0x72, 0x00, 0x79, 0x00, 0x4c, 0x00, + 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, 0x3e, 0x00, + 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x88, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x1c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x15, 0x00, 0x00, 0x00, 0x69, 0xce, 0x28, 0x2a, + 0xce, 0xd8, 0x1f, 0x77, 0x37, 0x9c, 0xe2, 0x44, + 0xf4, 0x01, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x40, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x44, 0x00, 0x45, 0x00, 0x53, 0x00, 0x4b, 0x00, + 0x54, 0x00, 0x4f, 0x00, 0x50, 0x00, 0x2d, 0x00, + 0x45, 0x00, 0x33, 0x00, 0x38, 0x00, 0x38, 0x00, + 0x44, 0x00, 0x38, 0x00, 0x50, 0x00, 0x5c, 0x00, + 0x41, 0x00, 0x64, 0x00, 0x6d, 0x00, 0x69, 0x00, + 0x6e, 0x00, 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, + 0x72, 0x00, 0x61, 0x00, 0x74, 0x00, 0x6f, 0x00, + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x1c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x15, 0x00, 0x00, 0x00, 0x69, 0xce, 0x28, 0x2a, + 0xce, 0xd8, 0x1f, 0x77, 0x37, 0x9c, 0xe2, 0x44, + 0xf4, 0x01, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x40, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x44, 0x00, 0x45, 0x00, 0x53, 0x00, 0x4b, 0x00, + 0x54, 0x00, 0x4f, 0x00, 0x50, 0x00, 0x2d, 0x00, + 0x45, 0x00, 0x33, 0x00, 0x38, 0x00, 0x38, 0x00, + 0x44, 0x00, 0x38, 0x00, 0x50, 0x00, 0x5c, 0x00, + 0x41, 0x00, 0x64, 0x00, 0x6d, 0x00, 0x69, 0x00, + 0x6e, 0x00, 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, + 0x72, 0x00, 0x61, 0x00, 0x74, 0x00, 0x6f, 0x00, + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ] + # fmt: on + ) + triggers = TriggerSet.decode(buf) + self.assertIsNotNone(triggers) + + def test_decode_triggers(self): + # fmt: off + buf = struct.pack( + "320B", + *[ + 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x08, 0xA1, 0x40, 0x42, 0x48, 0x48, 0x48, 0x48, + 0x7A, 0x7F, 0x59, 0xDC, 0x48, 0x48, 0x48, 0x48, + 0x22, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x49, 0x00, 0x6E, 0x00, 0x74, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x61, 0x00, 0x63, 0x00, 0x74, 0x00, + 0x69, 0x00, 0x76, 0x00, 0x65, 0x00, 0x55, 0x00, + 0x73, 0x00, 0x65, 0x00, 0x72, 0x00, 0x73, 0x00, + 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x05, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x0C, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x04, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x2C, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0x80, 0x51, 0x01, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0xAA, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2C, 0x01, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xC1, 0xD9, 0x04, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + ] + ) + # fmt: on + triggers = TriggerSet.decode(buf) + self.assertIsNotNone(triggers) + if not triggers: + return + self.assertGreater(len(triggers.triggers), 0) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index c8ddc19fd..e48684b31 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -28,6 +28,27 @@ def wintime_to_datetime( return renderers.UnparsableValue() +def windows_bytes_to_guid(buf: bytes) -> str: + """ + Converts 16 raw bytes to a windows GUID. + + Raises ValueError if the provided buffer is not exactly 16 bytes. + """ + if len(buf) != 16: + raise ValueError("Expected 16 bytes for GUID") + + head_components = [format(v, "x") for v in struct.unpack("HQ", + buf[8:10] + b"\x00\x00" + buf[10:16], + ) + ] + combined = head_components + tail_component + return "{" + "-".join(combined) + "}" + + def unixtime_to_datetime( unixtime: int, ) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: From fb4dd924ceaffa04fd39dd3a86009f110748cd85 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 11 Oct 2024 10:15:43 -0500 Subject: [PATCH 471/979] Linux: Netfilter - fix traceback on missing module name When the module name can't be found, `None` ends up in this field, causing a traceback when rendering. This fixes it by replacing `None` with `renderers.NotAvailableValue()`. This occurred in the context of a netfilter hook from a hidden module. --- volatility3/framework/plugins/linux/netfilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index d392a8371..f576c073c 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -714,7 +714,7 @@ class Netfilter(interfaces.plugins.PluginInterface): hook_name, priority, format_hints.Hex(hook_func), - module_name, + module_name or renderers.NotAvailableValue(), str(hooked), ) From e3521147d6c922a7e25d4481edcd6d4016888f74 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 13 Oct 2024 09:26:37 +1100 Subject: [PATCH 472/979] Linux - Add support for task threads in kernels >= 6.7 --- .../symbols/linux/extensions/__init__.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index aa18b8262..0dab0db43 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -359,20 +359,29 @@ class task_struct(generic.GenericIntelProcess): """ return not self.is_kernel_thread and self.tgid != self.pid + def _get_tasks_iterable(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns the respective iterable to obtain the threads in this process""" + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + task_struct_symname = f"{vmlinux.symbol_table_name}{constants.BANG}task_struct" + if vmlinux.get_type("task_struct").has_member("signal") and vmlinux.get_type( + "signal_struct" + ).has_member("thread_head"): + # kernels >= 6.7 - via signals + return self.signal.thread_head.to_list(task_struct_symname, "thread_node") + elif vmlinux.get_type("task_struct").has_member("thread_group"): + # kernels < 6.7 - via thread_group + return self.thread_group.to_list(task_struct_symname, "thread_group") + + raise AttributeError("Unable to find the root dentry") + def get_threads(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns a list of the task_struct based on the list_head - thread_node structure.""" - - task_symbol_table_name = self.get_symbol_table_name() - - # iterating through the thread_list from thread_group - # this allows iterating through pointers to grab the - # threads and using the thread_group offset to get the - # corresponding task_struct - for task in self.thread_group.to_list( - f"{task_symbol_table_name}{constants.BANG}task_struct", "thread_group" - ): - yield task + """Returns each thread in this process""" + tasks_iterable = self._get_tasks_iterable() + threads_seen = set([self.vol.offset]) + for task in tasks_iterable: + if task.vol.offset not in threads_seen: + threads_seen.add(task.vol.offset) + yield task class fs_struct(objects.StructType): From 15a51cf1ec1e139193097976c115c820861bc034 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 14 Oct 2024 16:13:56 +1100 Subject: [PATCH 473/979] Linux - Add hlist_head object extension. Fix #1313 --- .../framework/symbols/linux/__init__.py | 1 + .../symbols/linux/extensions/__init__.py | 39 ++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 219f120ee..1b7a331f8 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -22,6 +22,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): # Set-up Linux specific types self.set_type_class("file", extensions.struct_file) self.set_type_class("list_head", extensions.list_head) + self.set_type_class("hlist_head", extensions.hlist_head) self.set_type_class("mm_struct", extensions.mm_struct) self.set_type_class("super_block", extensions.super_block) self.set_type_class("task_struct", extensions.task_struct) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index aa18b8262..4b1df2a0b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -844,7 +844,7 @@ class dentry(objects.StructType): if self.has_member("d_sib") and self.has_member("d_children"): # kernels >= 6.8 walk_member = "d_sib" - list_head_member = self.d_children.first + list_head_member = self.d_children elif self.has_member("d_child") and self.has_member("d_subdirs"): # 2.5.0 <= kernels < 6.8 walk_member = "d_child" @@ -961,6 +961,43 @@ class list_head(objects.StructType, collections.abc.Iterable): return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) +class hlist_head(objects.StructType, collections.abc.Iterable): + def to_list( + self, + symbol_type: str, + member: str, + ) -> Iterator[interfaces.objects.ObjectInterface]: + """Returns an iterator of the entries in the list. + + This is a doubly linked list; however, it is not circular, so the 'forward' field + doesn't make sense. Also, the sentinel concept doesn't make sense here either; + unlike list_head, the head and nodes each have their own distinct types. A list_head + cannot be a node by itself. + - The 'pprev' of the first 'hlist_node' points to the 'hlist_head', not to the last node. + - The last element 'next' member is NULL + + Args: + symbol_type: Type of the list elements + member: Name of the list_head member in the list elements + + Yields: + Objects of the type specified via the "symbol_type" argument. + + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + + current = self.first + while current and current.is_readable(): + yield linux.LinuxUtilities.container_of( + current, symbol_type, member, vmlinux + ) + + current = current.next + + def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: + return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) + + class files_struct(objects.StructType): def get_fds(self) -> interfaces.objects.ObjectInterface: if self.has_member("fdt"): From f00c4c3c392a26bea29ba87d961c3a8f51092829 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 14 Oct 2024 16:35:43 +1100 Subject: [PATCH 474/979] Allows to create objects when using the same symbol table --- volatility3/framework/contexts/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 4c169728c..6961d9328 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -245,7 +245,7 @@ class Module(interfaces.context.ModuleInterface): """ if constants.BANG not in object_type: object_type = self.symbol_table_name + constants.BANG + object_type - else: + elif not object_type.startswith(self.symbol_table_name + constants.BANG): raise ValueError( "Cannot reference another module when constructing an object" ) From e739d96a33217e013f0b126ab2c11007cf059cee Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 14 Oct 2024 14:03:38 +0200 Subject: [PATCH 475/979] dwarf2json rust type confusion sanity check --- volatility3/schemas/__init__.py | 54 +++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index be120f2af..3ca00e5dc 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -6,8 +6,8 @@ import hashlib import json import logging import os -from typing import Any, Dict, Optional, Set - +import re +from typing import Any, Dict, Optional, Set, Tuple from volatility3.framework import constants vollog = logging.getLogger(__name__) @@ -77,6 +77,17 @@ def valid( input: Dict[str, Any], schema: Dict[str, Any], use_cache: bool = True ) -> bool: """Validates a json schema.""" + producer = input.get("metadata", {}).get("producer", {}) + if producer and producer.get("name") == "dwarf2json": + dwarf2json_version = parse_producer_version(producer.get("version", "")) + # No warnings if version couldn't be parsed, as it's not our role here + # to validate the schema. + if dwarf2json_version: + if dwarf2json_check_rust_type_confusion(input, dwarf2json_version): + vollog.warning( + "This ISF was generated by dwarf2json < 0.9.0, which is known to produce inaccurate results (see dwarf2json GitHub issue #63)." + ) + input_hash = create_json_hash(input, schema) if input_hash in cached_validations and use_cache: return True @@ -98,3 +109,42 @@ def valid( record_cached_validations(cached_validations) return True + + +def parse_producer_version(version_string: str) -> Optional[Tuple[int]]: + """Parses a producer version and returns a tuple of identifiers. + + Args: + version_string: string containing dot-separated integers, + expected to follow the Volatility3 versioning schema + + Returns: + A tuple containing each version identifier + """ + identifiers = re.search("^(\\d+)[.](\\d+)[.](\\d+)$", version_string) + if not identifiers: + return None + + return tuple(int(d) for d in identifiers.groups()) + + +# dwarf2json sanity checks # +def dwarf2json_check_rust_type_confusion( + input: Dict[str, Any], dwarf2json_version: Tuple[int] +) -> bool: + """dwarf2json sanity check for Rust and C types confusion: + - dwarf2json #63 + - volatility3 #1305 + + Args: + dwarf2json_version: a tuple containing each version identifier + + Returns: + True if the issue was detected + """ + + return "rust_helper_BUG" in input.get("symbols", {}) and dwarf2json_version < ( + 0, + 9, + 0, + ) From 2920694643d6951a980fe32df0e3afe295e5d418 Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 17:58:49 -0500 Subject: [PATCH 476/979] placeholder --- .../framework/plugins/windows/mftscan.py | 367 +++++++++--------- 1 file changed, 187 insertions(+), 180 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 9e6585345..0687c7796 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -20,6 +20,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._record_map = {} + @classmethod def get_requirements(cls): return [ @@ -33,8 +37,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] - def _generator(self): - layer = self.context.layers[self.config["primary"]] + def enumerate_mft_records(self, attr_callback): + phys_layer = self.context.layers[self.config["primary"]].config["memory_layer"] + + layer = self.context.layers[phys_layer] # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( @@ -47,87 +53,36 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): config_path=self.config_path, sub_path="windows", filename="mft", - class_types={"FILE_NAME_ENTRY": mft.MFTFileName, "MFT_ENTRY": mft.MFTEntry}, + class_types={"FILE_NAME_ENTRY": mft.MFTFileName, "MFT_ENTRY": mft.MFTEntry, "ATTRIBUTE": mft.MFTAttribute}, ) # get each of the individual Field Sets - mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" - si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" - fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + self.mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + self.attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" + self.si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" + self.fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" # Scan the layer for Raw MFT records and parse the fields - for offset, _rule_name, _name, _value in layer.scan( + for offset, _, _, _ in layer.scan( context=self.context, scanner=yarascan.YaraScanner(rules=rules) ): - with contextlib.suppress(exceptions.PagedInvalidAddressException): + with contextlib.suppress(exceptions.InvalidAddressException): mft_record = self.context.object( - mft_object, offset=offset, layer_name=layer.name + self.mft_object, offset=offset, layer_name=layer.name ) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset attr = self.context.object( - attribute_object, + self.attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr.Attr_Header.AttrType.is_valid_choice: - vollog.debug(f"Attr Type: {attr.Attr_Header.AttrType.lookup()}") - - # MFT Flags determine the file type or dir - # If we don't have a valid enum, coerce to hex so we can keep the record - try: - mft_flag = mft_record.Flags.lookup() - except ValueError: - mft_flag = hex(mft_record.Flags) - - # Standard Information Attribute - if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": - attr_data = attr.Attr_Data.cast(si_object) - yield 0, ( - format_hints.Hex(attr_data.vol.offset), - mft_record.get_signature(), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - renderers.NotApplicableValue(), - attr.Attr_Header.AttrType.lookup(), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - renderers.NotApplicableValue(), - ) - - # File Name Attribute - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - attr_data = attr.Attr_Data.cast(fn_object) - file_name = attr_data.get_full_name() - - # If we don't have a valid enum, coerce to hex so we can keep the record - try: - permissions = attr_data.Flags.lookup() - except ValueError: - permissions = hex(attr_data.Flags) - - yield 1, ( - format_hints.Hex(attr_data.vol.offset), - mft_record.get_signature(), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - permissions, - attr.Attr_Header.AttrType.lookup(), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - file_name, - ) + for record in attr_callback(mft_record, attr): + yield record # If there's no advancement the loop will never end, so break it now if attr.Attr_Header.Length == 0: @@ -135,12 +90,69 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Update the base offset to point to the next attribute attr_base_offset += attr.Attr_Header.Length + # Get the next attribute attr = self.context.object( - attribute_object, + self.attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) + def parse_mft_records(self, mft_record, attr): + # MFT Flags determine the file type or dir + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + mft_flag = mft_record.Flags.lookup() + except ValueError: + mft_flag = hex(mft_record.Flags) + + # Standard Information Attribute + if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": + attr_data = attr.Attr_Data.cast(self.si_object) + yield 0, ( + format_hints.Hex(attr_data.vol.offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + mft_flag, + renderers.NotApplicableValue(), + attr.Attr_Header.AttrType.lookup(), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + renderers.NotApplicableValue(), + ) + + # File Name Attribute + elif attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + attr_data = attr.Attr_Data.cast(self.fn_object) + file_name = attr_data.get_full_name() + + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + permissions = attr_data.Flags.lookup() + except ValueError: + permissions = hex(attr_data.Flags) + + yield 1, ( + format_hints.Hex(attr_data.vol.offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + mft_flag, + permissions, + attr.Attr_Header.AttrType.lookup(), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + file_name, + ) + + def _generator(self): + for record in self.enumerate_mft_records(self.parse_mft_records): + yield record + def generate_timeline(self): for row in self._generator(): _depth, row_data = row @@ -173,127 +185,98 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): self._generator(), ) - -class ADS(interfaces.plugins.PluginInterface): +class ADS(MFTScan): """Scans for Alternate Data Stream""" _required_framework_version = (2, 0, 0) - @classmethod - def get_requirements(cls): - return [ - requirements.TranslationLayerRequirement( - name="primary", - description="Memory layer for the kernel", - architectures=["Intel32", "Intel64"], - ), - requirements.VersionRequirement( - name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) - ), - ] + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # which DATA attribute should be displayed + self._display_first_data = False + + def _parse_data_record(self, mft_record, attr): + if attr.Attr_Header.NonResidentFlag: + return + + # regular $DATA + if self._display_first_data: + content = attr.get_resident_filecontent() + if content: + content = format_hints.HexBytes(content) + else: + content = renderers.NotAvailableValue() + + yield 0, ( + format_hints.Hex(self._record_map[mft_record.RecordNumber][2]), + mft_record.get_signature(), + mft_record.RecordNumber, + attr.Attr_Header.AttrType.lookup(), + self._record_map[mft_record.RecordNumber][0], + content, + ) + + # ADS $DATA + elif attr.Attr_Header.NameLength > 0: + ads_name = attr.get_resident_filename() + if not ads_name: + ads_name = renderers.NotAvailableValue() + + content = attr.get_resident_filecontent() + if content: + content = format_hints.HexBytes(content) + else: + content = renderers.NotAvailableValue() + + yield 0, ( + format_hints.Hex(self._record_map[mft_record.RecordNumber][2]), + mft_record.get_signature(), + mft_record.RecordNumber, + attr.Attr_Header.AttrType.lookup(), + self._record_map[mft_record.RecordNumber][0], + ads_name, + content, + ) + + def parse_data_records(self, mft_record, attr): + rec_num = mft_record.RecordNumber + if rec_num not in self._record_map: + # file name, DATA count, offset + self._record_map[rec_num] = [renderers.NotAvailableValue(), 0, None] + + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + attr_data = attr.Attr_Data.cast(self.fn_object) + rec_name = attr_data.get_full_name() + self._record_map[rec_num][0] = rec_name + elif attr.Attr_Header.AttrType.lookup() == "DATA": + # first data + self._record_map[rec_num][2] = attr.Attr_Data.vol.offset + + display_data = False + + # first DATA attribute of this record + if self._record_map[rec_num][1] == 0: + if self._display_first_data: + display_data = True + else: + self._record_map[rec_num][1] = 1 + + # at the second DATA attribute of this record + elif not self._display_first_data: + display_data = True + + if display_data: + for record in self._parse_data_record( + mft_record, attr + ): + yield record def _generator(self): - layer = self.context.layers[self.config["primary"]] - - # Yara Rule to scan for MFT Header Signatures - rules = yarascan.YaraScan.process_yara_options( - {"yara_string": "/FILE0|FILE\\*|BAAD/"} - ) - - # Read in the Symbol File - symbol_table = intermed.IntermediateSymbolTable.create( - context=self.context, - config_path=self.config_path, - sub_path="windows", - filename="mft", - class_types={ - "MFT_ENTRY": mft.MFTEntry, - "FILE_NAME_ENTRY": mft.MFTFileName, - "ATTRIBUTE": mft.MFTAttribute, - }, - ) - - # get each of the individual Field Sets - mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" - fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" - - # Scan the layer for Raw MFT records and parse the fields - for offset, _rule_name, _name, _value in layer.scan( - context=self.context, scanner=yarascan.YaraScanner(rules=rules) + for record in self.enumerate_mft_records( + self.parse_data_records ): - with contextlib.suppress(exceptions.PagedInvalidAddressException): - mft_record = self.context.object( - mft_object, offset=offset, layer_name=layer.name - ) - # We will update this on each pass in the next loop and use it as the new offset. - attr_base_offset = mft_record.FirstAttrOffset - - attr = self.context.object( - attribute_object, - offset=offset + attr_base_offset, - layer_name=layer.name, - ) - - # There is no field that has a count of Attributes - # Keep Attempting to read attributes until we get an invalid attr.AttrType - is_ads = False - file_name = renderers.NotAvailableValue - # The First $DATA Attr is the 'principal' file itself not the ADS - while attr.Attr_Header.AttrType.is_valid_choice: - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - attr_data = attr.Attr_Data.cast(fn_object) - file_name = attr_data.get_full_name() - if attr.Attr_Header.AttrType.lookup() == "DATA": - if is_ads: - if not attr.Attr_Header.NonResidentFlag: - # Resident files are the most interesting. - if attr.Attr_Header.NameLength > 0: - ads_name = attr.get_resident_filename() - if not ads_name: - ads_name = renderers.NotAvailableValue - - content = attr.get_resident_filecontent() - if content: - # Preparing for Disassembly - disasm = interfaces.renderers.BaseAbsentValue - architecture = layer.metadata.get( - "architecture", None - ) - if architecture: - disasm = interfaces.renderers.Disassembly( - content, 0, architecture.lower() - ) - content = format_hints.HexBytes(content) - else: - content = renderers.NotAvailableValue() - disasm = interfaces.renderers.BaseAbsentValue() - - yield 0, ( - format_hints.Hex(attr_data.vol.offset), - mft_record.get_signature(), - mft_record.RecordNumber, - attr.Attr_Header.AttrType.lookup(), - file_name, - ads_name, - content, - disasm, - ) - else: - is_ads = True - - # If there's no advancement the loop will never end, so break it now - if attr.Attr_Header.Length == 0: - break - - # Update the base offset to point to the next attribute - attr_base_offset += attr.Attr_Header.Length - # Get the next attribute - attr = self.context.object( - attribute_object, - offset=offset + attr_base_offset, - layer_name=layer.name, - ) + yield record def run(self): return renderers.TreeGrid( @@ -305,7 +288,31 @@ class ADS(interfaces.plugins.PluginInterface): ("Filename", str), ("ADS Filename", str), ("Hexdump", format_hints.HexBytes), - ("Disasm", interfaces.renderers.Disassembly), ], self._generator(), ) + +class ResidentData(ADS): + """Scans for Alternate Data Stream""" + + _required_framework_version = (2, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # which DATA attribute should be displayed + self._display_first_data = True + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Record Type", str), + ("Record Number", int), + ("MFT Type", str), + ("Filename", str), + ("Hexdump", format_hints.HexBytes), + ], + self._generator(), + ) + From 8926823199e35cd26858f6782943f4ec4e53e20e Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 18:08:08 -0500 Subject: [PATCH 477/979] Format fixes --- .../framework/plugins/windows/mftscan.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 0687c7796..d6f4684ec 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -53,7 +53,11 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): config_path=self.config_path, sub_path="windows", filename="mft", - class_types={"FILE_NAME_ENTRY": mft.MFTFileName, "MFT_ENTRY": mft.MFTEntry, "ATTRIBUTE": mft.MFTAttribute}, + class_types={ + "FILE_NAME_ENTRY": mft.MFTFileName, + "MFT_ENTRY": mft.MFTEntry, + "ATTRIBUTE": mft.MFTAttribute + }, ) # get each of the individual Field Sets @@ -185,6 +189,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): self._generator(), ) + class ADS(MFTScan): """Scans for Alternate Data Stream""" @@ -242,7 +247,7 @@ class ADS(MFTScan): def parse_data_records(self, mft_record, attr): rec_num = mft_record.RecordNumber if rec_num not in self._record_map: - # file name, DATA count, offset + # file name, DATA count, offset self._record_map[rec_num] = [renderers.NotAvailableValue(), 0, None] if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": @@ -267,15 +272,11 @@ class ADS(MFTScan): display_data = True if display_data: - for record in self._parse_data_record( - mft_record, attr - ): + for record in self._parse_data_record(mft_record, attr): yield record def _generator(self): - for record in self.enumerate_mft_records( - self.parse_data_records - ): + for record in self.enumerate_mft_records(self.parse_data_records): yield record def run(self): @@ -292,6 +293,7 @@ class ADS(MFTScan): self._generator(), ) + class ResidentData(ADS): """Scans for Alternate Data Stream""" @@ -315,4 +317,3 @@ class ResidentData(ADS): ], self._generator(), ) - From 105b4bab25511dc8f5b2461f5f2a53422b55dafa Mon Sep 17 00:00:00 2001 From: atcuno Date: Wed, 19 Jun 2024 18:09:18 -0500 Subject: [PATCH 478/979] Format fixes --- volatility3/framework/plugins/windows/mftscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index d6f4684ec..540ab531c 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -56,7 +56,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): class_types={ "FILE_NAME_ENTRY": mft.MFTFileName, "MFT_ENTRY": mft.MFTEntry, - "ATTRIBUTE": mft.MFTAttribute + "ATTRIBUTE": mft.MFTAttribute, }, ) From 9b8eea015a4544752805f0dcfa2bde33b33e3084 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Thu, 18 Jul 2024 11:57:13 -0500 Subject: [PATCH 479/979] Address feedback --- .../framework/plugins/windows/mftscan.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 540ab531c..e2dfa12a3 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,6 +5,8 @@ import contextlib import datetime import logging +from typing import Generator, Iterable + from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import conversion, format_hints @@ -23,6 +25,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._record_map = {} + self.mft_object = None + self.attribute_object = None + self.si_object = None + self.fn_object = None @classmethod def get_requirements(cls): @@ -201,12 +207,17 @@ class ADS(MFTScan): # which DATA attribute should be displayed self._display_first_data = False - def _parse_data_record(self, mft_record, attr): + def _parse_data_record( + self, + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + ) -> Generator[Iterable, None, None]: + # we only care about resident data if attr.Attr_Header.NonResidentFlag: return # regular $DATA - if self._display_first_data: + elif self._display_first_data: content = attr.get_resident_filecontent() if content: content = format_hints.HexBytes(content) @@ -244,7 +255,11 @@ class ADS(MFTScan): content, ) - def parse_data_records(self, mft_record, attr): + def parse_data_records( + self, + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + ) -> Generator[Iterable, None, None]: rec_num = mft_record.RecordNumber if rec_num not in self._record_map: # file name, DATA count, offset From 2c73d8812a24eb8e34b84cfc6be61bf06a0c18b5 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 21 Jul 2024 08:55:27 -0500 Subject: [PATCH 480/979] Split layer gathering and add KeyError checks --- volatility3/framework/plugins/windows/mftscan.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index e2dfa12a3..6a4033a6f 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -44,7 +44,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ] def enumerate_mft_records(self, attr_callback): - phys_layer = self.context.layers[self.config["primary"]].config["memory_layer"] + try: + primary = self.context.layers[self.config["primary"]] + except KeyError: + vollog.error("Unable to obtain primary layer for scanning. Please file a bug on GitHub about this issue.") + return + + try: + phys_layer = primary.config["memory_layer"] + except KeyError: + vollog.error("Unable to obtain memory layer from primary layer. Please file a bug on GitHub about this issue.") + return layer = self.context.layers[phys_layer] From a8c6db2fc4b7d7a618477639e64d2c554c8961b2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 21 Jul 2024 11:56:52 -0500 Subject: [PATCH 481/979] Formatting fixes --- volatility3/framework/plugins/windows/mftscan.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 6a4033a6f..ec8dd5fe8 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -47,13 +47,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): try: primary = self.context.layers[self.config["primary"]] except KeyError: - vollog.error("Unable to obtain primary layer for scanning. Please file a bug on GitHub about this issue.") + vollog.error( + "Unable to obtain primary layer for scanning. Please file a bug on GitHub about this issue." + ) return try: phys_layer = primary.config["memory_layer"] except KeyError: - vollog.error("Unable to obtain memory layer from primary layer. Please file a bug on GitHub about this issue.") + vollog.error( + "Unable to obtain memory layer from primary layer. Please file a bug on GitHub about this issue." + ) return layer = self.context.layers[phys_layer] From dea0e156890e620ef78aeeadce8139a81e8e9042 Mon Sep 17 00:00:00 2001 From: atcuno Date: Mon, 22 Jul 2024 16:27:05 -0500 Subject: [PATCH 482/979] Address feedback --- .../framework/plugins/windows/mftscan.py | 338 +++++++++++------- 1 file changed, 215 insertions(+), 123 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index ec8dd5fe8..0d858cec1 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,7 +5,7 @@ import contextlib import datetime import logging -from typing import Generator, Iterable +from typing import Generator, Iterable, Dict, Tuple from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -22,13 +22,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._record_map = {} - self.mft_object = None - self.attribute_object = None - self.si_object = None - self.fn_object = None + _version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -43,9 +37,15 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] - def enumerate_mft_records(self, attr_callback): + @staticmethod + def enumerate_mft_records( + context: interfaces.context.ContextInterface, + config: interfaces.configuration.HierarchicalDict, + config_path: str, + attr_callback + ) -> interfaces.objects.ObjectInterface: try: - primary = self.context.layers[self.config["primary"]] + primary = context.layers[config["primary"]] except KeyError: vollog.error( "Unable to obtain primary layer for scanning. Please file a bug on GitHub about this issue." @@ -60,7 +60,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) return - layer = self.context.layers[phys_layer] + layer = context.layers[phys_layer] # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( @@ -69,8 +69,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Read in the Symbol File symbol_table = intermed.IntermediateSymbolTable.create( - context=self.context, - config_path=self.config_path, + context=context, + config_path=config_path, sub_path="windows", filename="mft", class_types={ @@ -81,23 +81,21 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) # get each of the individual Field Sets - self.mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - self.attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" - self.si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" - self.fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" # Scan the layer for Raw MFT records and parse the fields for offset, _, _, _ in layer.scan( - context=self.context, scanner=yarascan.YaraScanner(rules=rules) + context=context, scanner=yarascan.YaraScanner(rules=rules) ): with contextlib.suppress(exceptions.InvalidAddressException): - mft_record = self.context.object( - self.mft_object, offset=offset, layer_name=layer.name + mft_record = context.object( + mft_object, offset=offset, layer_name=layer.name ) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset - attr = self.context.object( - self.attribute_object, + attr = context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -105,7 +103,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while attr.Attr_Header.AttrType.is_valid_choice: - for record in attr_callback(mft_record, attr): + for record in attr_callback(mft_record, attr, symbol_table): yield record # If there's no advancement the loop will never end, so break it now @@ -115,13 +113,14 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Update the base offset to point to the next attribute attr_base_offset += attr.Attr_Header.Length # Get the next attribute - attr = self.context.object( - self.attribute_object, + attr = context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) - def parse_mft_records(self, mft_record, attr): + @staticmethod + def parse_mft_records(mft_record, attr, symbol_table): # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record try: @@ -131,7 +130,8 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Standard Information Attribute if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": - attr_data = attr.Attr_Data.cast(self.si_object) + si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" + attr_data = attr.Attr_Data.cast(si_object) yield 0, ( format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), @@ -149,7 +149,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # File Name Attribute elif attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - attr_data = attr.Attr_Data.cast(self.fn_object) + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() # If we don't have a valid enum, coerce to hex so we can keep the record @@ -173,8 +175,114 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_name, ) + @staticmethod + def parse_data_record( + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + record_map: Dict[int, Tuple[str, int, int]], + return_first_record: bool, + ) -> Generator[Iterable, None, None]: + """ + Returns the parsed data from a MFT record + """ + # we only care about resident data + if attr.Attr_Header.NonResidentFlag: + return + + content = attr.get_resident_filecontent() + if content: + content = format_hints.HexBytes(content) + else: + content = renderers.NotAvailableValue() + + # past the first $DATA record, attempt to get the ADS name + # NotApplicableValue = 1st Data + # NotAvailableValue = > 1st Data, but name was not parsable + ads_name = renderers.NotApplicableValue() + if not return_first_record and attr.Attr_Header.NameLength > 0: + ads_name = attr.get_resident_filename() + if not ads_name: + ads_name = renderers.NotAvailableValue() + + yield ( + format_hints.Hex(record_map[mft_record.RecordNumber][2]), + mft_record.get_signature(), + mft_record.RecordNumber, + attr.Attr_Header.AttrType.lookup(), + record_map[mft_record.RecordNumber][0], + ads_name, + content, + ) + + @classmethod + def _do_parse_data_records( + cls, + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table, + record_map: Dict[int, Tuple[str, int, int]], + return_first_record: bool, + ) -> Generator[Iterable, None, None]: + """ + Parses DATA records while maintaining the FILE_NAME association + from previous parsing of the record + Suports returning the first/main $DATA as well as however many + ADS records a file might have + """ + rec_num = mft_record.RecordNumber + if rec_num not in record_map: + # file name, DATA count, offset + record_map[rec_num] = [renderers.NotAvailableValue(), 0, None] + + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + attr_data = attr.Attr_Data.cast(fn_object) + rec_name = attr_data.get_full_name() + record_map[rec_num][0] = rec_name + elif attr.Attr_Header.AttrType.lookup() == "DATA": + # first data + record_map[rec_num][2] = attr.Attr_Data.vol.offset + + display_data = False + + # first DATA attribute of this record + if record_map[rec_num][1] == 0 and return_first_record: + if return_first_record: + display_data = True + else: + record_map[rec_num][1] = 1 + + # at the second DATA attribute of this record + elif not return_first_record: + display_data = True + + if display_data: + for record in cls.parse_data_record( + mft_record, attr, record_map, return_first_record + ): + yield record + + @classmethod + def parse_data_records( + cls, + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table, + return_first_record: bool, + ): + """ + Callback for parsing data records through enumerate_mft_records + """ + record_map = {} + for record in cls._do_parse_data_records( + mft_record, attr, symbol_table, record_map, return_first_record + ): + yield record + def _generator(self): - for record in self.enumerate_mft_records(self.parse_mft_records): + for record in self.enumerate_mft_records( + self.context, self.config, self.config_path, self.parse_mft_records + ): yield record def generate_timeline(self): @@ -210,103 +318,53 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) -class ADS(MFTScan): +class ADS(interfaces.plugins.PluginInterface): """Scans for Alternate Data Stream""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 7, 0) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + _version = (1, 0, 0) - # which DATA attribute should be displayed - self._display_first_data = False + @classmethod + def get_requirements(cls): + return [ + requirements.PluginRequirement( + name="MFTScan", plugin=MFTScan, version=(2, 0, 0) + ), + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + ] - def _parse_data_record( - self, + @staticmethod + def parse_ads_data_records( mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, - ) -> Generator[Iterable, None, None]: - # we only care about resident data - if attr.Attr_Header.NonResidentFlag: - return - - # regular $DATA - elif self._display_first_data: - content = attr.get_resident_filecontent() - if content: - content = format_hints.HexBytes(content) - else: - content = renderers.NotAvailableValue() - - yield 0, ( - format_hints.Hex(self._record_map[mft_record.RecordNumber][2]), - mft_record.get_signature(), - mft_record.RecordNumber, - attr.Attr_Header.AttrType.lookup(), - self._record_map[mft_record.RecordNumber][0], - content, - ) - - # ADS $DATA - elif attr.Attr_Header.NameLength > 0: - ads_name = attr.get_resident_filename() - if not ads_name: - ads_name = renderers.NotAvailableValue() - - content = attr.get_resident_filecontent() - if content: - content = format_hints.HexBytes(content) - else: - content = renderers.NotAvailableValue() - - yield 0, ( - format_hints.Hex(self._record_map[mft_record.RecordNumber][2]), - mft_record.get_signature(), - mft_record.RecordNumber, - attr.Attr_Header.AttrType.lookup(), - self._record_map[mft_record.RecordNumber][0], - ads_name, - content, - ) - - def parse_data_records( - self, - mft_record: interfaces.objects.ObjectInterface, - attr: interfaces.objects.ObjectInterface, - ) -> Generator[Iterable, None, None]: - rec_num = mft_record.RecordNumber - if rec_num not in self._record_map: - # file name, DATA count, offset - self._record_map[rec_num] = [renderers.NotAvailableValue(), 0, None] - - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - attr_data = attr.Attr_Data.cast(self.fn_object) - rec_name = attr_data.get_full_name() - self._record_map[rec_num][0] = rec_name - elif attr.Attr_Header.AttrType.lookup() == "DATA": - # first data - self._record_map[rec_num][2] = attr.Attr_Data.vol.offset - - display_data = False - - # first DATA attribute of this record - if self._record_map[rec_num][1] == 0: - if self._display_first_data: - display_data = True - else: - self._record_map[rec_num][1] = 1 - - # at the second DATA attribute of this record - elif not self._display_first_data: - display_data = True - - if display_data: - for record in self._parse_data_record(mft_record, attr): - yield record + symbol_table, + ): + return MFTScan.parse_data_records(mft_record, attr, symbol_table, False) def _generator(self): - for record in self.enumerate_mft_records(self.parse_data_records): - yield record + for ( + offset, + rec_type, + rec_num, + attr_type, + file_name, + ads_name, + content, + ) in MFTScan.enumerate_mft_records( + self.context, self.config, self.config_path, self.parse_ads_data_records + ): + yield ( + 0, + (offset, rec_type, rec_num, attr_type, file_name, ads_name, content), + ) def run(self): return renderers.TreeGrid( @@ -323,16 +381,50 @@ class ADS(MFTScan): ) -class ResidentData(ADS): +class ResidentData(interfaces.plugins.PluginInterface): """Scans for Alternate Data Stream""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 7, 0) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + _version = (1, 0, 0) - # which DATA attribute should be displayed - self._display_first_data = True + @classmethod + def get_requirements(cls): + return [ + requirements.PluginRequirement( + name="MFTScan", plugin=MFTScan, version=(2, 0, 0) + ), + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + ] + + @staticmethod + def parse_first_data_records( + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table, + ): + return MFTScan.parse_data_records(mft_record, attr, symbol_table, True) + + def _generator(self): + for ( + offset, + rec_type, + rec_num, + attr_type, + file_name, + _, + content, + ) in MFTScan.enumerate_mft_records( + self.context, self.config, self.config_path, self.parse_first_data_records + ): + yield (0, (offset, rec_type, rec_num, attr_type, file_name, content)) def run(self): return renderers.TreeGrid( From da843b0367b7353274626f79d09ddb4377a9148b Mon Sep 17 00:00:00 2001 From: atcuno Date: Mon, 22 Jul 2024 16:27:39 -0500 Subject: [PATCH 483/979] Address feedback --- volatility3/framework/plugins/windows/mftscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 0d858cec1..6a2aac440 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -42,7 +42,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): context: interfaces.context.ContextInterface, config: interfaces.configuration.HierarchicalDict, config_path: str, - attr_callback + attr_callback, ) -> interfaces.objects.ObjectInterface: try: primary = context.layers[config["primary"]] From 33716be15b4088e7743bc938f2c0e83ab8b5ea18 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Wed, 31 Jul 2024 13:56:38 -0500 Subject: [PATCH 484/979] Rework for proper ADS recovery. Memory OOM issues --- .../framework/plugins/windows/mftscan.py | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 6a2aac440..430f3fc02 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -84,8 +84,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mft_object = symbol_table + constants.BANG + "MFT_ENTRY" attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" + record_map = {} + # Scan the layer for Raw MFT records and parse the fields - for offset, _, _, _ in layer.scan( + for offset, _rule_name, _name, _value in layer.scan( context=context, scanner=yarascan.YaraScanner(rules=rules) ): with contextlib.suppress(exceptions.InvalidAddressException): @@ -103,7 +105,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while attr.Attr_Header.AttrType.is_valid_choice: - for record in attr_callback(mft_record, attr, symbol_table): + for record in attr_callback(record_map, mft_record, attr, symbol_table): yield record # If there's no advancement the loop will never end, so break it now @@ -120,7 +122,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) @staticmethod - def parse_mft_records(mft_record, attr, symbol_table): + def parse_mft_records(record_map, mft_record, attr, symbol_table): # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record try: @@ -189,21 +191,27 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if attr.Attr_Header.NonResidentFlag: return + # we aren't looking ADS when we want the first data record + if return_first_record: + ads_name = renderers.NotApplicableValue() + + # skip records without a name if we want ADS entries + elif attr.Attr_Header.NameLength == 0: + return + + else: + # past the first $DATA record, attempt to get the ADS name + # NotAvailableValue = > 1st Data, but name was not parsable + ads_name = attr.get_resident_filename() + if not ads_name: + ads_name = renderers.NotAvailableValue() + content = attr.get_resident_filecontent() if content: content = format_hints.HexBytes(content) else: content = renderers.NotAvailableValue() - # past the first $DATA record, attempt to get the ADS name - # NotApplicableValue = 1st Data - # NotAvailableValue = > 1st Data, but name was not parsable - ads_name = renderers.NotApplicableValue() - if not return_first_record and attr.Attr_Header.NameLength > 0: - ads_name = attr.get_resident_filename() - if not ads_name: - ads_name = renderers.NotAvailableValue() - yield ( format_hints.Hex(record_map[mft_record.RecordNumber][2]), mft_record.get_signature(), @@ -246,14 +254,15 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): display_data = False # first DATA attribute of this record - if record_map[rec_num][1] == 0 and return_first_record: + if record_map[rec_num][1] == 0: if return_first_record: display_data = True - else: - record_map[rec_num][1] = 1 + + record_map[rec_num][1] = 1 # at the second DATA attribute of this record - elif not return_first_record: + elif record_map[rec_num][1] == 1 and not return_first_record: + print("at second record") display_data = True if display_data: @@ -265,6 +274,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def parse_data_records( cls, + record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, symbol_table, @@ -273,7 +283,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ Callback for parsing data records through enumerate_mft_records """ - record_map = {} for record in cls._do_parse_data_records( mft_record, attr, symbol_table, record_map, return_first_record ): @@ -343,11 +352,12 @@ class ADS(interfaces.plugins.PluginInterface): @staticmethod def parse_ads_data_records( + record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, symbol_table, ): - return MFTScan.parse_data_records(mft_record, attr, symbol_table, False) + return MFTScan.parse_data_records(record_map, mft_record, attr, symbol_table, False) def _generator(self): for ( @@ -382,7 +392,7 @@ class ADS(interfaces.plugins.PluginInterface): class ResidentData(interfaces.plugins.PluginInterface): - """Scans for Alternate Data Stream""" + """Scans for MFT Records with Resident Data""" _required_framework_version = (2, 7, 0) @@ -406,11 +416,12 @@ class ResidentData(interfaces.plugins.PluginInterface): @staticmethod def parse_first_data_records( + record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, symbol_table, ): - return MFTScan.parse_data_records(mft_record, attr, symbol_table, True) + return MFTScan.parse_data_records(record_map, mft_record, attr, symbol_table, True) def _generator(self): for ( From da41124b1cb936ab168555568cf17512f719b6b4 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 2 Sep 2024 14:20:02 -0500 Subject: [PATCH 485/979] Fix formatting with black --- volatility3/framework/plugins/windows/mftscan.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 430f3fc02..2929b0522 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -105,7 +105,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while attr.Attr_Header.AttrType.is_valid_choice: - for record in attr_callback(record_map, mft_record, attr, symbol_table): + for record in attr_callback( + record_map, mft_record, attr, symbol_table + ): yield record # If there's no advancement the loop will never end, so break it now @@ -357,7 +359,9 @@ class ADS(interfaces.plugins.PluginInterface): attr: interfaces.objects.ObjectInterface, symbol_table, ): - return MFTScan.parse_data_records(record_map, mft_record, attr, symbol_table, False) + return MFTScan.parse_data_records( + record_map, mft_record, attr, symbol_table, False + ) def _generator(self): for ( @@ -421,7 +425,9 @@ class ResidentData(interfaces.plugins.PluginInterface): attr: interfaces.objects.ObjectInterface, symbol_table, ): - return MFTScan.parse_data_records(record_map, mft_record, attr, symbol_table, True) + return MFTScan.parse_data_records( + record_map, mft_record, attr, symbol_table, True + ) def _generator(self): for ( From 685dc97298719b9bcfb24d5a020b7d4550580f4c Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Mon, 30 Sep 2024 18:11:23 -0500 Subject: [PATCH 486/979] fix resident data bug for duplicate mft record numbers --- .../framework/plugins/windows/mftscan.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 2929b0522..16425ca22 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -215,11 +215,11 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): content = renderers.NotAvailableValue() yield ( - format_hints.Hex(record_map[mft_record.RecordNumber][2]), + format_hints.Hex(record_map[mft_record.vol.offset][2]), mft_record.get_signature(), mft_record.RecordNumber, attr.Attr_Header.AttrType.lookup(), - record_map[mft_record.RecordNumber][0], + record_map[mft_record.vol.offset][0], ads_name, content, ) @@ -239,31 +239,29 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Suports returning the first/main $DATA as well as however many ADS records a file might have """ - rec_num = mft_record.RecordNumber - if rec_num not in record_map: + if mft_record.vol.offset not in record_map: # file name, DATA count, offset - record_map[rec_num] = [renderers.NotAvailableValue(), 0, None] - + record_map[mft_record.vol.offset] = [renderers.NotAvailableValue(), 0, None] if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" attr_data = attr.Attr_Data.cast(fn_object) rec_name = attr_data.get_full_name() - record_map[rec_num][0] = rec_name + record_map[mft_record.vol.offset][0] = rec_name elif attr.Attr_Header.AttrType.lookup() == "DATA": # first data - record_map[rec_num][2] = attr.Attr_Data.vol.offset + record_map[mft_record.vol.offset][2] = attr.Attr_Data.vol.offset display_data = False # first DATA attribute of this record - if record_map[rec_num][1] == 0: + if record_map[mft_record.vol.offset][1] == 0: if return_first_record: display_data = True - record_map[rec_num][1] = 1 + record_map[mft_record.vol.offset][1] = 1 # at the second DATA attribute of this record - elif record_map[rec_num][1] == 1 and not return_first_record: + elif record_map[mft_record.vol.offset][1] == 1 and not return_first_record: print("at second record") display_data = True From 79f81f0af9193878f0290d0ae05d5178e71502c6 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 15 Oct 2024 14:11:27 -0500 Subject: [PATCH 487/979] Address feedback --- .../framework/plugins/windows/mftscan.py | 37 +++++-------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 16425ca22..94dce8c4d 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -40,12 +40,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @staticmethod def enumerate_mft_records( context: interfaces.context.ContextInterface, - config: interfaces.configuration.HierarchicalDict, config_path: str, + primary_layer_name: str, attr_callback, ) -> interfaces.objects.ObjectInterface: try: - primary = context.layers[config["primary"]] + primary = context.layers[primary_layer_name] except KeyError: vollog.error( "Unable to obtain primary layer for scanning. Please file a bug on GitHub about this issue." @@ -105,10 +105,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while attr.Attr_Header.AttrType.is_valid_choice: - for record in attr_callback( + yield from attr_callback( record_map, mft_record, attr, symbol_table - ): - yield record + ) # If there's no advancement the loop will never end, so break it now if attr.Attr_Header.Length == 0: @@ -225,12 +224,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) @classmethod - def _do_parse_data_records( + def parse_data_records( cls, + record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, symbol_table, - record_map: Dict[int, Tuple[str, int, int]], return_first_record: bool, ) -> Generator[Iterable, None, None]: """ @@ -262,7 +261,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # at the second DATA attribute of this record elif record_map[mft_record.vol.offset][1] == 1 and not return_first_record: - print("at second record") display_data = True if display_data: @@ -271,26 +269,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ): yield record - @classmethod - def parse_data_records( - cls, - record_map: Dict[int, Tuple[str, int, int]], - mft_record: interfaces.objects.ObjectInterface, - attr: interfaces.objects.ObjectInterface, - symbol_table, - return_first_record: bool, - ): - """ - Callback for parsing data records through enumerate_mft_records - """ - for record in cls._do_parse_data_records( - mft_record, attr, symbol_table, record_map, return_first_record - ): - yield record - def _generator(self): for record in self.enumerate_mft_records( - self.context, self.config, self.config_path, self.parse_mft_records + self.context, self.config_path, self.config["primary"], self.parse_mft_records ): yield record @@ -371,7 +352,7 @@ class ADS(interfaces.plugins.PluginInterface): ads_name, content, ) in MFTScan.enumerate_mft_records( - self.context, self.config, self.config_path, self.parse_ads_data_records + self.context, self.config_path, self.config["primary"], self.parse_ads_data_records ): yield ( 0, @@ -437,7 +418,7 @@ class ResidentData(interfaces.plugins.PluginInterface): _, content, ) in MFTScan.enumerate_mft_records( - self.context, self.config, self.config_path, self.parse_first_data_records + self.context, self.config_path, self.config["primary"], self.parse_first_data_records ): yield (0, (offset, rec_type, rec_num, attr_type, file_name, content)) From adee265b1775a15913c0ce7d591cd9c6ccb32e18 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Tue, 15 Oct 2024 14:12:23 -0500 Subject: [PATCH 488/979] Address feedback --- .../framework/plugins/windows/mftscan.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 94dce8c4d..014516b7c 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -105,9 +105,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType while attr.Attr_Header.AttrType.is_valid_choice: - yield from attr_callback( - record_map, mft_record, attr, symbol_table - ) + yield from attr_callback(record_map, mft_record, attr, symbol_table) # If there's no advancement the loop will never end, so break it now if attr.Attr_Header.Length == 0: @@ -271,7 +269,10 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): def _generator(self): for record in self.enumerate_mft_records( - self.context, self.config_path, self.config["primary"], self.parse_mft_records + self.context, + self.config_path, + self.config["primary"], + self.parse_mft_records, ): yield record @@ -352,7 +353,10 @@ class ADS(interfaces.plugins.PluginInterface): ads_name, content, ) in MFTScan.enumerate_mft_records( - self.context, self.config_path, self.config["primary"], self.parse_ads_data_records + self.context, + self.config_path, + self.config["primary"], + self.parse_ads_data_records, ): yield ( 0, @@ -418,7 +422,10 @@ class ResidentData(interfaces.plugins.PluginInterface): _, content, ) in MFTScan.enumerate_mft_records( - self.context, self.config_path, self.config["primary"], self.parse_first_data_records + self.context, + self.config_path, + self.config["primary"], + self.parse_first_data_records, ): yield (0, (offset, rec_type, rec_num, attr_type, file_name, content)) From d98c7ebb1d21f3074d413e7097bde7448932db85 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 16 Oct 2024 15:05:02 +1100 Subject: [PATCH 489/979] linux: hidden_modules: Make the fast method the default. Remove vol2 and fall back to a 1-byte alignment scan if addresses aren't aligned to the L1 cache size --- .../framework/plugins/linux/hidden_modules.py | 216 +++--------------- 1 file changed, 26 insertions(+), 190 deletions(-) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index 6542f58f3..69b449261 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -32,12 +32,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), - requirements.BooleanRequirement( - name="fast", - description="Fast scan method. Recommended only for kernels 4.2 and above", - optional=True, - default=False, - ), ] @staticmethod @@ -86,134 +80,13 @@ class Hidden_modules(interfaces.plugins.PluginInterface): return modules_addr_min, modules_addr_max - @staticmethod - def _get_module_state_values_bytes( - context: interfaces.context.ContextInterface, - vmlinux_module_name: str, - ) -> List[bytes]: - """Retrieve the module state values bytes by introspecting its enum type - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - vmlinux_module_name: The name of the kernel module on which to operate - - Returns: - A list with the module state values bytes - """ - vmlinux = context.modules[vmlinux_module_name] - module_state_type_template = vmlinux.get_type("module").vol.members["state"][1] - data_format = module_state_type_template.base_type.vol.data_format - values = module_state_type_template.choices.values() - values_bytes = [ - objects.convert_value_to_data(value, int, data_format) - for value in sorted(values) - ] - return values_bytes - - @classmethod - def _get_hidden_modules_vol2( - cls, - context: interfaces.context.ContextInterface, - vmlinux_module_name: str, - known_module_addresses: Set[int], - modules_memory_boundaries: Tuple, - ) -> Iterable[interfaces.objects.ObjectInterface]: - """Enumerate hidden modules using the traditional implementation. - - This is a port of the Volatility2 plugin, with minor code improvements. - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - vmlinux_module_name: The name of the kernel module on which to operate - known_module_addresses: Set with known module addresses - modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. - - Yields: - module objects - """ - vmlinux = context.modules[vmlinux_module_name] - vmlinux_layer = context.layers[vmlinux.layer_name] - - check_nums = ( - 3000, - 2800, - 2700, - 2500, - 2300, - 2100, - 2000, - 1500, - 1300, - 1200, - 1024, - 512, - 256, - 128, - 96, - 64, - 48, - 32, - 24, - ) - modules_addr_min, modules_addr_max = modules_memory_boundaries - modules_addr_min = modules_addr_min & ~0xFFF - modules_addr_max = (modules_addr_max & ~0xFFF) + vmlinux_layer.page_size - - check_bufs = [] - replace_bufs = [] - minus_size = vmlinux.get_type("pointer").size - null_pointer_bytes = b"\x00" * minus_size - for num in check_nums: - check_bufs.append(b"\x00" * num) - replace_bufs.append((b"\xff" * (num - minus_size)) + null_pointer_bytes) - - all_ffs = b"\xff" * 4096 - scan_list = [] - for page_addr in range( - modules_addr_min, modules_addr_max, vmlinux_layer.page_size - ): - content_fixed = all_ffs - with contextlib.suppress( - exceptions.InvalidAddressException, - exceptions.PagedInvalidAddressException, - ): - content = vmlinux_layer.read(page_addr, vmlinux_layer.page_size) - - all_nulls = all(x == 0 for x in content) - if content and not all_nulls: - content_fixed = content - for check_bytes, replace_bytes in zip(check_bufs, replace_bufs): - content_fixed = content_fixed.replace( - check_bytes, replace_bytes - ) - - scan_list.append(content_fixed) - - scan_buf = b"".join(scan_list) - del scan_list - - module_state_values_bytes = cls._get_module_state_values_bytes( - context, vmlinux_module_name - ) - values_bytes_pattern = b"|".join(module_state_values_bytes) - # f'strings cannot be combined with bytes literals - for cur_addr in re.finditer(b"(?=(%s))" % values_bytes_pattern, scan_buf): - module_addr = modules_addr_min + cur_addr.start() - - if module_addr in known_module_addresses: - continue - - module = vmlinux.object("module", offset=module_addr, absolute=True) - if module and module.is_valid(): - yield module - @classmethod def _get_module_address_alignment( cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, ) -> int: - """Obtain the module memory address alignment. This is only used with the fast scan method. + """Obtain the module memory address alignment. struct module is aligned to the L1 cache line, which is typically 64 bytes for most common i386/AMD64/ARM64 configurations. In some cases, it can be 128 bytes, but this @@ -231,8 +104,24 @@ class Hidden_modules(interfaces.plugins.PluginInterface): # essential for retrieving type metadata in the future. return 64 + @staticmethod + def _validate_alignment_patterns( + addresses: Iterable[int], + address_alignment: int, + ) -> bool: + """Check if the memory addresses meet our alignments patterns + + Args: + addresses: Iterable with the address values + address_alignment: Number of bytes for alignment validation + + Returns: + True if all the addresses meet the alignment + """ + return all(addr % address_alignment == 0 for addr in addresses) + @classmethod - def _get_hidden_modules_fast( + def get_hidden_modules( cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, @@ -268,6 +157,14 @@ class Hidden_modules(interfaces.plugins.PluginInterface): module_address_alignment = cls._get_module_address_alignment( context, vmlinux_module_name ) + if not cls._validate_alignment_patterns( + known_module_addresses, module_address_alignment + ): + vollog.warning( + f"Module addresses aren't aligned to {module_address_alignment} bytes. " + "Switching to 1 byte aligment scan method." + ) + module_address_alignment = 1 mkobj_offset = vmlinux.get_type("module").relative_child_offset("mkobj") mod_offset = vmlinux.get_type("module_kobject").relative_child_offset("mod") @@ -302,66 +199,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): if module and module.is_valid(): yield module - @staticmethod - def _validate_alignment_patterns( - addresses: Iterable[int], - address_alignment: int, - ) -> bool: - """Check if the memory addresses meet our alignments patterns - - Args: - addresses: Iterable with the address values - address_alignment: Number of bytes for alignment validation - - Returns: - True if all the addresses meet the alignment - """ - return all(addr % address_alignment == 0 for addr in addresses) - - @classmethod - def get_hidden_modules( - cls, - context: interfaces.context.ContextInterface, - vmlinux_module_name: str, - known_module_addresses: Set[int], - modules_memory_boundaries: Tuple, - fast_method: bool = False, - ) -> Iterable[interfaces.objects.ObjectInterface]: - """Enumerate hidden modules - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - vmlinux_module_name: The name of the kernel module on which to operate - known_module_addresses: Set with known module addresses - modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. - fast_method: If True, it uses the fast method. Otherwise, it uses the traditional one. - Yields: - module objects - """ - if fast_method: - module_address_alignment = cls._get_module_address_alignment( - context, vmlinux_module_name - ) - if cls._validate_alignment_patterns( - known_module_addresses, module_address_alignment - ): - scan_method = cls._get_hidden_modules_fast - else: - vollog.warning( - f"Module addresses aren't aligned to {module_address_alignment} bytes. " - "Switching to the traditional scan method." - ) - scan_method = cls._get_hidden_modules_vol2 - else: - scan_method = cls._get_hidden_modules_vol2 - - yield from scan_method( - context, - vmlinux_module_name, - known_module_addresses, - modules_memory_boundaries, - ) - @classmethod def get_lsmod_module_addresses( cls, @@ -399,7 +236,6 @@ class Hidden_modules(interfaces.plugins.PluginInterface): vmlinux_module_name, known_module_addresses, modules_memory_boundaries, - fast_method=self.config.get("fast"), ): module_addr = module.vol.offset module_name = module.get_name() or renderers.NotAvailableValue() From 0ddd9210a8e95a9172a36f4939be68fff6edd8ed Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 16 Oct 2024 15:08:59 +1100 Subject: [PATCH 490/979] linux: hidden_modules: Remove unused module imports --- volatility3/framework/plugins/linux/hidden_modules.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index 69b449261..679f2b00a 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -1,9 +1,7 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import re import logging -import contextlib from typing import List, Set, Tuple, Iterable from volatility3.framework import renderers, interfaces, exceptions, objects from volatility3.framework.constants.architectures import LINUX_ARCHS From 18f7f035ee83080d2ea660acfef05453c8e04ab5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 18 Oct 2024 12:29:09 +1100 Subject: [PATCH 491/979] linux: fix datetime import and remove unused ones --- .../framework/symbols/linux/extensions/__init__.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0dab0db43..d9392b9ba 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -7,13 +7,12 @@ import logging import functools import binascii import stat -from datetime import datetime +import datetime import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, Dict from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion -from volatility3.framework.configuration import requirements from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS @@ -1882,7 +1881,7 @@ class kernel_cap_t(kernel_cap_struct): class timespec64(objects.StructType): - def to_datetime(self) -> datetime: + def to_datetime(self) -> datetime.datetime: """Returns the respective aware datetime""" dt = conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) @@ -1958,7 +1957,7 @@ class inode(objects.StructType): else: return None - def _time_member_to_datetime(self, member) -> datetime: + def _time_member_to_datetime(self, member) -> datetime.datetime: if self.has_member(f"{member}_sec") and self.has_member(f"{member}_nsec"): # kernels >= 6.11 it's i_*_sec -> time64_t and i_*_nsec -> u32 # Ref Linux commit 3aa63a569c64e708df547a8913c84e64a06e7853 @@ -1977,7 +1976,7 @@ class inode(objects.StructType): "Unsupported kernel inode type implementation" ) - def get_access_time(self) -> datetime: + def get_access_time(self) -> datetime.datetime: """Returns the inode's last access time This is updated when inode contents are read @@ -1986,7 +1985,7 @@ class inode(objects.StructType): """ return self._time_member_to_datetime("i_atime") - def get_modification_time(self) -> datetime: + def get_modification_time(self) -> datetime.datetime: """Returns the inode's last modification time This is updated when the inode contents change @@ -1996,7 +1995,7 @@ class inode(objects.StructType): return self._time_member_to_datetime("i_mtime") - def get_change_time(self) -> datetime: + def get_change_time(self) -> datetime.datetime: """Returns the inode's last change time This is updated when the inode metadata changes From f8224684078dd9937da4b0924f504a0bfb25b8bf Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 18 Oct 2024 12:40:02 +1100 Subject: [PATCH 492/979] linux: Implement boot time support in the Volatility 3 core framework --- .../framework/constants/linux/__init__.py | 3 + .../framework/symbols/linux/__init__.py | 98 +++++++++ .../symbols/linux/extensions/__init__.py | 197 ++++++++++++++++++ 3 files changed, 298 insertions(+) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 3eabc2341..a481e1ba9 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -302,3 +302,6 @@ class ELF_CLASS(IntEnum): ELFCLASSNONE = 0 ELFCLASS32 = 1 ELFCLASS64 = 2 + + +NSEC_PER_SEC = 1e9 diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 219f120ee..95b476263 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -3,11 +3,15 @@ # import math import contextlib +import datetime +import dataclasses from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects +from volatility3.framework.renderers import conversion +from volatility3.framework.constants.linux import NSEC_PER_SEC from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -830,3 +834,97 @@ class PageCache(object): page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page + + +@dataclasses.dataclass +class TimespecVol3(object): + """Internal helper class to handle all required timespec operations, convertions and + adjustments. + + NOTE: This is intended for exclusive use with get_boottime() and its related functions. + """ + + tv_sec: int = 0 + tv_nsec: int = 0 + + @classmethod + def new_from_timespec(cls, timespec) -> "TimespecVol3": + """Creates a new instance from a TimespecVol3 or timespec64 object""" + if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): + raise TypeError("It requires either a TimespecVol3 or timespec64 type") + + tv_sec = int(timespec.tv_sec) + tv_nsec = int(timespec.tv_nsec) + return cls(tv_sec=tv_sec, tv_nsec=tv_nsec) + + @classmethod + def new_from_nsec(cls, nsec) -> "TimespecVol3": + """Creates a new instance from an integer in nanoseconds""" + + # Based on ns_to_timespec64() + if nsec > 0: + tv_sec = nsec // NSEC_PER_SEC + tv_nsec = nsec % NSEC_PER_SEC + elif nsec < 0: + tv_sec = -((-nsec - 1) // NSEC_PER_SEC) - 1 + rem = (-nsec - 1) % NSEC_PER_SEC + tv_nsec = NSEC_PER_SEC - rem - 1 + else: + tv_sec = tv_nsec = 0 + + return cls(tv_sec=tv_sec, tv_nsec=tv_nsec) + + def to_datetime(self) -> datetime.datetime: + """Converts this TimespecVol3 to a UTC aware datetime""" + return conversion.unixtime_to_datetime( + self.tv_sec + self.tv_nsec / NSEC_PER_SEC + ) + + def to_timedelta(self) -> datetime.timedelta: + """Converts this TimespecVol3 to timedelta""" + return datetime.timedelta(seconds=self.tv_sec + self.tv_nsec / NSEC_PER_SEC) + + def __add__(self, timespec) -> "TimespecVol3": + """Returns a new TimespecVol3 object that sums the current values with those + in the timespec argument""" + if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): + raise TypeError("Cannot add a TimespecVol3 to this object") + + result = TimespecVol3( + tv_sec=self.tv_sec + timespec.tv_sec, + tv_nsec=self.tv_nsec + timespec.tv_nsec, + ) + + result.normalize() + + return result + + def __sub__(self, timespec) -> "TimespecVol3": + """Returns a new TimespecVol3 object that subtracts the values in the timespec + argument from the current object's values""" + if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): + raise TypeError("Cannot add a TimespecVol3 to this object") + + result = TimespecVol3( + tv_sec=self.tv_sec - timespec.tv_sec, + tv_nsec=self.tv_nsec - timespec.tv_nsec, + ) + result.normalize() + + return result + + def normalize(self): + """Normalize any overflow in tv_sec and tv_nsec after previous addition or subtractions""" + # Based on kernel's set_normalized_timespec64() + while self.tv_nsec >= NSEC_PER_SEC: + self.tv_nsec -= NSEC_PER_SEC + self.tv_sec += 1 + + while self.tv_nsec < 0: + self.tv_nsec += NSEC_PER_SEC + self.tv_sec -= 1 + + def negate(self): + """Negates the sign of both tv_sec and tv_nsec""" + self.tv_sec = -self.tv_sec + self.tv_nsec = -self.tv_nsec diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d9392b9ba..b3e836102 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -382,6 +382,203 @@ class task_struct(generic.GenericIntelProcess): threads_seen.add(task.vol.offset) yield task + def _get_task_start_time(self) -> datetime.timedelta: + """Returns the task's monotonic start_time as a timedelta. + + Returns: + The task's start time as a timedelta object. + """ + for member_name in ("start_boottime", "real_start_time", "start_time"): + if self.has_member(member_name): + start_time_obj = self.member(member_name) + start_time_obj_type = start_time_obj.vol.type_name + start_time_obj_type_name = start_time_obj_type.split(constants.BANG)[1] + if start_time_obj_type_name != "timespec": + # kernels >= 3.17 real_start_time and start_time are u64 + # kernels >= 5.5 uses start_boottime which is also a u64 + start_time = linux.TimespecVol3.new_from_nsec(start_time_obj) + else: + # kernels < 3.17 real_start_time and start_time are timespec + start_time = linux.TimespecVol3.new_from_timespec(start_time_obj) + + # This is relative to the boot time so it makes sense to be a timedelta. + return start_time.to_timedelta() + + raise AttributeError("Unsupported task_struct start_time member") + + def get_time_namespace(self) -> Optional[interfaces.objects.ObjectInterface]: + """Returns the task's time namespace""" + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + if not self.has_member("nsproxy"): + # kernels < 2.6.19: ab516013ad9ca47f1d3a936fa81303bfbf734d52 + return None + + if not vmlinux.get_type("nsproxy").has_member("time_ns"): + # kernels < 5.6 769071ac9f20b6a447410c7eaa55d1a5233ef40c + return None + + return self.nsproxy.time_ns + + def get_time_namespace_id(self) -> int: + """Returns the task's time namespace ID.""" + time_ns = self.get_time_namespace() + if not time_ns: + # kernels < 5.6 + return + + # We are good. ns_common (ns) was introduced in kernels 3.19. So by the time the + # time namespace was added in kernels 5.6, it already included the ns member. + return time_ns.ns.inum + + def _get_time_namespace_offsets( + self, + ) -> Optional[interfaces.objects.ObjectInterface]: + """Returns the time offsets from the task's time namespace.""" + time_ns = self.get_time_namespace() + if not time_ns: + # kernels < 5.6 + return + + if not time_ns.has_member("offsets"): + # kernels < 5.6 af993f58d69ee9c1f421dfc87c3ed231c113989c + return None + + return time_ns.offsets + + def get_time_namespace_monotonic_offset( + self, + ) -> Optional[interfaces.objects.ObjectInterface]: + """Gets task's time namespace monotonic offset + + Returns: + a kernel's timespec64 object with the monotonic offset + """ + time_namespace_offsets = self._get_time_namespace_offsets() + if not time_namespace_offsets: + return None + + return time_namespace_offsets.monotonic + + def _get_time_namespace_boottime_offset( + self, + ) -> Optional[interfaces.objects.ObjectInterface]: + """Gets task's time namespace boottime offset + + Returns: + a kernel's timespec64 object with the boottime offset + """ + time_namespace_offsets = self._get_time_namespace_offsets() + if not time_namespace_offsets: + return None + + return time_namespace_offsets.boottime + + def _get_boottime_raw(self) -> "linux.TimespecVol3": + """Returns the boot time in a TimespecVol3.""" + + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + if vmlinux.has_symbol("tk_core"): + # kernels >= 3.17 | tk_core | 3fdb14fd1df70325e1e91e1203a699a4803ed741 + tk_core = vmlinux.object_from_symbol("tk_core") + timekeeper = tk_core.timekeeper + if not timekeeper.offs_real.has_member("tv64"): + # kernels >= 4.10 - Tested on Ubuntu 6.8.0-41 + boottime_nsec = timekeeper.offs_real - timekeeper.offs_boot + else: + # 3.17 <= kernels < 4.10 - Tested on Ubuntu 4.4.0-142 + boottime_nsec = timekeeper.offs_real.tv64 - timekeeper.offs_boot.tv64 + return linux.TimespecVol3.new_from_nsec(boottime_nsec) + + elif vmlinux.has_symbol("timekeeper") and vmlinux.get_type( + "timekeeper" + ).has_member("wall_to_monotonic"): + # 3.4 <= kernels < 3.17 - Tested on Ubuntu 3.13.0-185 + timekeeper = vmlinux.object_from_symbol("timekeeper") + + # timekeeper.wall_to_monotonic is timespec + boottime = linux.TimespecVol3.new_from_timespec( + timekeeper.wall_to_monotonic + ) + + boottime += timekeeper.total_sleep_time + + boottime.negate() + boottime.normalize() + + return boottime + + elif vmlinux.has_symbol("wall_to_monotonic"): + # kernels < 3.4 - Tested on Debian7 3.2.0-4 (3.2.57-3+deb7u2) + wall_to_monotonic = vmlinux.object_from_symbol("wall_to_monotonic") + boottime = linux.TimespecVol3.new_from_timespec(wall_to_monotonic) + if vmlinux.has_symbol("total_sleep_time"): + # 2.6.23 <= kernels < 3.4 7c3f1a573237b90ef331267260358a0ec4ac9079 + total_sleep_time = vmlinux.object_from_symbol("total_sleep_time") + full_type_name = total_sleep_time.vol.type_name + type_name = full_type_name.split(constants.BANG)[1] + if type_name == "timespec": + # kernels >= 2.6.32 total_sleep_time is a timespec + boottime += total_sleep_time + else: + # kernels < 2.6.32 total_sleep_time is an unsigned long as seconds + boottime.tv_sec += total_sleep_time + + boottime.negate() + boottime.normalize() + + return boottime + + raise exceptions.VolatilityException("Unsupported") + + def get_boottime(self, root_time_namespace: bool = True) -> datetime.datetime: + """Returns the boot time in UTC as a datetime. + + Args: + root_time_namespace: If True, it returns the boot time as seen from the root + time namespace. Otherwise, it returns the boot time relative to the + task's time namespace. + + Returns: + A datetime with the UTC boot time. + """ + boottime = self._get_boottime_raw() + if not boottime: + return None + + if not root_time_namespace: + # Shift boot timestamp according to the task's time namespace offset + boottime_offset_timespec = self._get_time_namespace_boottime_offset() + if boottime_offset_timespec: + # Time namespace support is from kernels 5.6 + boottime -= boottime_offset_timespec + + return boottime.to_datetime() + + def get_create_time(self) -> datetime.datetime: + """Retrieves the task's start time from its time namespace. + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + task: A reference task + + Returns: + A datetime with task's start time + """ + # Typically, we want to see the creation time seen from the root time namespace + boottime = self.get_boottime(root_time_namespace=True) + + # The kernel exports only tv_sec to procfs, see kernel's show_stat(). + # This means user-space tools, like those in the procps package (e.g., ps, top, etc.), + # only use the boot time seconds to compute dates relatives to this. + boottime = boottime.replace(microsecond=0) + + task_start_time_timedelta = self._get_task_start_time() + + # NOTE: Do NOT apply the task's time namespace offsets here. While the kernel uses + # timens_add_boottime_ns(), it's not needed here since we're seeing it from the + # root time namespace, not within the task's own time namespace + return boottime + task_start_time_timedelta + class fs_struct(objects.StructType): def get_root_dentry(self): From 69512dc9911fc6ce8d0e1ab65926663341513974 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 18 Oct 2024 12:58:02 +1100 Subject: [PATCH 493/979] Linux: pslist: Add creation time column and timeline support to the linux.pslist plugin --- volatility3/framework/plugins/linux/pslist.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 1888bd7b8..b05d69c7a 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -1,6 +1,7 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import datetime from typing import Any, Callable, Iterable, List, Tuple from volatility3.framework import interfaces, renderers @@ -9,15 +10,16 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux.extensions import elf +from volatility3.plugins import timeliner from volatility3.plugins.linux import elfs -class PsList(interfaces.plugins.PluginInterface): +class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 0, 0) - _version = (2, 2, 1) + _version = (2, 3, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -81,7 +83,7 @@ class PsList(interfaces.plugins.PluginInterface): @classmethod def get_task_fields( cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False - ) -> Tuple[int, int, int, str]: + ) -> Tuple[int, int, int, int, str, datetime.datetime]: """Extract the fields needed for the final output Args: @@ -96,13 +98,14 @@ class PsList(interfaces.plugins.PluginInterface): tid = task.pid ppid = task.parent.tgid if task.parent else 0 name = utility.array_to_string(task.comm) + start_time = task.get_create_time() if decorate_comm: if task.is_kernel_thread: name = f"[{name}]" elif task.is_user_thread: name = f"{{{name}}}" - task_fields = (task.vol.offset, pid, tid, ppid, name) + task_fields = (task.vol.offset, pid, tid, ppid, name, start_time) return task_fields def _get_file_output(self, task: interfaces.objects.ObjectInterface) -> str: @@ -177,7 +180,9 @@ class PsList(interfaces.plugins.PluginInterface): else: file_output = "Disabled" - offset, pid, tid, ppid, name = self.get_task_fields(task, decorate_comm) + offset, pid, tid, ppid, name, creation_time = self.get_task_fields( + task, decorate_comm + ) yield 0, ( format_hints.Hex(offset), @@ -185,6 +190,7 @@ class PsList(interfaces.plugins.PluginInterface): tid, ppid, name, + creation_time or renderers.NotAvailableValue(), file_output, ) @@ -233,8 +239,23 @@ class PsList(interfaces.plugins.PluginInterface): ("TID", int), ("PPID", int), ("COMM", str), + ("CREATION TIME", datetime.datetime), ("File output", str), ] return renderers.TreeGrid( columns, self._generator(filter_func, include_threads, decorate_comm, dump) ) + + def generate_timeline(self): + pids = self.config.get("pid") + filter_func = self.create_pid_filter(pids) + for task in self.list_tasks( + self.context, self.config["kernel"], filter_func, include_threads=True + ): + offset, user_pid, user_tid, _user_ppid, name, creation_time = ( + self.get_task_fields(task) + ) + + description = f"Process {user_pid}/{user_tid} {name} ({offset})" + + yield (description, timeliner.TimeLinerType.CREATED, creation_time) From d25df2357acf157641a6b30a7a9a30d0261bf147 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 18 Oct 2024 12:59:23 +1100 Subject: [PATCH 494/979] Linux: pslist: Add the boottime plugin --- .../framework/plugins/linux/boottime.py | 98 +++++++++++++++++++ volatility3/framework/plugins/timeliner.py | 13 ++- 2 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 volatility3/framework/plugins/linux/boottime.py diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py new file mode 100644 index 000000000..eeee418f8 --- /dev/null +++ b/volatility3/framework/plugins/linux/boottime.py @@ -0,0 +1,98 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import datetime +from typing import List, Tuple, Iterable + + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.plugins import timeliner +from volatility3.plugins.linux import pslist + + +class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): + """Shows the time the system was started""" + + _required_framework_version = (2, 11, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 3, 0) + ), + ] + + @classmethod + def get_time_namespaces_bootime( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Iterable[Tuple[int, int, int, str, datetime.datetime]]: + """Enumerates tasks' boot times based on their time namespaces. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + pids: Pid list + unique: Filter unique time namespaces + + Yields: + A tuple with the fields to show in the plugin output. + """ + time_namespace_ids = set() + for task in pslist.PsList.list_tasks(context, vmlinux_module_name): + time_namespace_id = task.get_time_namespace_id() + # If it cannot get the time namespace i.e. kernels < 5.6, this still works + # using None to just get the first tasks + if time_namespace_id in time_namespace_ids: + continue + time_namespace_ids.add(time_namespace_id) + boottime = task.get_boottime(root_time_namespace=False) + + fields = ( + time_namespace_id, + boottime, + ) + yield fields + + def _generator(self): + for ( + time_namespace_id, + boottime, + ) in self.get_time_namespaces_bootime( + self.context, + self.config["kernel"], + ): + fields = [ + time_namespace_id or renderers.NotAvailableValue(), + boottime, + ] + yield 0, fields + + def generate_timeline(self): + for ( + time_namespace_id, + boottime, + ) in self.get_time_namespaces_bootime( + self.context, + self.config["kernel"], + ): + description = f"System boot time for time namespace {time_namespace_id}" + + yield description, timeliner.TimeLinerType.BOOTTIME, boottime + + def run(self): + columns = [ + ("TIME NS", int), + ("Boot Time", datetime.datetime), + ] + return renderers.TreeGrid(columns, self._generator()) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index c754e43ef..70da0c4fb 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -23,6 +23,7 @@ class TimeLinerType(enum.IntEnum): MODIFIED = 2 ACCESSED = 3 CHANGED = 4 + BOOTTIME = 5 class TimeLinerInterface(metaclass=abc.ABCMeta): @@ -171,6 +172,10 @@ class Timeliner(interfaces.plugins.PluginInterface): TimeLinerType.CHANGED, renderers.NotApplicableValue(), ), + times.get( + TimeLinerType.BOOTTIME, + renderers.NotApplicableValue(), + ), ], ) ) @@ -178,11 +183,11 @@ class Timeliner(interfaces.plugins.PluginInterface): # Write each entry because the body file doesn't need to be sorted if fp: times = self.timeline[(plugin_name, item)] - # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime + # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime|boottime if self._any_time_present(times): fp.write( - "|{} - {}|0|0|0|0|0|{}|{}|{}|{}\n".format( + "|{} - {}|0|0|0|0|0|{}|{}|{}|{}|{}\n".format( plugin_name, self._sanitize_body_format(item), self._text_format( @@ -197,6 +202,9 @@ class Timeliner(interfaces.plugins.PluginInterface): self._text_format( times.get(TimeLinerType.CREATED, "0") ), + self._text_format( + times.get(TimeLinerType.BOOTTIME, "0") + ), ) ) except Exception as e: @@ -320,6 +328,7 @@ class Timeliner(interfaces.plugins.PluginInterface): ("Modified Date", datetime.datetime), ("Accessed Date", datetime.datetime), ("Changed Date", datetime.datetime), + ("Boot Date", datetime.datetime), ], generator=self._generator(plugins_to_run), ) From 1651ecb1d70bb32868be864c473ef4a346141381 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 18 Oct 2024 13:58:27 +1100 Subject: [PATCH 495/979] linux: boottime api: Fix explicit returns mixed with implicit (fall through) returns --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b3e836102..2784aeda7 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -424,7 +424,7 @@ class task_struct(generic.GenericIntelProcess): time_ns = self.get_time_namespace() if not time_ns: # kernels < 5.6 - return + return None # We are good. ns_common (ns) was introduced in kernels 3.19. So by the time the # time namespace was added in kernels 5.6, it already included the ns member. @@ -437,7 +437,7 @@ class task_struct(generic.GenericIntelProcess): time_ns = self.get_time_namespace() if not time_ns: # kernels < 5.6 - return + return None if not time_ns.has_member("offsets"): # kernels < 5.6 af993f58d69ee9c1f421dfc87c3ed231c113989c From 65daab6cfa986130a33d6c63faab00d5e500534a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 22 Oct 2024 21:47:13 +0100 Subject: [PATCH 496/979] Linux: Fix minor coding style transgression --- volatility3/framework/plugins/linux/ptrace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index e501f3294..e467ee644 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -6,7 +6,7 @@ import logging from typing import List, Iterator from volatility3.framework import renderers, interfaces -from volatility3.framework.constants.architectures import LINUX_ARCHS +from volatility3.framework.constants import architectures from volatility3.framework.objects import utility from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins @@ -27,7 +27,7 @@ class Ptrace(plugins.PluginInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=LINUX_ARCHS, + architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 2, 0) From db8667f88abec074f3165ecfd04219e7ef72fbd1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 23 Oct 2024 08:55:44 +1100 Subject: [PATCH 497/979] Linux - Fix hlist_head object extension by dropping the iterator since it can't determine the correct member_name --- volatility3/framework/symbols/linux/extensions/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 4b1df2a0b..acf78278b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -994,9 +994,6 @@ class hlist_head(objects.StructType, collections.abc.Iterable): current = current.next - def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: - return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) - class files_struct(objects.StructType): def get_fds(self) -> interfaces.objects.ObjectInterface: From 708d9330216853bfebcaa9d5799053656e422e10 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 23 Oct 2024 21:23:14 +0200 Subject: [PATCH 498/979] use additional_locals in run method --- volatility3/cli/volshell/generic.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index c1ca7b5fa..39a7aa126 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -58,7 +58,7 @@ class Volshell(interfaces.plugins.PluginInterface): ] def run( - self, additional_locals: Dict[str, Any] = None + self, additional_locals: Dict[str, Any] = {} ) -> interfaces.renderers.TreeGrid: """Runs the interactive volshell plugin. @@ -94,7 +94,10 @@ class Volshell(interfaces.plugins.PluginInterface): """ sys.ps1 = f"({self.current_layer}) >>> " - self.__console = code.InteractiveConsole(locals=self._construct_locals_dict()) + # Dict self._construct_locals_dict() will have priority on keys + combined_locals = self._construct_locals_dict().copy() + combined_locals.update(additional_locals) + self.__console = code.InteractiveConsole(locals=combined_locals) # Since we have to do work to add the option only once for all different modes of volshell, we can't # rely on the default having been set if self.config.get("script", None) is not None: From 7f2d8eabb7f513148a7868d9a758c10f43d0450b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 23 Oct 2024 21:39:57 +0200 Subject: [PATCH 499/979] set correct locals priority --- volatility3/cli/volshell/generic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 39a7aa126..a0477656e 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -95,8 +95,8 @@ class Volshell(interfaces.plugins.PluginInterface): sys.ps1 = f"({self.current_layer}) >>> " # Dict self._construct_locals_dict() will have priority on keys - combined_locals = self._construct_locals_dict().copy() - combined_locals.update(additional_locals) + combined_locals = additional_locals.copy() + combined_locals.update(self._construct_locals_dict()) self.__console = code.InteractiveConsole(locals=combined_locals) # Since we have to do work to add the option only once for all different modes of volshell, we can't # rely on the default having been set From 388a3e64587478b0a07121a41f2b05a684c4292e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 24 Oct 2024 14:39:22 -0500 Subject: [PATCH 500/979] Linux: Adds missing abc.Iterable implementation `hlist_head` is missing an implementation for `collections.abc.Iterable`, resulting in a crash at runtime. This fixes the issue by providing the required `__iter__` implementation for `hlist_head`. --- volatility3/framework/symbols/linux/extensions/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 14be5ec0a..9bd401db2 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1037,6 +1037,9 @@ class hlist_head(objects.StructType, collections.abc.Iterable): current = current.next + def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: + return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) + class files_struct(objects.StructType): def get_fds(self) -> interfaces.objects.ObjectInterface: From 4c2d2f98da5c6fcbfe8e8aa2efa590438cdf1444 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 24 Oct 2024 15:39:15 -0500 Subject: [PATCH 501/979] Removes collections.abc.Iterable superclass Per @gcmoreira this is the correct fix vs the one previously proposed, since self.vol.member_name cannot be used as the member name to iterate this type. --- volatility3/framework/symbols/linux/extensions/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9bd401db2..eba775714 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1004,7 +1004,7 @@ class list_head(objects.StructType, collections.abc.Iterable): return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) -class hlist_head(objects.StructType, collections.abc.Iterable): +class hlist_head(objects.StructType): def to_list( self, symbol_type: str, @@ -1037,9 +1037,6 @@ class hlist_head(objects.StructType, collections.abc.Iterable): current = current.next - def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: - return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) - class files_struct(objects.StructType): def get_fds(self) -> interfaces.objects.ObjectInterface: From 0413cc5e2634ad4a872c96cf85e0eaacae23d921 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 25 Oct 2024 12:46:32 -0500 Subject: [PATCH 502/979] Windows: Scheduled tasks - code review fixes --- test/__init__.py | 0 test/plugins/__init__.py | 0 test/plugins/windows/__init__.py | 0 test/plugins/windows/test_scheduled_tasks.py | 387 ++++++++++++++++ .../plugins/windows/scheduled_tasks.py | 416 +----------------- 5 files changed, 406 insertions(+), 397 deletions(-) create mode 100644 test/__init__.py create mode 100644 test/plugins/__init__.py create mode 100644 test/plugins/windows/__init__.py create mode 100644 test/plugins/windows/test_scheduled_tasks.py diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/plugins/__init__.py b/test/plugins/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/plugins/windows/__init__.py b/test/plugins/windows/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/plugins/windows/test_scheduled_tasks.py b/test/plugins/windows/test_scheduled_tasks.py new file mode 100644 index 000000000..8f771b323 --- /dev/null +++ b/test/plugins/windows/test_scheduled_tasks.py @@ -0,0 +1,387 @@ +import sys +import struct +import traceback +import unittest +sys.path.insert(0, "../../volatility3") +from volatility3.plugins.windows import scheduled_tasks + +class TestActionsDecoding(unittest.TestCase): + def test_decode_exe_action(self): + # fmt: off + buf = struct.pack( + "512B", + *[ + 0x03, 0x00, 0x16, 0x00, 0x00, 0x00, 0x4c, 0x00, + 0x6f, 0x00, 0x63, 0x00, 0x61, 0x00, 0x6c, 0x00, + 0x53, 0x00, 0x79, 0x00, 0x73, 0x00, 0x74, 0x00, + 0x65, 0x00, 0x6d, 0x00, 0x66, 0x66, 0x00, 0x00, + 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, 0x25, 0x00, + 0x77, 0x00, 0x69, 0x00, 0x6e, 0x00, 0x64, 0x00, + 0x69, 0x00, 0x72, 0x00, 0x25, 0x00, 0x5c, 0x00, + 0x73, 0x00, 0x79, 0x00, 0x73, 0x00, 0x74, 0x00, + 0x65, 0x00, 0x6d, 0x00, 0x33, 0x00, 0x32, 0x00, + 0x5c, 0x00, 0x57, 0x00, 0x69, 0x00, 0x6e, 0x00, + 0x64, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x73, 0x00, + 0x50, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x53, 0x00, 0x68, 0x00, 0x65, 0x00, + 0x6c, 0x00, 0x6c, 0x00, 0x5c, 0x00, 0x76, 0x00, + 0x31, 0x00, 0x2e, 0x00, 0x30, 0x00, 0x5c, 0x00, + 0x70, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x73, 0x00, 0x68, 0x00, 0x65, 0x00, + 0x6c, 0x00, 0x6c, 0x00, 0x2e, 0x00, 0x65, 0x00, + 0x78, 0x00, 0x65, 0x00, 0x62, 0x01, 0x00, 0x00, + 0x2d, 0x00, 0x45, 0x00, 0x78, 0x00, 0x65, 0x00, + 0x63, 0x00, 0x75, 0x00, 0x74, 0x00, 0x69, 0x00, + 0x6f, 0x00, 0x6e, 0x00, 0x50, 0x00, 0x6f, 0x00, + 0x6c, 0x00, 0x69, 0x00, 0x63, 0x00, 0x79, 0x00, + 0x20, 0x00, 0x55, 0x00, 0x6e, 0x00, 0x72, 0x00, + 0x65, 0x00, 0x73, 0x00, 0x74, 0x00, 0x72, 0x00, + 0x69, 0x00, 0x63, 0x00, 0x74, 0x00, 0x65, 0x00, + 0x64, 0x00, 0x20, 0x00, 0x2d, 0x00, 0x4e, 0x00, + 0x6f, 0x00, 0x6e, 0x00, 0x49, 0x00, 0x6e, 0x00, + 0x74, 0x00, 0x65, 0x00, 0x72, 0x00, 0x61, 0x00, + 0x63, 0x00, 0x74, 0x00, 0x69, 0x00, 0x76, 0x00, + 0x65, 0x00, 0x20, 0x00, 0x2d, 0x00, 0x4e, 0x00, + 0x6f, 0x00, 0x50, 0x00, 0x72, 0x00, 0x6f, 0x00, + 0x66, 0x00, 0x69, 0x00, 0x6c, 0x00, 0x65, 0x00, + 0x20, 0x00, 0x2d, 0x00, 0x57, 0x00, 0x69, 0x00, + 0x6e, 0x00, 0x64, 0x00, 0x6f, 0x00, 0x77, 0x00, + 0x53, 0x00, 0x74, 0x00, 0x79, 0x00, 0x6c, 0x00, + 0x65, 0x00, 0x20, 0x00, 0x48, 0x00, 0x69, 0x00, + 0x64, 0x00, 0x64, 0x00, 0x65, 0x00, 0x6e, 0x00, + 0x20, 0x00, 0x22, 0x00, 0x26, 0x00, 0x20, 0x00, + 0x25, 0x00, 0x77, 0x00, 0x69, 0x00, 0x6e, 0x00, + 0x64, 0x00, 0x69, 0x00, 0x72, 0x00, 0x25, 0x00, + 0x5c, 0x00, 0x73, 0x00, 0x79, 0x00, 0x73, 0x00, + 0x74, 0x00, 0x65, 0x00, 0x6d, 0x00, 0x33, 0x00, + 0x32, 0x00, 0x5c, 0x00, 0x57, 0x00, 0x69, 0x00, + 0x6e, 0x00, 0x64, 0x00, 0x6f, 0x00, 0x77, 0x00, + 0x73, 0x00, 0x50, 0x00, 0x6f, 0x00, 0x77, 0x00, + 0x65, 0x00, 0x72, 0x00, 0x53, 0x00, 0x68, 0x00, + 0x65, 0x00, 0x6c, 0x00, 0x6c, 0x00, 0x5c, 0x00, + 0x76, 0x00, 0x31, 0x00, 0x2e, 0x00, 0x30, 0x00, + 0x5c, 0x00, 0x4d, 0x00, 0x6f, 0x00, 0x64, 0x00, + 0x75, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x73, 0x00, + 0x5c, 0x00, 0x53, 0x00, 0x6d, 0x00, 0x62, 0x00, + 0x53, 0x00, 0x68, 0x00, 0x61, 0x00, 0x72, 0x00, + 0x65, 0x00, 0x5c, 0x00, 0x44, 0x00, 0x69, 0x00, + 0x73, 0x00, 0x61, 0x00, 0x62, 0x00, 0x6c, 0x00, + 0x65, 0x00, 0x55, 0x00, 0x6e, 0x00, 0x75, 0x00, + 0x73, 0x00, 0x65, 0x00, 0x64, 0x00, 0x53, 0x00, + 0x6d, 0x00, 0x62, 0x00, 0x31, 0x00, 0x2e, 0x00, + 0x70, 0x00, 0x73, 0x00, 0x31, 0x00, 0x20, 0x00, + 0x2d, 0x00, 0x53, 0x00, 0x63, 0x00, 0x65, 0x00, + 0x6e, 0x00, 0x61, 0x00, 0x72, 0x00, 0x69, 0x00, + 0x6f, 0x00, 0x20, 0x00, 0x43, 0x00, 0x6c, 0x00, + 0x69, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74, 0x00, + 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ] + ) + + try: + actions = scheduled_tasks.ActionSet.decode(buf).actions # type: ignore + self.assertEqual(len(actions), 1) + self.assertEqual(actions[0].action_type, scheduled_tasks.ActionType.Exe) + except Exception: + self.fail( + "ActionDecoder.decode should not raise exception:\n%s" + % traceback.format_exc() + ) + + +class TestTriggersDecoding(unittest.TestCase): + def test_decode_all_triggers(self): + """ + Tests decoding a set of all triggers that can be constructed via the + Task Scheduler GUI interface. Ensures that the correct number of bytes + is being consumed for each trigger structure. + """ + buf = struct.pack( + "1808B", + # fmt: off + *[ + 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x38, 0x21, 0x41, 0x42, 0x48, 0x48, 0x48, 0x48, + 0xa0, 0x12, 0xa0, 0xa4, 0x48, 0x48, 0x48, 0x48, + 0x0e, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x41, 0x00, 0x75, 0x00, 0x74, 0x00, 0x68, 0x00, + 0x6f, 0x00, 0x72, 0x00, 0x00, 0x00, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x1c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x15, 0x00, 0x00, 0x00, 0x69, 0xce, 0x28, 0x2a, + 0xce, 0xd8, 0x1f, 0x77, 0x37, 0x9c, 0xe2, 0x44, + 0xf4, 0x01, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x40, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x44, 0x00, 0x45, 0x00, 0x53, 0x00, 0x4b, 0x00, + 0x54, 0x00, 0x4f, 0x00, 0x50, 0x00, 0x2d, 0x00, + 0x45, 0x00, 0x33, 0x00, 0x38, 0x00, 0x38, 0x00, + 0x44, 0x00, 0x38, 0x00, 0x50, 0x00, 0x5c, 0x00, + 0x41, 0x00, 0x64, 0x00, 0x6d, 0x00, 0x69, 0x00, + 0x6e, 0x00, 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, + 0x72, 0x00, 0x61, 0x00, 0x74, 0x00, 0x6f, 0x00, + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x2c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x80, 0xf4, 0x03, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0xdd, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x07, 0x0a, 0x00, 0x00, 0x00, 0x09, 0x00, + 0x80, 0x48, 0x11, 0xf8, 0x36, 0x1a, 0xdb, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x2e, 0xe2, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x31, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0xaa, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0xee, 0xee, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0xcc, 0xcc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x65, 0x00, 0x78, 0x00, 0x65, 0x00, + 0x22, 0x00, 0x20, 0x00, 0x53, 0x00, 0x74, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x3c, 0x00, 0x51, 0x00, 0x75, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x79, 0x00, 0x4c, 0x00, 0x69, 0x00, + 0x73, 0x00, 0x74, 0x00, 0x3e, 0x00, 0x3c, 0x00, + 0x51, 0x00, 0x75, 0x00, 0x65, 0x00, 0x72, 0x00, + 0x79, 0x00, 0x20, 0x00, 0x49, 0x00, 0x64, 0x00, + 0x3d, 0x00, 0x22, 0x00, 0x30, 0x00, 0x22, 0x00, + 0x20, 0x00, 0x50, 0x00, 0x61, 0x00, 0x74, 0x00, + 0x68, 0x00, 0x3d, 0x00, 0x22, 0x00, 0x49, 0x00, + 0x6e, 0x00, 0x74, 0x00, 0x65, 0x00, 0x72, 0x00, + 0x6e, 0x00, 0x65, 0x00, 0x74, 0x00, 0x20, 0x00, + 0x45, 0x00, 0x78, 0x00, 0x70, 0x00, 0x6c, 0x00, + 0x6f, 0x00, 0x72, 0x00, 0x65, 0x00, 0x72, 0x00, + 0x22, 0x00, 0x3e, 0x00, 0x3c, 0x00, 0x53, 0x00, + 0x65, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x63, 0x00, + 0x74, 0x00, 0x20, 0x00, 0x50, 0x00, 0x61, 0x00, + 0x74, 0x00, 0x68, 0x00, 0x3d, 0x00, 0x22, 0x00, + 0x49, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x6e, 0x00, 0x65, 0x00, 0x74, 0x00, + 0x20, 0x00, 0x45, 0x00, 0x78, 0x00, 0x70, 0x00, + 0x6c, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x22, 0x00, 0x3e, 0x00, 0x2a, 0x00, + 0x5b, 0x00, 0x53, 0x00, 0x79, 0x00, 0x73, 0x00, + 0x74, 0x00, 0x65, 0x00, 0x6d, 0x00, 0x5b, 0x00, + 0x45, 0x00, 0x76, 0x00, 0x65, 0x00, 0x6e, 0x00, + 0x74, 0x00, 0x49, 0x00, 0x44, 0x00, 0x3d, 0x00, + 0x32, 0x00, 0x5d, 0x00, 0x5d, 0x00, 0x3c, 0x00, + 0x2f, 0x00, 0x53, 0x00, 0x65, 0x00, 0x6c, 0x00, + 0x65, 0x00, 0x63, 0x00, 0x74, 0x00, 0x3e, 0x00, + 0x3c, 0x00, 0x2f, 0x00, 0x51, 0x00, 0x75, 0x00, + 0x65, 0x00, 0x72, 0x00, 0x79, 0x00, 0x3e, 0x00, + 0x3c, 0x00, 0x2f, 0x00, 0x51, 0x00, 0x75, 0x00, + 0x65, 0x00, 0x72, 0x00, 0x79, 0x00, 0x4c, 0x00, + 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, 0x3e, 0x00, + 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x88, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x1c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x15, 0x00, 0x00, 0x00, 0x69, 0xce, 0x28, 0x2a, + 0xce, 0xd8, 0x1f, 0x77, 0x37, 0x9c, 0xe2, 0x44, + 0xf4, 0x01, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x40, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x44, 0x00, 0x45, 0x00, 0x53, 0x00, 0x4b, 0x00, + 0x54, 0x00, 0x4f, 0x00, 0x50, 0x00, 0x2d, 0x00, + 0x45, 0x00, 0x33, 0x00, 0x38, 0x00, 0x38, 0x00, + 0x44, 0x00, 0x38, 0x00, 0x50, 0x00, 0x5c, 0x00, + 0x41, 0x00, 0x64, 0x00, 0x6d, 0x00, 0x69, 0x00, + 0x6e, 0x00, 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, + 0x72, 0x00, 0x61, 0x00, 0x74, 0x00, 0x6f, 0x00, + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x1c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x15, 0x00, 0x00, 0x00, 0x69, 0xce, 0x28, 0x2a, + 0xce, 0xd8, 0x1f, 0x77, 0x37, 0x9c, 0xe2, 0x44, + 0xf4, 0x01, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x40, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x44, 0x00, 0x45, 0x00, 0x53, 0x00, 0x4b, 0x00, + 0x54, 0x00, 0x4f, 0x00, 0x50, 0x00, 0x2d, 0x00, + 0x45, 0x00, 0x33, 0x00, 0x38, 0x00, 0x38, 0x00, + 0x44, 0x00, 0x38, 0x00, 0x50, 0x00, 0x5c, 0x00, + 0x41, 0x00, 0x64, 0x00, 0x6d, 0x00, 0x69, 0x00, + 0x6e, 0x00, 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, + 0x72, 0x00, 0x61, 0x00, 0x74, 0x00, 0x6f, 0x00, + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ] + # fmt: on + ) + triggers = scheduled_tasks.TriggerSet.decode(buf) + self.assertIsNotNone(triggers) + + def test_decode_triggers(self): + # fmt: off + buf = struct.pack( + "320B", + *[ + 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x08, 0xA1, 0x40, 0x42, 0x48, 0x48, 0x48, 0x48, + 0x7A, 0x7F, 0x59, 0xDC, 0x48, 0x48, 0x48, 0x48, + 0x22, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x49, 0x00, 0x6E, 0x00, 0x74, 0x00, 0x65, 0x00, + 0x72, 0x00, 0x61, 0x00, 0x63, 0x00, 0x74, 0x00, + 0x69, 0x00, 0x76, 0x00, 0x65, 0x00, 0x55, 0x00, + 0x73, 0x00, 0x65, 0x00, 0x72, 0x00, 0x73, 0x00, + 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + 0x05, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x0C, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x04, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x2C, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0x80, 0x51, 0x01, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0xAA, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2C, 0x01, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xC1, 0xD9, 0x04, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, + 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, + ] + ) + # fmt: on + triggers = scheduled_tasks.TriggerSet.decode(buf) + self.assertIsNotNone(triggers) + if not triggers: + return + self.assertGreater(len(triggers.triggers), 0) diff --git a/volatility3/framework/plugins/windows/scheduled_tasks.py b/volatility3/framework/plugins/windows/scheduled_tasks.py index 6d3ee586d..374d2d47a 100644 --- a/volatility3/framework/plugins/windows/scheduled_tasks.py +++ b/volatility3/framework/plugins/windows/scheduled_tasks.py @@ -11,8 +11,6 @@ import io import itertools import logging import struct -import traceback -import unittest from typing import Dict, Iterator, List, Optional, Tuple, Union from volatility3.framework import exceptions, interfaces, renderers @@ -52,6 +50,16 @@ class TimeMode(enum.Enum): Unknown = "Unknown" +TIME_MODE_DESCRIPTION = { + TimeMode.Once: "Once", + TimeMode.Daily: "Daily", + TimeMode.Weekly: "Weekly", + TimeMode.DaysInMonths: "Days In Months", + TimeMode.DaysInWeeksInMonths: "Days In Weeks In Months", + TimeMode.Unknown: "Unknown", +} + + class ActionType(enum.Enum): """ Enumeration that maps action types to their magic number encodings @@ -193,14 +201,9 @@ class _ScheduledTasksReader(io.BytesIO): filetime = self.decode_filetime() if filetime is None: return None + return is_localized, filetime - def read_filetime(self) -> Optional[datetime.datetime]: - return datetime.datetime.now() - - def seek_relative(self, offset) -> int: - return self.seek(offset, io.SEEK_CUR) - def read_bool(self, aligned=False) -> Optional[bool]: try: val = struct.unpack("?", self.read(1))[0] @@ -561,13 +564,14 @@ class _JobSchedule: repetition_duration_secs = reader.read_u4() execution_time_limit_secs = reader.read_u4() mode_index = reader.read_u4() - try: - if mode_index is not None: - mode = list(TimeMode)[mode_index] - else: - mode = None - except IndexError: - mode = TimeMode.Unknown + if mode_index is not None: + try: + mode = TimeMode(mode_index) + except ValueError: + mode = TimeMode.Unknown + else: + mode = None + data1 = reader.read_u2() data2 = reader.read_u2() data3 = reader.read_u2() @@ -1412,385 +1416,3 @@ class ScheduledTasks(interfaces.plugins.PluginInterface, timeliner.TimeLinerInte for indent, entry in self._generator() ), ) - - -class TestActionsDecoding(unittest.TestCase): - def test_decode_exe_action(self): - # fmt: off - buf = struct.pack( - "512B", - *[ - 0x03, 0x00, 0x16, 0x00, 0x00, 0x00, 0x4c, 0x00, - 0x6f, 0x00, 0x63, 0x00, 0x61, 0x00, 0x6c, 0x00, - 0x53, 0x00, 0x79, 0x00, 0x73, 0x00, 0x74, 0x00, - 0x65, 0x00, 0x6d, 0x00, 0x66, 0x66, 0x00, 0x00, - 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, 0x25, 0x00, - 0x77, 0x00, 0x69, 0x00, 0x6e, 0x00, 0x64, 0x00, - 0x69, 0x00, 0x72, 0x00, 0x25, 0x00, 0x5c, 0x00, - 0x73, 0x00, 0x79, 0x00, 0x73, 0x00, 0x74, 0x00, - 0x65, 0x00, 0x6d, 0x00, 0x33, 0x00, 0x32, 0x00, - 0x5c, 0x00, 0x57, 0x00, 0x69, 0x00, 0x6e, 0x00, - 0x64, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x73, 0x00, - 0x50, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x65, 0x00, - 0x72, 0x00, 0x53, 0x00, 0x68, 0x00, 0x65, 0x00, - 0x6c, 0x00, 0x6c, 0x00, 0x5c, 0x00, 0x76, 0x00, - 0x31, 0x00, 0x2e, 0x00, 0x30, 0x00, 0x5c, 0x00, - 0x70, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x65, 0x00, - 0x72, 0x00, 0x73, 0x00, 0x68, 0x00, 0x65, 0x00, - 0x6c, 0x00, 0x6c, 0x00, 0x2e, 0x00, 0x65, 0x00, - 0x78, 0x00, 0x65, 0x00, 0x62, 0x01, 0x00, 0x00, - 0x2d, 0x00, 0x45, 0x00, 0x78, 0x00, 0x65, 0x00, - 0x63, 0x00, 0x75, 0x00, 0x74, 0x00, 0x69, 0x00, - 0x6f, 0x00, 0x6e, 0x00, 0x50, 0x00, 0x6f, 0x00, - 0x6c, 0x00, 0x69, 0x00, 0x63, 0x00, 0x79, 0x00, - 0x20, 0x00, 0x55, 0x00, 0x6e, 0x00, 0x72, 0x00, - 0x65, 0x00, 0x73, 0x00, 0x74, 0x00, 0x72, 0x00, - 0x69, 0x00, 0x63, 0x00, 0x74, 0x00, 0x65, 0x00, - 0x64, 0x00, 0x20, 0x00, 0x2d, 0x00, 0x4e, 0x00, - 0x6f, 0x00, 0x6e, 0x00, 0x49, 0x00, 0x6e, 0x00, - 0x74, 0x00, 0x65, 0x00, 0x72, 0x00, 0x61, 0x00, - 0x63, 0x00, 0x74, 0x00, 0x69, 0x00, 0x76, 0x00, - 0x65, 0x00, 0x20, 0x00, 0x2d, 0x00, 0x4e, 0x00, - 0x6f, 0x00, 0x50, 0x00, 0x72, 0x00, 0x6f, 0x00, - 0x66, 0x00, 0x69, 0x00, 0x6c, 0x00, 0x65, 0x00, - 0x20, 0x00, 0x2d, 0x00, 0x57, 0x00, 0x69, 0x00, - 0x6e, 0x00, 0x64, 0x00, 0x6f, 0x00, 0x77, 0x00, - 0x53, 0x00, 0x74, 0x00, 0x79, 0x00, 0x6c, 0x00, - 0x65, 0x00, 0x20, 0x00, 0x48, 0x00, 0x69, 0x00, - 0x64, 0x00, 0x64, 0x00, 0x65, 0x00, 0x6e, 0x00, - 0x20, 0x00, 0x22, 0x00, 0x26, 0x00, 0x20, 0x00, - 0x25, 0x00, 0x77, 0x00, 0x69, 0x00, 0x6e, 0x00, - 0x64, 0x00, 0x69, 0x00, 0x72, 0x00, 0x25, 0x00, - 0x5c, 0x00, 0x73, 0x00, 0x79, 0x00, 0x73, 0x00, - 0x74, 0x00, 0x65, 0x00, 0x6d, 0x00, 0x33, 0x00, - 0x32, 0x00, 0x5c, 0x00, 0x57, 0x00, 0x69, 0x00, - 0x6e, 0x00, 0x64, 0x00, 0x6f, 0x00, 0x77, 0x00, - 0x73, 0x00, 0x50, 0x00, 0x6f, 0x00, 0x77, 0x00, - 0x65, 0x00, 0x72, 0x00, 0x53, 0x00, 0x68, 0x00, - 0x65, 0x00, 0x6c, 0x00, 0x6c, 0x00, 0x5c, 0x00, - 0x76, 0x00, 0x31, 0x00, 0x2e, 0x00, 0x30, 0x00, - 0x5c, 0x00, 0x4d, 0x00, 0x6f, 0x00, 0x64, 0x00, - 0x75, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x73, 0x00, - 0x5c, 0x00, 0x53, 0x00, 0x6d, 0x00, 0x62, 0x00, - 0x53, 0x00, 0x68, 0x00, 0x61, 0x00, 0x72, 0x00, - 0x65, 0x00, 0x5c, 0x00, 0x44, 0x00, 0x69, 0x00, - 0x73, 0x00, 0x61, 0x00, 0x62, 0x00, 0x6c, 0x00, - 0x65, 0x00, 0x55, 0x00, 0x6e, 0x00, 0x75, 0x00, - 0x73, 0x00, 0x65, 0x00, 0x64, 0x00, 0x53, 0x00, - 0x6d, 0x00, 0x62, 0x00, 0x31, 0x00, 0x2e, 0x00, - 0x70, 0x00, 0x73, 0x00, 0x31, 0x00, 0x20, 0x00, - 0x2d, 0x00, 0x53, 0x00, 0x63, 0x00, 0x65, 0x00, - 0x6e, 0x00, 0x61, 0x00, 0x72, 0x00, 0x69, 0x00, - 0x6f, 0x00, 0x20, 0x00, 0x43, 0x00, 0x6c, 0x00, - 0x69, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74, 0x00, - 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - ] - ) - - try: - actions = ActionSet.decode(buf) # type: ignore - self.assertEqual(len(actions), 1) - self.assertEqual(actions[0].action_type, ActionType.Exe) - except Exception: - self.fail( - "ActionDecoder.decode should not raise exception:\n%s" - % traceback.format_exc() - ) - - -class TestTriggersDecoding(unittest.TestCase): - def test_decode_all_triggers(self): - """ - Tests decoding a set of all triggers that can be constructed via the - Task Scheduler GUI interface. Ensures that the correct number of bytes - is being consumed for each trigger structure. - """ - buf = struct.pack( - "1808B", - # fmt: off - *[ - 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x38, 0x21, 0x41, 0x42, 0x48, 0x48, 0x48, 0x48, - 0xa0, 0x12, 0xa0, 0xa4, 0x48, 0x48, 0x48, 0x48, - 0x0e, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x41, 0x00, 0x75, 0x00, 0x74, 0x00, 0x68, 0x00, - 0x6f, 0x00, 0x72, 0x00, 0x00, 0x00, 0x48, 0x48, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x01, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x1c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - 0x15, 0x00, 0x00, 0x00, 0x69, 0xce, 0x28, 0x2a, - 0xce, 0xd8, 0x1f, 0x77, 0x37, 0x9c, 0xe2, 0x44, - 0xf4, 0x01, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x40, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x44, 0x00, 0x45, 0x00, 0x53, 0x00, 0x4b, 0x00, - 0x54, 0x00, 0x4f, 0x00, 0x50, 0x00, 0x2d, 0x00, - 0x45, 0x00, 0x33, 0x00, 0x38, 0x00, 0x38, 0x00, - 0x44, 0x00, 0x38, 0x00, 0x50, 0x00, 0x5c, 0x00, - 0x41, 0x00, 0x64, 0x00, 0x6d, 0x00, 0x69, 0x00, - 0x6e, 0x00, 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, - 0x72, 0x00, 0x61, 0x00, 0x74, 0x00, 0x6f, 0x00, - 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x80, 0xf4, 0x03, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0xdd, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x07, 0x0a, 0x00, 0x00, 0x00, 0x09, 0x00, - 0x80, 0x48, 0x11, 0xf8, 0x36, 0x1a, 0xdb, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x01, 0x2e, 0xe2, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc2, 0x31, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0xaa, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0xee, 0xee, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xda, 0xaf, 0x8d, 0x09, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0xcc, 0xcc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x65, 0x00, 0x78, 0x00, 0x65, 0x00, - 0x22, 0x00, 0x20, 0x00, 0x53, 0x00, 0x74, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3c, 0x00, 0x51, 0x00, 0x75, 0x00, 0x65, 0x00, - 0x72, 0x00, 0x79, 0x00, 0x4c, 0x00, 0x69, 0x00, - 0x73, 0x00, 0x74, 0x00, 0x3e, 0x00, 0x3c, 0x00, - 0x51, 0x00, 0x75, 0x00, 0x65, 0x00, 0x72, 0x00, - 0x79, 0x00, 0x20, 0x00, 0x49, 0x00, 0x64, 0x00, - 0x3d, 0x00, 0x22, 0x00, 0x30, 0x00, 0x22, 0x00, - 0x20, 0x00, 0x50, 0x00, 0x61, 0x00, 0x74, 0x00, - 0x68, 0x00, 0x3d, 0x00, 0x22, 0x00, 0x49, 0x00, - 0x6e, 0x00, 0x74, 0x00, 0x65, 0x00, 0x72, 0x00, - 0x6e, 0x00, 0x65, 0x00, 0x74, 0x00, 0x20, 0x00, - 0x45, 0x00, 0x78, 0x00, 0x70, 0x00, 0x6c, 0x00, - 0x6f, 0x00, 0x72, 0x00, 0x65, 0x00, 0x72, 0x00, - 0x22, 0x00, 0x3e, 0x00, 0x3c, 0x00, 0x53, 0x00, - 0x65, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x63, 0x00, - 0x74, 0x00, 0x20, 0x00, 0x50, 0x00, 0x61, 0x00, - 0x74, 0x00, 0x68, 0x00, 0x3d, 0x00, 0x22, 0x00, - 0x49, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x65, 0x00, - 0x72, 0x00, 0x6e, 0x00, 0x65, 0x00, 0x74, 0x00, - 0x20, 0x00, 0x45, 0x00, 0x78, 0x00, 0x70, 0x00, - 0x6c, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x65, 0x00, - 0x72, 0x00, 0x22, 0x00, 0x3e, 0x00, 0x2a, 0x00, - 0x5b, 0x00, 0x53, 0x00, 0x79, 0x00, 0x73, 0x00, - 0x74, 0x00, 0x65, 0x00, 0x6d, 0x00, 0x5b, 0x00, - 0x45, 0x00, 0x76, 0x00, 0x65, 0x00, 0x6e, 0x00, - 0x74, 0x00, 0x49, 0x00, 0x44, 0x00, 0x3d, 0x00, - 0x32, 0x00, 0x5d, 0x00, 0x5d, 0x00, 0x3c, 0x00, - 0x2f, 0x00, 0x53, 0x00, 0x65, 0x00, 0x6c, 0x00, - 0x65, 0x00, 0x63, 0x00, 0x74, 0x00, 0x3e, 0x00, - 0x3c, 0x00, 0x2f, 0x00, 0x51, 0x00, 0x75, 0x00, - 0x65, 0x00, 0x72, 0x00, 0x79, 0x00, 0x3e, 0x00, - 0x3c, 0x00, 0x2f, 0x00, 0x51, 0x00, 0x75, 0x00, - 0x65, 0x00, 0x72, 0x00, 0x79, 0x00, 0x4c, 0x00, - 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, 0x3e, 0x00, - 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x88, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x01, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x01, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x1c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - 0x15, 0x00, 0x00, 0x00, 0x69, 0xce, 0x28, 0x2a, - 0xce, 0xd8, 0x1f, 0x77, 0x37, 0x9c, 0xe2, 0x44, - 0xf4, 0x01, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x40, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x44, 0x00, 0x45, 0x00, 0x53, 0x00, 0x4b, 0x00, - 0x54, 0x00, 0x4f, 0x00, 0x50, 0x00, 0x2d, 0x00, - 0x45, 0x00, 0x33, 0x00, 0x38, 0x00, 0x38, 0x00, - 0x44, 0x00, 0x38, 0x00, 0x50, 0x00, 0x5c, 0x00, - 0x41, 0x00, 0x64, 0x00, 0x6d, 0x00, 0x69, 0x00, - 0x6e, 0x00, 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, - 0x72, 0x00, 0x61, 0x00, 0x74, 0x00, 0x6f, 0x00, - 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x01, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x77, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x01, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x01, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x1c, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - 0x15, 0x00, 0x00, 0x00, 0x69, 0xce, 0x28, 0x2a, - 0xce, 0xd8, 0x1f, 0x77, 0x37, 0x9c, 0xe2, 0x44, - 0xf4, 0x01, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x40, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x44, 0x00, 0x45, 0x00, 0x53, 0x00, 0x4b, 0x00, - 0x54, 0x00, 0x4f, 0x00, 0x50, 0x00, 0x2d, 0x00, - 0x45, 0x00, 0x33, 0x00, 0x38, 0x00, 0x38, 0x00, - 0x44, 0x00, 0x38, 0x00, 0x50, 0x00, 0x5c, 0x00, - 0x41, 0x00, 0x64, 0x00, 0x6d, 0x00, 0x69, 0x00, - 0x6e, 0x00, 0x69, 0x00, 0x73, 0x00, 0x74, 0x00, - 0x72, 0x00, 0x61, 0x00, 0x74, 0x00, 0x6f, 0x00, - 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 - ] - # fmt: on - ) - triggers = TriggerSet.decode(buf) - self.assertIsNotNone(triggers) - - def test_decode_triggers(self): - # fmt: off - buf = struct.pack( - "320B", - *[ - 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x08, 0xA1, 0x40, 0x42, 0x48, 0x48, 0x48, 0x48, - 0x7A, 0x7F, 0x59, 0xDC, 0x48, 0x48, 0x48, 0x48, - 0x22, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x49, 0x00, 0x6E, 0x00, 0x74, 0x00, 0x65, 0x00, - 0x72, 0x00, 0x61, 0x00, 0x63, 0x00, 0x74, 0x00, - 0x69, 0x00, 0x76, 0x00, 0x65, 0x00, 0x55, 0x00, - 0x73, 0x00, 0x65, 0x00, 0x72, 0x00, 0x73, 0x00, - 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x00, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - 0x05, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x0C, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - 0x04, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x2C, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, - 0x80, 0x51, 0x01, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, - 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0xAA, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xB9, 0x61, 0x1A, 0xA8, 0xB9, 0x61, 0x1A, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x2C, 0x01, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0xC1, 0xD9, 0x04, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0x48, 0x48, 0x48, - 0x01, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, 0x48, - ] - ) - # fmt: on - triggers = TriggerSet.decode(buf) - self.assertIsNotNone(triggers) - if not triggers: - return - self.assertGreater(len(triggers.triggers), 0) From efdbc27b95e141fe739b8f7e8edd1357c61b4601 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 25 Oct 2024 14:06:41 -0500 Subject: [PATCH 503/979] Windows: Scheduled tasks - removes special time type Removes the TaskSchedulerTime dataclass and discards the is_localized value parsed with the timestamp. --- .../plugins/windows/scheduled_tasks.py | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/windows/scheduled_tasks.py b/volatility3/framework/plugins/windows/scheduled_tasks.py index 374d2d47a..277a0d856 100644 --- a/volatility3/framework/plugins/windows/scheduled_tasks.py +++ b/volatility3/framework/plugins/windows/scheduled_tasks.py @@ -137,16 +137,6 @@ class SidType(enum.Enum): LogonSession = 11 -@dataclasses.dataclass -class TaskSchedulerTime: - """ - A class containing datetime information about when a task will run - """ - - is_localized: bool - filetime: Optional[datetime.datetime] - - @dataclasses.dataclass class TaskSchedulerTimePeriod: """ @@ -196,13 +186,13 @@ NULL = "\u0000" class _ScheduledTasksReader(io.BytesIO): - def read_task_scheduler_time(self) -> Optional[Tuple[bool, datetime.datetime]]: - is_localized = bool(self.read_aligned_u1()) + def read_task_scheduler_time(self) -> Optional[datetime.datetime]: + _ = bool(self.read_aligned_u1()) # is_localized filetime = self.decode_filetime() if filetime is None: return None - return is_localized, filetime + return filetime def read_bool(self, aligned=False) -> Optional[bool]: try: @@ -584,8 +574,8 @@ class _JobSchedule: reader.seek(4, io.SEEK_CUR) # pad return cls( - start_boundary[1] if start_boundary is not None else None, - end_boundary[1] if end_boundary is not None else None, + start_boundary, + end_boundary, repetition_interval_secs, repetition_duration_secs, execution_time_limit_secs, @@ -833,8 +823,8 @@ class TaskTrigger: reader.seek((8 - (reader.tell() - cur)) % 8, io.SEEK_CUR) # pad to block return cls( - start_boundary[1] if start_boundary is not None else None, - end_boundary[1] if end_boundary is not None else None, + start_boundary, + end_boundary, repetition_interval_secs, trigger_enabled, trigger_type, From 094cdf18f14d795f385e504506390ba1602c0b55 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 10:59:18 +1100 Subject: [PATCH 504/979] Linux: netfilter plugin: Fix hooked field to match vol2 output --- volatility3/framework/plugins/linux/netfilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index f576c073c..66075f907 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -190,7 +190,7 @@ class AbstractNetfilter(ABC): priority = int(hook_ops.priority) hook_ops_hook = hook_ops.hook module_name = self.get_module_name_for_address(hook_ops_hook) - hooked = module_name is not None + hooked = module_name is None yield netns, proto_name, hook_name, priority, hook_ops_hook, module_name, hooked From 4895af47bdec7779c7806dca47c657ae686a6e0b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 13:46:19 +1100 Subject: [PATCH 505/979] Linux: Boottime timeliner: Rollback timeliner event type changes and use the created time for the boot time plugin --- volatility3/framework/plugins/linux/boottime.py | 2 +- volatility3/framework/plugins/timeliner.py | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py index eeee418f8..8f63ee7f8 100644 --- a/volatility3/framework/plugins/linux/boottime.py +++ b/volatility3/framework/plugins/linux/boottime.py @@ -88,7 +88,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ): description = f"System boot time for time namespace {time_namespace_id}" - yield description, timeliner.TimeLinerType.BOOTTIME, boottime + yield description, timeliner.TimeLinerType.CREATED, boottime def run(self): columns = [ diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 70da0c4fb..c754e43ef 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -23,7 +23,6 @@ class TimeLinerType(enum.IntEnum): MODIFIED = 2 ACCESSED = 3 CHANGED = 4 - BOOTTIME = 5 class TimeLinerInterface(metaclass=abc.ABCMeta): @@ -172,10 +171,6 @@ class Timeliner(interfaces.plugins.PluginInterface): TimeLinerType.CHANGED, renderers.NotApplicableValue(), ), - times.get( - TimeLinerType.BOOTTIME, - renderers.NotApplicableValue(), - ), ], ) ) @@ -183,11 +178,11 @@ class Timeliner(interfaces.plugins.PluginInterface): # Write each entry because the body file doesn't need to be sorted if fp: times = self.timeline[(plugin_name, item)] - # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime|boottime + # Body format is: MD5|name|inode|mode_as_string|UID|GID|size|atime|mtime|ctime|crtime if self._any_time_present(times): fp.write( - "|{} - {}|0|0|0|0|0|{}|{}|{}|{}|{}\n".format( + "|{} - {}|0|0|0|0|0|{}|{}|{}|{}\n".format( plugin_name, self._sanitize_body_format(item), self._text_format( @@ -202,9 +197,6 @@ class Timeliner(interfaces.plugins.PluginInterface): self._text_format( times.get(TimeLinerType.CREATED, "0") ), - self._text_format( - times.get(TimeLinerType.BOOTTIME, "0") - ), ) ) except Exception as e: @@ -328,7 +320,6 @@ class Timeliner(interfaces.plugins.PluginInterface): ("Modified Date", datetime.datetime), ("Accessed Date", datetime.datetime), ("Changed Date", datetime.datetime), - ("Boot Date", datetime.datetime), ], generator=self._generator(plugins_to_run), ) From c4274c942c4e94e909de44bbe3e12b47a15143f2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 13:49:58 +1100 Subject: [PATCH 506/979] Linux: Fix exception message in TimespecVol3::__sub__() --- volatility3/framework/symbols/linux/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 95b476263..8c28ca562 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -903,7 +903,7 @@ class TimespecVol3(object): """Returns a new TimespecVol3 object that subtracts the values in the timespec argument from the current object's values""" if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): - raise TypeError("Cannot add a TimespecVol3 to this object") + raise TypeError("Cannot substract this object to a TimespecVol3") result = TimespecVol3( tv_sec=self.tv_sec - timespec.tv_sec, From 57ffd5b939df456b7e573d91d6b23bc01f36f353 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 14:24:16 +1100 Subject: [PATCH 507/979] Linux: Boottime API: Refactor TimespecVol3::negate() to return a new object instead of modifying the original. It also normalizes its values, aligning with the behavior of the other addition and subtraction operators --- volatility3/framework/symbols/linux/__init__.py | 14 +++++++++++--- .../framework/symbols/linux/extensions/__init__.py | 10 ++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 8c28ca562..4123674ed 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -909,6 +909,7 @@ class TimespecVol3(object): tv_sec=self.tv_sec - timespec.tv_sec, tv_nsec=self.tv_nsec - timespec.tv_nsec, ) + result.normalize() return result @@ -925,6 +926,13 @@ class TimespecVol3(object): self.tv_sec -= 1 def negate(self): - """Negates the sign of both tv_sec and tv_nsec""" - self.tv_sec = -self.tv_sec - self.tv_nsec = -self.tv_nsec + """Returns a new TimespecVol3 object with the values of the current object negated""" + + result = TimespecVol3( + tv_sec=-self.tv_sec, + tv_nsec=-self.tv_nsec, + ) + + result.normalize() + + return result diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 2784aeda7..6fd55cb56 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -502,10 +502,7 @@ class task_struct(generic.GenericIntelProcess): boottime += timekeeper.total_sleep_time - boottime.negate() - boottime.normalize() - - return boottime + return boottime.negate() elif vmlinux.has_symbol("wall_to_monotonic"): # kernels < 3.4 - Tested on Debian7 3.2.0-4 (3.2.57-3+deb7u2) @@ -523,10 +520,7 @@ class task_struct(generic.GenericIntelProcess): # kernels < 2.6.32 total_sleep_time is an unsigned long as seconds boottime.tv_sec += total_sleep_time - boottime.negate() - boottime.normalize() - - return boottime + return boottime.negate() raise exceptions.VolatilityException("Unsupported") From bee2a398eb4d3f3b28644ed2039bb516ccc3e75e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 15:13:26 +1100 Subject: [PATCH 508/979] Linux: Minor: Add comment/header on each set of constants --- volatility3/framework/constants/linux/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 056c384ae..7bfa9268d 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -304,6 +304,7 @@ class ELF_CLASS(IntEnum): ELFCLASS64 = 2 +# PTrace PT_OPT_FLAG_SHIFT = 3 PTRACE_EVENT_FORK = 1 @@ -341,4 +342,5 @@ class PT_FLAGS(Flag): return str(self).replace(self.__class__.__name__ + ".", "") +# Boot time NSEC_PER_SEC = 1e9 From 526007f948b4b1fcefacb84e344d0b6ff49e945b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 16:41:47 +1100 Subject: [PATCH 509/979] Linux: hidden_modules: Use child_template --- volatility3/framework/plugins/linux/hidden_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index 679f2b00a..09b8851ec 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -167,7 +167,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): mkobj_offset = vmlinux.get_type("module").relative_child_offset("mkobj") mod_offset = vmlinux.get_type("module_kobject").relative_child_offset("mod") offset_to_mkobj_mod = mkobj_offset + mod_offset - mod_member_template = vmlinux.get_type("module_kobject").vol.members["mod"][1] + mod_member_template = vmlinux.get_type("module_kobject").child_template("mod") mod_size = mod_member_template.size mod_member_data_format = mod_member_template.data_format From cbe071ff3ef90c4c2ae78bc953390bd69deb7c02 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 17:19:30 +1100 Subject: [PATCH 510/979] Linux: hidden_modules: Import the whole architectures module --- volatility3/framework/plugins/linux/hidden_modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index 09b8851ec..b7513a952 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -4,7 +4,7 @@ import logging from typing import List, Set, Tuple, Iterable from volatility3.framework import renderers, interfaces, exceptions, objects -from volatility3.framework.constants.architectures import LINUX_ARCHS +from volatility3.framework.constants import architectures from volatility3.framework.renderers import format_hints from volatility3.framework.configuration import requirements from volatility3.plugins.linux import lsmod @@ -25,7 +25,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=LINUX_ARCHS, + architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) From dfd8a1f19565ddfdd9521206bf4fb9773ba52578 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 17:22:48 +1100 Subject: [PATCH 511/979] Linux: hidden_modules: Include kernel version and commit details --- volatility3/framework/plugins/linux/hidden_modules.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index b7513a952..7fd5f9fad 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -48,10 +48,12 @@ class Hidden_modules(interfaces.plugins.PluginInterface): """ vmlinux = context.modules[vmlinux_module_name] if vmlinux.has_symbol("mod_tree"): + # Kernel >= 5.19 58d208de3e8d87dbe196caf0b57cc58c7a3836ca mod_tree = vmlinux.object_from_symbol("mod_tree") modules_addr_min = mod_tree.addr_min modules_addr_max = mod_tree.addr_max elif vmlinux.has_symbol("module_addr_min"): + # 2.6.27 <= kernel < 5.19 3a642e99babe0617febb6f402e1e063479f489db modules_addr_min = vmlinux.object_from_symbol("module_addr_min") modules_addr_max = vmlinux.object_from_symbol("module_addr_max") From 1c6a5480170d38760f53e3d1caa2a6dafcba2f46 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 17:30:47 +1100 Subject: [PATCH 512/979] Linux: hidden_modules: Simplify symbols type checks --- .../framework/plugins/linux/hidden_modules.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index 7fd5f9fad..7a01341c1 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -58,21 +58,10 @@ class Hidden_modules(interfaces.plugins.PluginInterface): modules_addr_max = vmlinux.object_from_symbol("module_addr_max") if isinstance(modules_addr_min, objects.Void): - # Crap ISF! Here's my best-effort workaround - vollog.warning( - "Your ISF symbols are missing type information. You may need to update " - "the ISF using the latest version of dwarf2json" + raise exceptions.VolatilityException( + "Your ISF symbols lack type information. You may need to update the" + "ISF using the latest version of dwarf2json" ) - # See issue #1041. In the Linux kernel these are "unsigned long" - for type_name in ("long unsigned int", "unsigned long"): - if vmlinux.has_type(type_name): - modules_addr_min = modules_addr_min.cast(type_name) - modules_addr_max = modules_addr_max.cast(type_name) - break - else: - raise exceptions.VolatilityException( - "Bad ISF! Please update the ISF using the latest version of dwarf2json" - ) else: raise exceptions.VolatilityException( "Cannot find the module memory allocation area. Unsupported kernel" From 8960bdab086bef47121c16ba92a630dd70a18a26 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 29 Oct 2024 17:33:06 +1100 Subject: [PATCH 513/979] Linux: hidden_modules: Add a symbol table check for a recent dwarf2json version --- volatility3/framework/plugins/linux/hidden_modules.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index 7a01341c1..fd4b28943 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -232,6 +232,13 @@ class Hidden_modules(interfaces.plugins.PluginInterface): yield (0, fields) def run(self): + if self.context.symbol_space.verify_table_versions( + "dwarf2json", lambda version, _: (not version) or version < (0, 8, 0) + ): + raise exceptions.SymbolSpaceError( + "Invalid symbol table, please ensure the ISF table produced by dwarf2json was created with version 0.8.0 or later" + ) + headers = [ ("Address", format_hints.Hex), ("Name", str), From 4b76b699ad97f93fd9cf2949e6ab49c526784f3c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 09:56:05 +1100 Subject: [PATCH 514/979] Linux: hidden_modules: Add docstrings and comments to enhance the documentation of the module verification process. Move the hardcoded values to the linux constants file. --- .../framework/constants/linux/__init__.py | 8 ++++ .../symbols/linux/extensions/__init__.py | 37 ++++++++++++++----- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 0567b8574..7c485d3c3 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -339,3 +339,11 @@ class PT_FLAGS(Flag): def flags(self) -> str: """Returns the ptrace flags string""" return str(self).replace(self.__class__.__name__ + ".", "") + + +# Valid sizes for modules. Note that the Linux kernel does not define these values; they +# are based on empirical observations of typical memory allocations for kernel modules. +# We use this to verify that the found module falls within reasonable limits. +MODULE_MAXIMUM_CORE_SIZE = 20000000 +MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 +MODULE_MINIMUM_SIZE = 4096 diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index cc43df8f5..fedb3d96f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,12 +13,24 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion -from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY -from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS -from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS -from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES -from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES -from volatility3.framework.constants.linux import CAPABILITIES, PT_FLAGS +from volatility3.framework.constants.linux import ( + SOCK_TYPES, + SOCK_FAMILY, + IP_PROTOCOLS, + IPV6_PROTOCOLS, + TCP_STATES, + NETLINK_PROTOCOLS, + ETH_PROTOCOLS, + BLUETOOTH_STATES, + BLUETOOTH_PROTOCOLS, + SOCKET_STATES, + CAPABILITIES, + PT_FLAGS, + MODULE_MAXIMUM_CORE_SIZE, + MODULE_MAXIMUM_CORE_TEXT_SIZE, + MODULE_MINIMUM_SIZE, +) + from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed @@ -36,16 +48,23 @@ class module(generic.GenericIntelProcess): self._mod_mem_type = None # Initialize _mod_mem_type to None for memoization def is_valid(self): + """Determine whether it is a valid module object by verifying the self-referential + in module_kobject. This also confirms that the module is actively allocated and + not a remnant of freed memory or a failed module load attempt by verifying the + module memory section sizes. + """ layer = self._context.layers[self.vol.layer_name] # Make sure the entire module content is readable if not layer.is_valid(self.vol.offset, self.vol.size): return False core_size = self.get_core_size() + core_text_size = self.get_core_text_size() + init_size = self.get_init_size() if not ( - 1 <= core_size <= 20000000 - and core_size + self.get_init_size() >= 4096 - and 1 <= self.get_core_text_size() <= 20000000 + 0 < core_text_size <= MODULE_MAXIMUM_CORE_TEXT_SIZE + and 0 < core_size <= MODULE_MAXIMUM_CORE_SIZE + and core_size + init_size >= MODULE_MINIMUM_SIZE ): return False From 722ccd527e351f90d2a34dd31d29c39bf65be513 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 12:56:09 +1100 Subject: [PATCH 515/979] Linux: Extensions: Clean up the Linux constants imports in the object extension file --- .../symbols/linux/extensions/__init__.py | 83 ++++++++----------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fedb3d96f..1f3e0504c 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -13,24 +13,7 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion -from volatility3.framework.constants.linux import ( - SOCK_TYPES, - SOCK_FAMILY, - IP_PROTOCOLS, - IPV6_PROTOCOLS, - TCP_STATES, - NETLINK_PROTOCOLS, - ETH_PROTOCOLS, - BLUETOOTH_STATES, - BLUETOOTH_PROTOCOLS, - SOCKET_STATES, - CAPABILITIES, - PT_FLAGS, - MODULE_MAXIMUM_CORE_SIZE, - MODULE_MAXIMUM_CORE_TEXT_SIZE, - MODULE_MINIMUM_SIZE, -) - +from volatility3.framework.constants import linux as linux_constants from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed @@ -62,9 +45,9 @@ class module(generic.GenericIntelProcess): core_text_size = self.get_core_text_size() init_size = self.get_init_size() if not ( - 0 < core_text_size <= MODULE_MAXIMUM_CORE_TEXT_SIZE - and 0 < core_size <= MODULE_MAXIMUM_CORE_SIZE - and core_size + init_size >= MODULE_MINIMUM_SIZE + 0 < core_text_size <= linux_constants.MODULE_MAXIMUM_CORE_TEXT_SIZE + and 0 < core_size <= linux_constants.MODULE_MAXIMUM_CORE_SIZE + and core_size + init_size >= linux_constants.MODULE_MINIMUM_SIZE ): return False @@ -383,7 +366,7 @@ class task_struct(generic.GenericIntelProcess): Returns: bool: True, if this task is a kernel thread. Otherwise, False. """ - return (self.flags & constants.linux.PF_KTHREAD) != 0 + return (self.flags & linux_constants.PF_KTHREAD) != 0 @property def is_thread_group_leader(self) -> bool: @@ -460,7 +443,11 @@ class task_struct(generic.GenericIntelProcess): def get_ptrace_tracee_flags(self) -> Optional[str]: """Returns a string with the ptrace flags""" - return PT_FLAGS(self.ptrace).flags if self.is_being_ptraced else None + return ( + linux_constants.PT_FLAGS(self.ptrace).flags + if self.is_being_ptraced + else None + ) class fs_struct(objects.StructType): @@ -1567,18 +1554,18 @@ class socket(objects.StructType): def get_state(self): socket_state_idx = self.state - if 0 <= socket_state_idx < len(SOCKET_STATES): - return SOCKET_STATES[socket_state_idx] + if 0 <= socket_state_idx < len(linux_constants.SOCKET_STATES): + return linux_constants.SOCKET_STATES[socket_state_idx] class sock(objects.StructType): def get_family(self): family_idx = self.__sk_common.skc_family - if 0 <= family_idx < len(SOCK_FAMILY): - return SOCK_FAMILY[family_idx] + if 0 <= family_idx < len(linux_constants.SOCK_FAMILY): + return linux_constants.SOCK_FAMILY[family_idx] def get_type(self): - return SOCK_TYPES.get(self.sk_type, "") + return linux_constants.SOCK_TYPES.get(self.sk_type, "") def get_inode(self): if not self.sk_socket: @@ -1612,8 +1599,8 @@ class unix_sock(objects.StructType): # Unix socket states reuse (a subset) of the inet_sock states contants if self.sk.get_type() == "STREAM": state_idx = self.sk.__sk_common.skc_state - if 0 <= state_idx < len(TCP_STATES): - return TCP_STATES[state_idx] + if 0 <= state_idx < len(linux_constants.TCP_STATES): + return linux_constants.TCP_STATES[state_idx] else: # Return the generic socket state return self.sk.sk_socket.get_state() @@ -1625,15 +1612,15 @@ class unix_sock(objects.StructType): class inet_sock(objects.StructType): def get_family(self): family_idx = self.sk.__sk_common.skc_family - if 0 <= family_idx < len(SOCK_FAMILY): - return SOCK_FAMILY[family_idx] + if 0 <= family_idx < len(linux_constants.SOCK_FAMILY): + return linux_constants.SOCK_FAMILY[family_idx] def get_protocol(self): # If INET6 family and a proto is defined, we use that specific IPv6 protocol. # Otherwise, we use the standard IP protocol. - protocol = IP_PROTOCOLS.get(self.sk.sk_protocol) + protocol = linux_constants.IP_PROTOCOLS.get(self.sk.sk_protocol) if self.get_family() == "AF_INET6": - protocol = IPV6_PROTOCOLS.get(self.sk.sk_protocol, protocol) + protocol = linux_constants.IPV6_PROTOCOLS.get(self.sk.sk_protocol, protocol) return protocol def get_state(self): @@ -1641,8 +1628,8 @@ class inet_sock(objects.StructType): if self.sk.get_type() == "STREAM": state_idx = self.sk.__sk_common.skc_state - if 0 <= state_idx < len(TCP_STATES): - return TCP_STATES[state_idx] + if 0 <= state_idx < len(linux_constants.TCP_STATES): + return linux_constants.TCP_STATES[state_idx] else: # Return the generic socket state return self.sk.sk_socket.get_state() @@ -1725,8 +1712,8 @@ class inet_sock(objects.StructType): class netlink_sock(objects.StructType): def get_protocol(self): protocol_idx = self.sk.sk_protocol - if 0 <= protocol_idx < len(NETLINK_PROTOCOLS): - return NETLINK_PROTOCOLS[protocol_idx] + if 0 <= protocol_idx < len(linux_constants.NETLINK_PROTOCOLS): + return linux_constants.NETLINK_PROTOCOLS[protocol_idx] def get_state(self): # Return the generic socket state @@ -1768,8 +1755,8 @@ class packet_sock(objects.StructType): eth_proto = socket_module.htons(self.num) if eth_proto == 0: return None - elif eth_proto in ETH_PROTOCOLS: - return ETH_PROTOCOLS[eth_proto] + elif eth_proto in linux_constants.ETH_PROTOCOLS: + return linux_constants.ETH_PROTOCOLS[eth_proto] else: return f"0x{eth_proto:x}" @@ -1781,13 +1768,13 @@ class packet_sock(objects.StructType): class bt_sock(objects.StructType): def get_protocol(self): type_idx = self.sk.sk_protocol - if 0 <= type_idx < len(BLUETOOTH_PROTOCOLS): - return BLUETOOTH_PROTOCOLS[type_idx] + if 0 <= type_idx < len(linux_constants.BLUETOOTH_PROTOCOLS): + return linux_constants.BLUETOOTH_PROTOCOLS[type_idx] def get_state(self): state_idx = self.sk.__sk_common.skc_state - if 0 <= state_idx < len(BLUETOOTH_STATES): - return BLUETOOTH_STATES[state_idx] + if 0 <= state_idx < len(linux_constants.BLUETOOTH_STATES): + return linux_constants.BLUETOOTH_STATES[state_idx] class xdp_sock(objects.StructType): @@ -1905,7 +1892,7 @@ class kernel_cap_struct(objects.StructType): Returns: int: The latest capability ID supported by the framework. """ - return len(CAPABILITIES) - 1 + return len(linux_constants.CAPABILITIES) - 1 def get_kernel_cap_full(self) -> int: """Return the maximum value allowed for this kernel for a capability @@ -1934,7 +1921,7 @@ class kernel_cap_struct(objects.StructType): """ capabilities = [] - for bit, name in enumerate(CAPABILITIES): + for bit, name in enumerate(linux_constants.CAPABILITIES): if capabilities_bitfield & (1 << bit) != 0: capabilities.append(name) @@ -1995,10 +1982,10 @@ class kernel_cap_struct(objects.StructType): Returns: bool: "True" if the given capability is enabled. """ - if capability not in CAPABILITIES: + if capability not in linux_constants.CAPABILITIES: raise AttributeError(f"Unknown capability with name '{capability}'") - cap_value = 1 << CAPABILITIES.index(capability) + cap_value = 1 << linux_constants.CAPABILITIES.index(capability) return cap_value & self.get_capabilities() != 0 From 42d918c05d6a2d08d025d63f51b9c6764076be12 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 15:21:13 +1100 Subject: [PATCH 516/979] Linux: Boottime API: Refactor Timespec Methods. Move TimespecVol3 methods to an abstract class Timespec64Abstract, which is now inherited by Timespec64 and Timespec64Concrete. --- .../framework/symbols/linux/__init__.py | 106 ------------- .../symbols/linux/extensions/__init__.py | 140 ++++++++++++++++-- 2 files changed, 127 insertions(+), 119 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 84f546a9f..1f2812c10 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -3,15 +3,11 @@ # import math import contextlib -import datetime -import dataclasses from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects -from volatility3.framework.renderers import conversion -from volatility3.framework.constants.linux import NSEC_PER_SEC from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions @@ -836,105 +832,3 @@ class PageCache(object): page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page - - -@dataclasses.dataclass -class TimespecVol3(object): - """Internal helper class to handle all required timespec operations, convertions and - adjustments. - - NOTE: This is intended for exclusive use with get_boottime() and its related functions. - """ - - tv_sec: int = 0 - tv_nsec: int = 0 - - @classmethod - def new_from_timespec(cls, timespec) -> "TimespecVol3": - """Creates a new instance from a TimespecVol3 or timespec64 object""" - if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): - raise TypeError("It requires either a TimespecVol3 or timespec64 type") - - tv_sec = int(timespec.tv_sec) - tv_nsec = int(timespec.tv_nsec) - return cls(tv_sec=tv_sec, tv_nsec=tv_nsec) - - @classmethod - def new_from_nsec(cls, nsec) -> "TimespecVol3": - """Creates a new instance from an integer in nanoseconds""" - - # Based on ns_to_timespec64() - if nsec > 0: - tv_sec = nsec // NSEC_PER_SEC - tv_nsec = nsec % NSEC_PER_SEC - elif nsec < 0: - tv_sec = -((-nsec - 1) // NSEC_PER_SEC) - 1 - rem = (-nsec - 1) % NSEC_PER_SEC - tv_nsec = NSEC_PER_SEC - rem - 1 - else: - tv_sec = tv_nsec = 0 - - return cls(tv_sec=tv_sec, tv_nsec=tv_nsec) - - def to_datetime(self) -> datetime.datetime: - """Converts this TimespecVol3 to a UTC aware datetime""" - return conversion.unixtime_to_datetime( - self.tv_sec + self.tv_nsec / NSEC_PER_SEC - ) - - def to_timedelta(self) -> datetime.timedelta: - """Converts this TimespecVol3 to timedelta""" - return datetime.timedelta(seconds=self.tv_sec + self.tv_nsec / NSEC_PER_SEC) - - def __add__(self, timespec) -> "TimespecVol3": - """Returns a new TimespecVol3 object that sums the current values with those - in the timespec argument""" - if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): - raise TypeError("Cannot add a TimespecVol3 to this object") - - result = TimespecVol3( - tv_sec=self.tv_sec + timespec.tv_sec, - tv_nsec=self.tv_nsec + timespec.tv_nsec, - ) - - result.normalize() - - return result - - def __sub__(self, timespec) -> "TimespecVol3": - """Returns a new TimespecVol3 object that subtracts the values in the timespec - argument from the current object's values""" - if not isinstance(timespec, (TimespecVol3, extensions.timespec64)): - raise TypeError("Cannot substract this object to a TimespecVol3") - - result = TimespecVol3( - tv_sec=self.tv_sec - timespec.tv_sec, - tv_nsec=self.tv_nsec - timespec.tv_nsec, - ) - - result.normalize() - - return result - - def normalize(self): - """Normalize any overflow in tv_sec and tv_nsec after previous addition or subtractions""" - # Based on kernel's set_normalized_timespec64() - while self.tv_nsec >= NSEC_PER_SEC: - self.tv_nsec -= NSEC_PER_SEC - self.tv_sec += 1 - - while self.tv_nsec < 0: - self.tv_nsec += NSEC_PER_SEC - self.tv_sec -= 1 - - def negate(self): - """Returns a new TimespecVol3 object with the values of the current object negated""" - - result = TimespecVol3( - tv_sec=-self.tv_sec, - tv_nsec=-self.tv_nsec, - ) - - result.normalize() - - return result diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e952ed968..ea20850b7 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import abc import collections.abc import logging import functools @@ -18,12 +19,13 @@ from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES -from volatility3.framework.constants.linux import CAPABILITIES, PT_FLAGS +from volatility3.framework.constants.linux import CAPABILITIES, PT_FLAGS, NSEC_PER_SEC from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf + vollog = logging.getLogger(__name__) # Keep these in a basic module, to prevent import cycles when symbol providers require them @@ -431,10 +433,10 @@ class task_struct(generic.GenericIntelProcess): if start_time_obj_type_name != "timespec": # kernels >= 3.17 real_start_time and start_time are u64 # kernels >= 5.5 uses start_boottime which is also a u64 - start_time = linux.TimespecVol3.new_from_nsec(start_time_obj) + start_time = Timespec64Concrete.new_from_nsec(start_time_obj) else: # kernels < 3.17 real_start_time and start_time are timespec - start_time = linux.TimespecVol3.new_from_timespec(start_time_obj) + start_time = Timespec64Concrete.new_from_timespec(start_time_obj) # This is relative to the boot time so it makes sense to be a timedelta. return start_time.to_timedelta() @@ -508,8 +510,8 @@ class task_struct(generic.GenericIntelProcess): return time_namespace_offsets.boottime - def _get_boottime_raw(self) -> "linux.TimespecVol3": - """Returns the boot time in a TimespecVol3.""" + def _get_boottime_raw(self) -> "Timespec64Concrete": + """Returns the boot time in a Timespec64Concrete object.""" vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) if vmlinux.has_symbol("tk_core"): @@ -522,7 +524,7 @@ class task_struct(generic.GenericIntelProcess): else: # 3.17 <= kernels < 4.10 - Tested on Ubuntu 4.4.0-142 boottime_nsec = timekeeper.offs_real.tv64 - timekeeper.offs_boot.tv64 - return linux.TimespecVol3.new_from_nsec(boottime_nsec) + return Timespec64Concrete.new_from_nsec(boottime_nsec) elif vmlinux.has_symbol("timekeeper") and vmlinux.get_type( "timekeeper" @@ -531,7 +533,7 @@ class task_struct(generic.GenericIntelProcess): timekeeper = vmlinux.object_from_symbol("timekeeper") # timekeeper.wall_to_monotonic is timespec - boottime = linux.TimespecVol3.new_from_timespec( + boottime = Timespec64Concrete.new_from_timespec( timekeeper.wall_to_monotonic ) @@ -542,7 +544,7 @@ class task_struct(generic.GenericIntelProcess): elif vmlinux.has_symbol("wall_to_monotonic"): # kernels < 3.4 - Tested on Debian7 3.2.0-4 (3.2.57-3+deb7u2) wall_to_monotonic = vmlinux.object_from_symbol("wall_to_monotonic") - boottime = linux.TimespecVol3.new_from_timespec(wall_to_monotonic) + boottime = Timespec64Concrete.new_from_timespec(wall_to_monotonic) if vmlinux.has_symbol("total_sleep_time"): # 2.6.23 <= kernels < 3.4 7c3f1a573237b90ef331267260358a0ec4ac9079 total_sleep_time = vmlinux.object_from_symbol("total_sleep_time") @@ -2168,12 +2170,124 @@ class kernel_cap_t(kernel_cap_struct): return cap_value & self.get_kernel_cap_full() -class timespec64(objects.StructType): - def to_datetime(self) -> datetime.datetime: - """Returns the respective aware datetime""" +class Timespec64Abstract(abc.ABC): + """Abstract class to handle all required timespec64 operations, convertions and + adjustments.""" - dt = conversion.unixtime_to_datetime(self.tv_sec + self.tv_nsec / 1e9) - return dt + @classmethod + def new_from_timespec(cls, other) -> "Timespec64Concrete": + """Creates a new instance from an Timespec64Abstract subclass object""" + if not isinstance(other, Timespec64Abstract): + raise TypeError("Requires an object subclass of Timespec64Abstract") + + tv_sec = int(other.tv_sec) + tv_nsec = int(other.tv_nsec) + return Timespec64Concrete(tv_sec=tv_sec, tv_nsec=tv_nsec) + + @classmethod + def new_from_nsec(cls, nsec) -> "Timespec64Concrete": + """Creates a new instance from an integer in nanoseconds""" + + # Based on ns_to_timespec64() + if nsec > 0: + tv_sec = nsec // NSEC_PER_SEC + tv_nsec = nsec % NSEC_PER_SEC + elif nsec < 0: + tv_sec = -((-nsec - 1) // NSEC_PER_SEC) - 1 + rem = (-nsec - 1) % NSEC_PER_SEC + tv_nsec = NSEC_PER_SEC - rem - 1 + else: + tv_sec = tv_nsec = 0 + + return Timespec64Concrete(tv_sec=tv_sec, tv_nsec=tv_nsec) + + def to_datetime(self) -> datetime.datetime: + """Converts this Timespec64Abstract subclass object to a UTC aware datetime""" + + # pylint: disable=E1101 + return conversion.unixtime_to_datetime( + self.tv_sec + self.tv_nsec / NSEC_PER_SEC + ) + + def to_timedelta(self) -> datetime.timedelta: + """Converts this Timespec64Abstract subclass object to timedelta""" + # pylint: disable=E1101 + return datetime.timedelta(seconds=self.tv_sec + self.tv_nsec / NSEC_PER_SEC) + + def __add__(self, other) -> "Timespec64Concrete": + """Returns a new Timespec64Concrete object that sums the current values with those + in the timespec argument""" + if not isinstance(other, Timespec64Abstract): + raise TypeError("Requires an object subclass of Timespec64Abstract") + + # pylint: disable=E1101 + result = Timespec64Concrete( + tv_sec=self.tv_sec + other.tv_sec, + tv_nsec=self.tv_nsec + other.tv_nsec, + ) + + result.normalize() + + return result + + def __sub__(self, other) -> "Timespec64Concrete": + """Returns a new Timespec64Abstract object that subtracts the values in the timespec + argument from the current object's values""" + if not isinstance(other, Timespec64Abstract): + raise TypeError("Requires an object subclass of Timespec64Abstract") + + # pylint: disable=E1101 + result = Timespec64Concrete( + tv_sec=self.tv_sec - other.tv_sec, + tv_nsec=self.tv_nsec - other.tv_nsec, + ) + + result.normalize() + + return result + + def normalize(self): + """Normalize any overflow in tv_sec and tv_nsec.""" + # Based on kernel's set_normalized_timespec64() + + # pylint: disable=E1101 + while self.tv_nsec >= NSEC_PER_SEC: + self.tv_nsec -= NSEC_PER_SEC + self.tv_sec += 1 + + while self.tv_nsec < 0: + self.tv_nsec += NSEC_PER_SEC + self.tv_sec -= 1 + + def negate(self): + """Returns a new Timespec64Concrete object with the values of the current object negated""" + # pylint: disable=E1101 + result = Timespec64Concrete( + tv_sec=-self.tv_sec, + tv_nsec=-self.tv_nsec, + ) + + result.normalize() + + return result + + +class Timespec64Concrete(Timespec64Abstract): + """Handle all required timespec64 operations, convertions and adjustments. + This is used to dynamically create timespec64-like objects, each its own variables + and the same methods as a timespec64 object extension. + """ + + def __init__(self, tv_sec=0, tv_nsec=0): + self.tv_sec = tv_sec + self.tv_nsec = tv_nsec + + +class timespec64(Timespec64Abstract, objects.StructType): + """Handle all required timespec64 operations, convertions and adjustments. + This works as an extension of the timespec64 object while maintaining the same methods + as a Timespec64Concrete object. + """ class inode(objects.StructType): From 7abe92cf2f2ddac1a5e7f15a27afe341e1810901 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 15:36:46 +1100 Subject: [PATCH 517/979] Linux: Boottime API: Minor. Move negate() up --- .../symbols/linux/extensions/__init__.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index ea20850b7..f20d93755 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2244,6 +2244,18 @@ class Timespec64Abstract(abc.ABC): result.normalize() + return self + other.negate() + + def negate(self): + """Returns a new Timespec64Concrete object with the values of the current object negated""" + # pylint: disable=E1101 + result = Timespec64Concrete( + tv_sec=-self.tv_sec, + tv_nsec=-self.tv_nsec, + ) + + result.normalize() + return result def normalize(self): @@ -2259,18 +2271,6 @@ class Timespec64Abstract(abc.ABC): self.tv_nsec += NSEC_PER_SEC self.tv_sec -= 1 - def negate(self): - """Returns a new Timespec64Concrete object with the values of the current object negated""" - # pylint: disable=E1101 - result = Timespec64Concrete( - tv_sec=-self.tv_sec, - tv_nsec=-self.tv_nsec, - ) - - result.normalize() - - return result - class Timespec64Concrete(Timespec64Abstract): """Handle all required timespec64 operations, convertions and adjustments. From e197fbaf5fe7a7fba8fabce73533725af9110d74 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 15:39:40 +1100 Subject: [PATCH 518/979] Linux: Boottime API: Refactor __sub__ to operate through __add__() and negate() for improved clarity and reuse --- .../framework/symbols/linux/extensions/__init__.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f20d93755..c80ffa515 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2236,17 +2236,9 @@ class Timespec64Abstract(abc.ABC): if not isinstance(other, Timespec64Abstract): raise TypeError("Requires an object subclass of Timespec64Abstract") - # pylint: disable=E1101 - result = Timespec64Concrete( - tv_sec=self.tv_sec - other.tv_sec, - tv_nsec=self.tv_nsec - other.tv_nsec, - ) - - result.normalize() - return self + other.negate() - def negate(self): + def negate(self) -> "Timespec64Concrete": """Returns a new Timespec64Concrete object with the values of the current object negated""" # pylint: disable=E1101 result = Timespec64Concrete( From a05397e8b68c280ba74c58bc968cc6692d25e613 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 15:57:00 +1100 Subject: [PATCH 519/979] Linux: Boottime API: Minor. Fix docstring typo --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index c80ffa515..a9127907e 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2266,7 +2266,7 @@ class Timespec64Abstract(abc.ABC): class Timespec64Concrete(Timespec64Abstract): """Handle all required timespec64 operations, convertions and adjustments. - This is used to dynamically create timespec64-like objects, each its own variables + This is used to dynamically create timespec64-like objects, each with its own variables and the same methods as a timespec64 object extension. """ From cf94b078cdd2534493209c08c5b926c85dbcd699 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 16:11:03 +1100 Subject: [PATCH 520/979] Linux: PageCache API: Fix typing --- volatility3/framework/symbols/linux/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 1f2812c10..3289775b6 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -615,7 +615,7 @@ class IDStorage(ABC): return nodep - def _iter_node(self, nodep, height) -> int: + def _iter_node(self, nodep, height) -> Iterator[int]: node = self.nodep_to_node(nodep) node_slots = node.slots for off in range(self.CHUNK_SIZE): @@ -632,7 +632,7 @@ class IDStorage(ABC): for child_node in self._iter_node(nodep, height - 1): yield child_node - def get_entries(self, root: interfaces.objects.ObjectInterface) -> int: + def get_entries(self, root: interfaces.objects.ObjectInterface) -> Iterator[int]: """Walks the tree data structure Args: @@ -818,7 +818,7 @@ class PageCache(object): self._page_cache = page_cache self._idstorage = IDStorage.choose_id_storage(context, kernel_module_name) - def get_cached_pages(self) -> interfaces.objects.ObjectInterface: + def get_cached_pages(self) -> Iterator[interfaces.objects.ObjectInterface]: """Returns all page cache contents Yields: From 84a2233d04b9ef81c97467765a61f12e0403837e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 16:25:28 +1100 Subject: [PATCH 521/979] Linux: vfsmount type: Fix type comparison --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index eba775714..f3c29043f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1299,7 +1299,7 @@ class vfsmount(objects.StructType): bool: 'True' if the given argument points to the the same 'vfsmount' as 'self'. """ - if type(vfsmount_ptr) == objects.Pointer: + if isinstance(vfsmount_ptr, objects.Pointer): return self.vol.offset == vfsmount_ptr else: raise exceptions.VolatilityException( From aecf8d432706bff371632c28f82c9f9f025da2e6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 18:53:20 +1100 Subject: [PATCH 522/979] Linux: module extension object: Refactor to replace custom property cache with functools.cached_property that's available from Python 3.8 --- .../symbols/linux/extensions/__init__.py | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index eba775714..7dce16267 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -31,29 +31,24 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._mod_mem_type = None # Initialize _mod_mem_type to None for memoization - - @property + @functools.cached_property def mod_mem_type(self): """Return the mod_mem_type enum choices if available or an empty dict if not""" # mod_mem_type and module_memory were added in kernel 6.4 which replaces # module_layout for storing the information around core_layout etc. # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information + symbol_table_name = self.get_symbol_table_name() + mod_mem_type_symname = symbol_table_name + constants.BANG + "mod_mem_type" + symbol_space = self._context.symbol_space + try: + mod_mem_type = symbol_space.get_enumeration(mod_mem_type_symname).choices + except exceptions.SymbolError: + mod_mem_type = {} + vollog.debug( + "Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" + ) - if self._mod_mem_type is None: - try: - self._mod_mem_type = self._context.symbol_space.get_enumeration( - self.get_symbol_table_name() + constants.BANG + "mod_mem_type" - ).choices - except exceptions.SymbolError: - vollog.debug( - "Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" - ) - # set to empty dict to show that the enum was not found, and so shouldn't be searched for again - self._mod_mem_type = {} - return self._mod_mem_type + return mod_mem_type def get_module_base(self): if self.has_member("mem"): # kernels 6.4+ From b09859668751c992be2065d60ebe2220cb4d93cc Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 30 Oct 2024 19:50:13 +1100 Subject: [PATCH 523/979] Linux: module extension object: Add type annotation to the mod_mem_type property --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7dce16267..fe3c7af49 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -32,7 +32,7 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): @functools.cached_property - def mod_mem_type(self): + def mod_mem_type(self) -> Dict: """Return the mod_mem_type enum choices if available or an empty dict if not""" # mod_mem_type and module_memory were added in kernel 6.4 which replaces # module_layout for storing the information around core_layout etc. From e183f81de6cbea7725ac0fc2953b6b5003efb945 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 31 Oct 2024 08:28:07 +1100 Subject: [PATCH 524/979] linux: hidden_modules: Fix mess with merge #1283 --- .../symbols/linux/extensions/__init__.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 339896e9d..aa3e8c675 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -63,19 +63,18 @@ class module(generic.GenericIntelProcess): # mod_mem_type and module_memory were added in kernel 6.4 which replaces # module_layout for storing the information around core_layout etc. # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information + symbol_table_name = self.get_symbol_table_name() + mod_mem_type_symname = symbol_table_name + constants.BANG + "mod_mem_type" + symbol_space = self._context.symbol_space + try: + mod_mem_type = symbol_space.get_enumeration(mod_mem_type_symname).choices + except exceptions.SymbolError: + mod_mem_type = {} + vollog.debug( + "Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" + ) - if self._mod_mem_type is None: - try: - self._mod_mem_type = self._context.symbol_space.get_enumeration( - self.get_symbol_table_name() + constants.BANG + "mod_mem_type" - ).choices - except exceptions.SymbolError: - vollog.debug( - "Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" - ) - # set to empty dict to show that the enum was not found, and so shouldn't be searched for again - self._mod_mem_type = {} - return self._mod_mem_type + return mod_mem_type def _get_mem_type(self, mod_mem_type_name): module_mem_index = self.mod_mem_type.get(mod_mem_type_name) From d5a0b93383fda59267bdd9b42e716b70ad66595c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:28:40 +0100 Subject: [PATCH 525/979] add TAINT_FLAGS constant --- .../framework/constants/linux/__init__.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 7c485d3c3..9f25c9225 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -347,3 +347,88 @@ class PT_FLAGS(Flag): MODULE_MAXIMUM_CORE_SIZE = 20000000 MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 MODULE_MINIMUM_SIZE = 4096 + + +TAINT_FLAGS = { + "P": { + "shift": 1 << 0, + "desc": "PROPRIETARY_MODULE", + "when_present": True, + "module": True, + }, + "G": { + "shift": 1 << 0, + "desc": "PROPRIETARY_MODULE", + "when_present": False, + "module": True, + }, + "F": { + "shift": 1 << 1, + "desc": "FORCED_MODULE", + "when_present": True, + "module": False, + }, + # CPU_OUT_OF_SPEC was TAINT_UNSAFE_SMP on < 3.15-rc1 : https://lore.kernel.org/linux-kernel//20140303080432.GA25489@localhost/t/#:~:text=liked%20your%20proposal%3A-,%3E%20Right,-%2C%20I%20was%20about + "S": { + "shift": 1 << 2, + "desc": "CPU_OUT_OF_SPEC", + "when_present": True, + "module": False, + }, + "R": { + "shift": 1 << 3, + "desc": "FORCED_RMMOD", + "when_present": True, + "module": False, + }, + "M": { + "shift": 1 << 4, + "desc": "MACHINE_CHECK", + "when_present": True, + "module": False, + }, + "B": {"shift": 1 << 5, "desc": "BAD_PAGE", "when_present": True, "module": False}, + "U": {"shift": 1 << 6, "desc": "USER", "when_present": True, "module": False}, + "D": {"shift": 1 << 7, "desc": "DIE", "when_present": True, "module": False}, + "A": { + "shift": 1 << 8, + "desc": "OVERRIDDEN_ACPI_TABLE", + "when_present": True, + "module": False, + }, + "W": {"shift": 1 << 9, "desc": "WARN", "when_present": True, "module": False}, + "C": {"shift": 1 << 10, "desc": "CRAP", "when_present": True, "module": True}, + "I": { + "shift": 1 << 11, + "desc": "FIRMWARE_WORKAROUND", + "when_present": True, + "module": False, + }, + "O": {"shift": 1 << 12, "desc": "OOT_MODULE", "when_present": True, "module": True}, + "E": { + "shift": 1 << 13, + "desc": "UNSIGNED_MODULE", + "when_present": True, + "module": True, + }, + "L": { + "shift": 1 << 14, + "desc": "SOFTLOCKUP", + "when_present": True, + "module": False, + }, + "K": {"shift": 1 << 15, "desc": "LIVEPATCH", "when_present": True, "module": True}, + "X": {"shift": 1 << 16, "desc": "AUX", "when_present": True, "module": True}, + "T": {"shift": 1 << 17, "desc": "RANDSTRUCT", "when_present": True, "module": True}, + "N": {"shift": 1 << 18, "desc": "TEST", "when_present": True, "module": True}, +} +"""Flags used to taint kernel and modules, for debugging purposes. + +Map based on 6.12-rc5. + +Documentation : + - https://www.kernel.org/doc/Documentation/admin-guide/sysctl/kernel.rst#:~:text=guide/sysrq.rst.-,tainted,-%3D%3D%3D%3D%3D%3D%3D%0A%0ANon%2Dzero%20if + - https://www.kernel.org/doc/Documentation/admin-guide/tainted-kernels.rst#:~:text=More%20detailed%20explanation%20for%20tainting + - taint_flag kernel struct + - taint_flags kernel constant +""" From e1b343a284436ee4d92a7b8a6daf0a98dd01fdeb Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:33:54 +0100 Subject: [PATCH 526/979] add module taints parsing apis --- .../symbols/linux/extensions/__init__.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index aa3e8c675..89e2cde27 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -279,6 +279,77 @@ class module(generic.GenericIntelProcess): return None + def _module_flags_taints_pre_4_10_rc1(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Returns: + The raw taints string. + """ + taints_string = "" + for char, infos in linux_constants.TAINT_FLAGS.items(): + if infos["module"] and self.taints_value & infos["shift"]: + taints_string += char + + return taints_string + + def _module_flags_taints_post_4_10_rc1(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self.taint_flags_list): + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taint_flag.module and (self.taints_value & (1 << i)): + taints_string += c_true + elif taint_flag.module and c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self.taint_flags_list: + return self._module_flags_taints_post_4_10_rc1() + return self._module_flags_taints_pre_4_10_rc1() + + def get_taints_parsed(self) -> List[str]: + """Convert the module's taints string to a 1-1 descriptor mapping. + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for c in self.get_taints_as_plain_string(): + infos = linux_constants.TAINT_FLAGS.get(c) + if not infos: + comprehensive_taints.append(f"") + elif infos["when_present"]: + comprehensive_taints.append(infos["desc"]) + + return comprehensive_taints + @property def section_symtab(self): if self.has_member("kallsyms"): @@ -307,6 +378,17 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("module -> strtab: Unable to get strtab") + @property + def taints_value(self) -> int: + return self.taints + + @property + def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: + kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + if kernel.has_symbol("taint_flags"): + return list(kernel.object_from_symbol("taint_flags")) + return None + class task_struct(generic.GenericIntelProcess): def add_process_layer( From c88ebe89270355d188770c74b39eb8acef1f3549 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:35:54 +0100 Subject: [PATCH 527/979] introduce modxview linux plugin --- .../framework/plugins/linux/modxview.py | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 volatility3/framework/plugins/linux/modxview.py diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py new file mode 100644 index 000000000..66b644164 --- /dev/null +++ b/volatility3/framework/plugins/linux/modxview.py @@ -0,0 +1,195 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import List, Dict, Set, Iterator +from volatility3.plugins.linux import lsmod, check_modules, hidden_modules +from volatility3.framework import interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.symbols.linux import extensions +from volatility3.framework.constants import architectures + +vollog = logging.getLogger(__name__) + + +class Modxview(interfaces.plugins.PluginInterface): + """Centralize lsmod, check_modules and hidden_modules results to efficiently + spot modules presence and taints.""" + + _version = (1, 0, 0) + _required_framework_version = (2, 11, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="check_modules", + plugin=check_modules.Check_modules, + version=(0, 0, 0), + ), + requirements.PluginRequirement( + name="hidden_modules", + plugin=hidden_modules.Hidden_modules, + version=(1, 0, 0), + ), + requirements.BooleanRequirement( + name="plain_taints", + description="Display the plain taints string for each module.", + optional=True, + default=False, + ), + ] + + @classmethod + def run_lsmod( + cls, context: interfaces.context.ContextInterface, kernel_name: str + ) -> List[extensions.module]: + """Wrapper for the lsmod plugin.""" + return list(lsmod.Lsmod.list_modules(context, kernel_name)) + + @classmethod + def run_check_modules( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + ) -> List[extensions.module]: + """Wrapper for the check_modules plugin. + Here, we extract the /sys/module/ list.""" + kernel = context.modules[kernel_name] + sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( + context, kernel_name + ) + + # Convert get_kset_modules() offsets back to module objects + return [ + kernel.object(object_type="module", offset=m_offset, absolute=True) + for m_offset in sysfs_modules.values() + ] + + @classmethod + def run_hidden_modules( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + known_modules_addresses: Set[int], + ) -> List[extensions.module]: + """Wrapper for the hidden_modules plugin.""" + modules_memory_boundaries = ( + hidden_modules.Hidden_modules.get_modules_memory_boundaries( + context, kernel_name + ) + ) + return list( + hidden_modules.Hidden_modules.get_hidden_modules( + context, + kernel_name, + known_modules_addresses, + modules_memory_boundaries, + ) + ) + + @classmethod + def flatten_run_modules_results( + cls, run_results: Dict[str, List[extensions.module]], deduplicate: bool = True + ) -> Iterator[extensions.module]: + """Flatten a dictionary mapping plugin names and modules list, to a single merged list. + This is useful to get a generic lookup list of all the detected modules. + + Args: + run_results: dictionary of plugin names mapping a list of detected modules + deduplicate: remove duplicate modules, based on their offsets + + Returns: + Iterator of modules objects + """ + seen_addresses = set() + for modules in run_results.values(): + for module in modules: + if deduplicate and module.vol.offset in seen_addresses: + continue + yield module + + @classmethod + def run_modules_scanners( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + run_hidden_modules: bool = True, + ) -> Dict[str, List[extensions.module]]: + """Run module scanning plugins and aggregate the results. + + Args: + run_hidden_modules: specify if the hidden_modules plugin should be run + Returns: + Dictionary mapping each plugin to its corresponding result + """ + + kernel = context.modules[kernel_name] + run_results = {} + run_results["lsmod"] = cls.run_lsmod(context, kernel_name) + run_results["check_modules"] = cls.run_check_modules(context, kernel_name) + if run_hidden_modules: + known_module_addresses = set( + context.layers[kernel.layer_name].canonicalize(module.vol.offset) + for module in run_results["lsmod"] + run_results["check_modules"] + ) + run_results["hidden_modules"] = cls.run_hidden_modules( + context, kernel_name, known_module_addresses + ) + + return run_results + + def _generator(self): + kernel_name = self.config["kernel"] + run_results = self.run_modules_scanners(self.context, kernel_name) + modules_offsets = {} + for key in ["lsmod", "check_modules", "hidden_modules"]: + modules_offsets[key] = set(module.vol.offset for module in run_results[key]) + + seen_addresses = set() + for modules_list in run_results.values(): + for module in modules_list: + if module.vol.offset in seen_addresses: + continue + seen_addresses.add(module.vol.offset) + + if self.config.get("plain_taints"): + taints = module.get_taints_as_plain_string() + else: + taints = ",".join(module.get_taints_parsed()) + + yield ( + 0, + ( + module.get_name() or NotAvailableValue(), + format_hints.Hex(module.vol.offset), + module.vol.offset in modules_offsets["lsmod"], + module.vol.offset in modules_offsets["check_modules"], + module.vol.offset in modules_offsets["hidden_modules"], + taints or NotAvailableValue(), + ), + ) + + def run(self): + columns = [ + ("Name", str), + ("Address", format_hints.Hex), + ("In /proc/modules", bool), + ("In /sys/module/", bool), + ("Hidden", bool), + ("Taints", str), + ] + + return TreeGrid( + columns, + self._generator(), + ) From 9440f53429a1f9c7d77d51eeb75c2b5938da040f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:05:25 +0100 Subject: [PATCH 528/979] use a dict of dataclasses for taint_flags --- .../framework/constants/linux/__init__.py | 112 +++++++----------- .../symbols/linux/extensions/__init__.py | 12 +- 2 files changed, 47 insertions(+), 77 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 9f25c9225..6cf8585f5 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -6,6 +6,7 @@ Linux-specific values that aren't found in debug symbols """ from enum import IntEnum, Flag +from dataclasses import dataclass KERNEL_NAME = "__kernel__" @@ -349,78 +350,47 @@ MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 MODULE_MINIMUM_SIZE = 4096 +@dataclass +class TaintFlag: + shift: int + desc: str + when_present: bool + module: bool + + TAINT_FLAGS = { - "P": { - "shift": 1 << 0, - "desc": "PROPRIETARY_MODULE", - "when_present": True, - "module": True, - }, - "G": { - "shift": 1 << 0, - "desc": "PROPRIETARY_MODULE", - "when_present": False, - "module": True, - }, - "F": { - "shift": 1 << 1, - "desc": "FORCED_MODULE", - "when_present": True, - "module": False, - }, - # CPU_OUT_OF_SPEC was TAINT_UNSAFE_SMP on < 3.15-rc1 : https://lore.kernel.org/linux-kernel//20140303080432.GA25489@localhost/t/#:~:text=liked%20your%20proposal%3A-,%3E%20Right,-%2C%20I%20was%20about - "S": { - "shift": 1 << 2, - "desc": "CPU_OUT_OF_SPEC", - "when_present": True, - "module": False, - }, - "R": { - "shift": 1 << 3, - "desc": "FORCED_RMMOD", - "when_present": True, - "module": False, - }, - "M": { - "shift": 1 << 4, - "desc": "MACHINE_CHECK", - "when_present": True, - "module": False, - }, - "B": {"shift": 1 << 5, "desc": "BAD_PAGE", "when_present": True, "module": False}, - "U": {"shift": 1 << 6, "desc": "USER", "when_present": True, "module": False}, - "D": {"shift": 1 << 7, "desc": "DIE", "when_present": True, "module": False}, - "A": { - "shift": 1 << 8, - "desc": "OVERRIDDEN_ACPI_TABLE", - "when_present": True, - "module": False, - }, - "W": {"shift": 1 << 9, "desc": "WARN", "when_present": True, "module": False}, - "C": {"shift": 1 << 10, "desc": "CRAP", "when_present": True, "module": True}, - "I": { - "shift": 1 << 11, - "desc": "FIRMWARE_WORKAROUND", - "when_present": True, - "module": False, - }, - "O": {"shift": 1 << 12, "desc": "OOT_MODULE", "when_present": True, "module": True}, - "E": { - "shift": 1 << 13, - "desc": "UNSIGNED_MODULE", - "when_present": True, - "module": True, - }, - "L": { - "shift": 1 << 14, - "desc": "SOFTLOCKUP", - "when_present": True, - "module": False, - }, - "K": {"shift": 1 << 15, "desc": "LIVEPATCH", "when_present": True, "module": True}, - "X": {"shift": 1 << 16, "desc": "AUX", "when_present": True, "module": True}, - "T": {"shift": 1 << 17, "desc": "RANDSTRUCT", "when_present": True, "module": True}, - "N": {"shift": 1 << 18, "desc": "TEST", "when_present": True, "module": True}, + "P": TaintFlag( + shift=1 << 0, desc="PROPRIETARY_MODULE", when_present=True, module=True + ), + "G": TaintFlag( + shift=1 << 0, desc="PROPRIETARY_MODULE", when_present=False, module=True + ), + "F": TaintFlag(shift=1 << 1, desc="FORCED_MODULE", when_present=True, module=False), + "S": TaintFlag( + shift=1 << 2, desc="CPU_OUT_OF_SPEC", when_present=True, module=False + ), + "R": TaintFlag(shift=1 << 3, desc="FORCED_RMMOD", when_present=True, module=False), + "M": TaintFlag(shift=1 << 4, desc="MACHINE_CHECK", when_present=True, module=False), + "B": TaintFlag(shift=1 << 5, desc="BAD_PAGE", when_present=True, module=False), + "U": TaintFlag(shift=1 << 6, desc="USER", when_present=True, module=False), + "D": TaintFlag(shift=1 << 7, desc="DIE", when_present=True, module=False), + "A": TaintFlag( + shift=1 << 8, desc="OVERRIDDEN_ACPI_TABLE", when_present=True, module=False + ), + "W": TaintFlag(shift=1 << 9, desc="WARN", when_present=True, module=False), + "C": TaintFlag(shift=1 << 10, desc="CRAP", when_present=True, module=True), + "I": TaintFlag( + shift=1 << 11, desc="FIRMWARE_WORKAROUND", when_present=True, module=False + ), + "O": TaintFlag(shift=1 << 12, desc="OOT_MODULE", when_present=True, module=True), + "E": TaintFlag( + shift=1 << 13, desc="UNSIGNED_MODULE", when_present=True, module=True + ), + "L": TaintFlag(shift=1 << 14, desc="SOFTLOCKUP", when_present=True, module=False), + "K": TaintFlag(shift=1 << 15, desc="LIVEPATCH", when_present=True, module=True), + "X": TaintFlag(shift=1 << 16, desc="AUX", when_present=True, module=True), + "T": TaintFlag(shift=1 << 17, desc="RANDSTRUCT", when_present=True, module=True), + "N": TaintFlag(shift=1 << 18, desc="TEST", when_present=True, module=True), } """Flags used to taint kernel and modules, for debugging purposes. diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 89e2cde27..42c1a470d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -287,8 +287,8 @@ class module(generic.GenericIntelProcess): The raw taints string. """ taints_string = "" - for char, infos in linux_constants.TAINT_FLAGS.items(): - if infos["module"] and self.taints_value & infos["shift"]: + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if taint_flag.module and self.taints_value & taint_flag.shift: taints_string += char return taints_string @@ -342,11 +342,11 @@ class module(generic.GenericIntelProcess): """ comprehensive_taints = [] for c in self.get_taints_as_plain_string(): - infos = linux_constants.TAINT_FLAGS.get(c) - if not infos: + taint_flag = linux_constants.TAINT_FLAGS.get(c) + if not taint_flag: comprehensive_taints.append(f"") - elif infos["when_present"]: - comprehensive_taints.append(infos["desc"]) + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) return comprehensive_taints From 9d08c4681ae1cf18ddf4bd53ff970f6a9bc26573 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:07:23 +0100 Subject: [PATCH 529/979] add module offset to seen_addresses --- volatility3/framework/plugins/linux/modxview.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 66b644164..f44984926 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -116,6 +116,7 @@ class Modxview(interfaces.plugins.PluginInterface): for module in modules: if deduplicate and module.vol.offset in seen_addresses: continue + seen_addresses.add(module.vol.offset) yield module @classmethod From b209ea36a284ae1a75ba18222cbe8db1c1eede4f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:12:52 +0100 Subject: [PATCH 530/979] remove slashes in columns --- volatility3/framework/plugins/linux/modxview.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index f44984926..d79f5e7a9 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -184,8 +184,8 @@ class Modxview(interfaces.plugins.PluginInterface): columns = [ ("Name", str), ("Address", format_hints.Hex), - ("In /proc/modules", bool), - ("In /sys/module/", bool), + ("In procfs", bool), + ("In sysfs", bool), ("Hidden", bool), ("Taints", str), ] From 512c1abe801b731f9f36551db51075d4d41413e1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 4 Nov 2024 17:45:56 +1100 Subject: [PATCH 531/979] linux: VMCoreInfo plugin: Add VMCoreInfo API and its respective plugin --- .../framework/constants/linux/__init__.py | 6 ++ .../framework/plugins/linux/vmcoreinfo.py | 49 +++++++++ .../framework/symbols/linux/__init__.py | 102 +++++++++++++++++- 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 volatility3/framework/plugins/linux/vmcoreinfo.py diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 7c485d3c3..dde256754 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -347,3 +347,9 @@ class PT_FLAGS(Flag): MODULE_MAXIMUM_CORE_SIZE = 20000000 MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 MODULE_MINIMUM_SIZE = 4096 + +# VMCOREINFO +VMCOREINFO_MAGIC = b"VMCOREINFO\x00" +# Aligned to 4 bytes. See storenote() in kernels < 4.19 or append_kcore_note() in kernels >= 4.19 +VMCOREINFO_MAGIC_ALIGNED = VMCOREINFO_MAGIC + b"\x00" +OSRELEASE_TAG = b"OSRELEASE=" diff --git a/volatility3/framework/plugins/linux/vmcoreinfo.py b/volatility3/framework/plugins/linux/vmcoreinfo.py new file mode 100644 index 000000000..ea30520e3 --- /dev/null +++ b/volatility3/framework/plugins/linux/vmcoreinfo.py @@ -0,0 +1,49 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from typing import List + +from volatility3.framework import renderers, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.symbols import linux +from volatility3.framework.renderers import format_hints + + +class VMCoreInfo(plugins.PluginInterface): + """Enumerate VMCoreInfo tables""" + + _required_framework_version = (2, 11, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.TranslationLayerRequirement( + name="primary", description="Memory layer to scan" + ), + requirements.VersionRequirement( + name="VMCoreInfo", component=linux.VMCoreInfo, version=(1, 0, 0) + ), + ] + + def _generator(self): + layer_name = self.config["primary"] + for ( + vmcoreinfo_offset, + vmcoreinfo, + ) in linux.VMCoreInfo.search_vmcoreinfo_elf_note( + context=self.context, + layer_name=layer_name, + ): + for key, value in vmcoreinfo.items(): + yield 0, (format_hints.Hex(vmcoreinfo_offset), key, value) + + def run(self): + headers = [ + ("Offset", format_hints.Hex), + ("Key", str), + ("Value", str), + ] + return renderers.TreeGrid(headers, self._generator()) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3289775b6..b0b6b7325 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -2,15 +2,18 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import math +import string import contextlib from abc import ABC, abstractmethod -from typing import Iterator, List, Tuple, Optional, Union +from typing import Iterator, List, Tuple, Optional, Union, Dict from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions +from volatility3.framework.layers import scanners +from volatility3.framework.constants import linux as linux_constants class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -832,3 +835,100 @@ class PageCache(object): page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page + + +class VMCoreInfo(interfaces.configuration.VersionableInterface): + _required_framework_version = (2, 11, 0) + + _version = (1, 0, 0) + + @staticmethod + def _vmcoreinfo_data_to_dict( + vmcoreinfo_data, + ) -> Optional[Dict[str, str]]: + """Converts the input VMCoreInfo data buffer into a dictionary""" + + # Ensure the whole buffer is printable + if not all(c in string.printable.encode() for c in vmcoreinfo_data): + # Abort, we are in the wrong place + return None + + vmcoreinfo_dict = dict() + for line in vmcoreinfo_data.decode().splitlines(): + if not line: + break + + key, value = line.split("=", 1) + vmcoreinfo_dict[key] = value + + return vmcoreinfo_dict + + @classmethod + def search_vmcoreinfo_elf_note( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Iterator[Tuple[int, Dict[str, str]]]: + """Enumerates each VMCoreInfo ELF note table found in memory along with its offset. + + This approach is independent of any external ISF symbol or type, requiring only the + Elf64_Note found in 'elf.json', which is already included in the framework. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The layer within the context in which the module exists + progress_callback: A function that takes a percentage (and an optional description) that will be called periodically + + Yields: + Tuples with the VMCoreInfo ELF note offset and the VMCoreInfo table parsed in a dictionary. + """ + + elf_table_name = intermed.IntermediateSymbolTable.create( + context, "elf_symbol_table", "linux", "elf" + ) + module = context.module(elf_table_name, layer_name, 0) + layer = context.layers[layer_name] + + # Both Elf32_Note and Elf64_Note are of the same size + elf_note_size = context.symbol_space[elf_table_name].get_type("Elf64_Note").size + + for vmcoreinfo_offset in layer.scan( + scanner=scanners.BytesScanner(linux_constants.VMCOREINFO_MAGIC_ALIGNED), + context=context, + progress_callback=progress_callback, + ): + # vmcoreinfo_note kernels >= 2.6.24 fd59d231f81cb02870b9cf15f456a897f3669b4e + vmcoreinfo_elf_note_offset = vmcoreinfo_offset - elf_note_size + + # Elf32_Note and Elf64_Note are identical, so either can be used interchangeably here + elf_note = module.object( + object_type="Elf64_Note", + offset=vmcoreinfo_elf_note_offset, + absolute=True, + ) + + # Ensure that we are within an ELF note + if ( + elf_note.n_namesz != len(linux_constants.VMCOREINFO_MAGIC) + or elf_note.n_type != 0 + or elf_note.n_descsz == 0 + ): + continue + + vmcoreinfo_data_offset = vmcoreinfo_offset + len( + linux_constants.VMCOREINFO_MAGIC_ALIGNED + ) + + # Also, confirm this with the first tag, which has consistently been OSRELEASE + vmcoreinfo_data = layer.read(vmcoreinfo_data_offset, elf_note.n_descsz) + if not vmcoreinfo_data.startswith(linux_constants.OSRELEASE_TAG): + continue + + table = cls._vmcoreinfo_data_to_dict(vmcoreinfo_data) + if not table: + # Wrong VMCoreInfo note offset, keep trying + continue + + # A valid VMCoreInfo ELF note exists at 'vmcoreinfo_elf_note_offset' + yield vmcoreinfo_elf_note_offset, table From b4b78342763e743b25828bb6d684ba5e1be5f1d1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 4 Nov 2024 17:47:57 +1100 Subject: [PATCH 532/979] linux: K/ASLR via VMCoreInfo --- volatility3/framework/automagic/linux.py | 108 ++++++++++++++++++++++- 1 file changed, 105 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 52a73f45a..fde060733 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -76,8 +76,12 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): isf_url=isf_path, ) context.symbol_space.append(table) + kaslr_shift, aslr_shift = cls.find_aslr( - context, table_name, layer_name, progress_callback=progress_callback + context, + table_name, + layer_name, + progress_callback=progress_callback, ) layer_class: Type = intel.Intel @@ -118,7 +122,7 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): return None @classmethod - def find_aslr( + def find_aslr_classic( cls, context: interfaces.context.ContextInterface, symbol_table: str, @@ -126,7 +130,17 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): progress_callback: constants.ProgressCallback = None, ) -> Tuple[int, int]: """Determines the offset of the actual DTB in physical space and its - symbol offset.""" + symbol offset. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + symbol_table: The name of the kernel module on which to operate + layer_name: The layer within the context in which the module exists + progress_callback: A function that takes a percentage (and an optional description) that will be called periodically + + Returns: + kaslr_shirt and aslr_shift + """ init_task_symbol = symbol_table + constants.BANG + "init_task" init_task_json_address = context.symbol_space.get_symbol( init_task_symbol @@ -184,6 +198,59 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): vollog.debug("Scanners could not determine any ASLR shifts, using 0 for both") return 0, 0 + @classmethod + def find_aslr_vmcoreinfo( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[Tuple[int, int]]: + """Determines the ASLR offsets using the VMCOREINFO ELF note + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The layer within the context in which the module exists + progress_callback: A function that takes a percentage (and an optional description) that will be called periodically + + Returns: + kaslr_shirt and aslr_shift + """ + + for ( + _vmcoreinfo_offset, + vmcoreinfo, + ) in linux.VMCoreInfo.search_vmcoreinfo_elf_note( + context=context, + layer_name=layer_name, + progress_callback=progress_callback, + ): + + phys_base_str = vmcoreinfo.get("NUMBER(phys_base)") + if phys_base_str is None: + # We are in kernel (x86) < 4.10 401721ecd1dcb0a428aa5d6832ee05ffbdbffbbe where it was SYMBOL(phys_base) + # It's the symbol address instead of the value itself, which is useless for calculating the physical address. + # raise Exception("Kernel < 4.10") + continue + + kerneloffset_str = vmcoreinfo.get("KERNELOFFSET") + if kerneloffset_str is None: + # KERNELOFFSET: (x86) kernels < 3.13 b6085a865762236bb84934161273cdac6dd11c2d + continue + + aslr_shift = int(kerneloffset_str, 16) + kaslr_shift = int(phys_base_str) + aslr_shift + + vollog.debug( + "Linux ASLR shift values found in VMCOREINFO ELF note: physical 0x%x virtual 0x%x", + kaslr_shift, + aslr_shift, + ) + + return kaslr_shift, aslr_shift + + vollog.debug("The vmcoreinfo scanner could not determine any ASLR shifts") + return None + @classmethod def virtual_to_physical_address(cls, addr: int) -> int: """Converts a virtual linux address to a physical one (does not account @@ -192,6 +259,41 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): return addr - 0xFFFFFFFF80000000 return addr - 0xC0000000 + @classmethod + def find_aslr( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Tuple[int, int]: + """Determines the offset of the actual DTB in physical space and its + symbol offset. + Args: + context: The context to retrieve required elements (layers, symbol tables) from + symbol_table: The name of the kernel module on which to operate + layer_name: The layer within the context in which the module exists + progress_callback: A function that takes a percentage (and an optional description) that will be called periodically + + Returns: + kaslr_shirt and aslr_shift + """ + + aslr_shifts = cls.find_aslr_vmcoreinfo( + context, layer_name, progress_callback=progress_callback + ) + if aslr_shifts: + kaslr_shift, aslr_shift = aslr_shifts + else: + # Fallback to the traditional scanner method + kaslr_shift, aslr_shift = cls.find_aslr_classic( + context, + symbol_table, + layer_name, + progress_callback=progress_callback, + ) + return kaslr_shift, aslr_shift + class LinuxSymbolFinder(symbol_finder.SymbolFinder): """Linux symbol loader based on uname signature strings.""" From 6610779e5c070e7b3225b47cbb3a0980caa1881f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 4 Nov 2024 19:30:30 +1100 Subject: [PATCH 533/979] linux: VMCoreInfo: Remove debug comment --- volatility3/framework/automagic/linux.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index fde060733..248bbf04d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -229,7 +229,6 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if phys_base_str is None: # We are in kernel (x86) < 4.10 401721ecd1dcb0a428aa5d6832ee05ffbdbffbbe where it was SYMBOL(phys_base) # It's the symbol address instead of the value itself, which is useless for calculating the physical address. - # raise Exception("Kernel < 4.10") continue kerneloffset_str = vmcoreinfo.get("KERNELOFFSET") From 2efb4e7d28d60a337aabae448de24da37c3feb7e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 5 Nov 2024 19:56:15 +1100 Subject: [PATCH 534/979] Merge branch 'develop' into linux_boottime_support --- .../framework/constants/linux/__init__.py | 8 + .../framework/plugins/linux/hidden_modules.py | 246 ++++++++++++++++++ .../framework/symbols/linux/__init__.py | 6 +- .../symbols/linux/extensions/__init__.py | 225 +++++++++------- 4 files changed, 383 insertions(+), 102 deletions(-) create mode 100644 volatility3/framework/plugins/linux/hidden_modules.py diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 7bfa9268d..6e49e6f37 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -344,3 +344,11 @@ class PT_FLAGS(Flag): # Boot time NSEC_PER_SEC = 1e9 + + +# Valid sizes for modules. Note that the Linux kernel does not define these values; they +# are based on empirical observations of typical memory allocations for kernel modules. +# We use this to verify that the found module falls within reasonable limits. +MODULE_MAXIMUM_CORE_SIZE = 20000000 +MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 +MODULE_MINIMUM_SIZE = 4096 diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py new file mode 100644 index 000000000..fd4b28943 --- /dev/null +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -0,0 +1,246 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import List, Set, Tuple, Iterable +from volatility3.framework import renderers, interfaces, exceptions, objects +from volatility3.framework.constants import architectures +from volatility3.framework.renderers import format_hints +from volatility3.framework.configuration import requirements +from volatility3.plugins.linux import lsmod + +vollog = logging.getLogger(__name__) + + +class Hidden_modules(interfaces.plugins.PluginInterface): + """Carves memory to find hidden kernel modules""" + + _required_framework_version = (2, 10, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + ] + + @staticmethod + def get_modules_memory_boundaries( + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Tuple[int]: + """Determine the boundaries of the module allocation area + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + + Returns: + A tuple containing the minimum and maximum addresses for the module allocation area. + """ + vmlinux = context.modules[vmlinux_module_name] + if vmlinux.has_symbol("mod_tree"): + # Kernel >= 5.19 58d208de3e8d87dbe196caf0b57cc58c7a3836ca + mod_tree = vmlinux.object_from_symbol("mod_tree") + modules_addr_min = mod_tree.addr_min + modules_addr_max = mod_tree.addr_max + elif vmlinux.has_symbol("module_addr_min"): + # 2.6.27 <= kernel < 5.19 3a642e99babe0617febb6f402e1e063479f489db + modules_addr_min = vmlinux.object_from_symbol("module_addr_min") + modules_addr_max = vmlinux.object_from_symbol("module_addr_max") + + if isinstance(modules_addr_min, objects.Void): + raise exceptions.VolatilityException( + "Your ISF symbols lack type information. You may need to update the" + "ISF using the latest version of dwarf2json" + ) + else: + raise exceptions.VolatilityException( + "Cannot find the module memory allocation area. Unsupported kernel" + ) + + return modules_addr_min, modules_addr_max + + @classmethod + def _get_module_address_alignment( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> int: + """Obtain the module memory address alignment. + + struct module is aligned to the L1 cache line, which is typically 64 bytes for most + common i386/AMD64/ARM64 configurations. In some cases, it can be 128 bytes, but this + will still work. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + + Returns: + The struct module alignment + """ + # FIXME: When dwarf2json/ISF supports type alignments. Read it directly from the type metadata + # Additionally, while 'context' and 'vmlinux_module_name' are currently unused, they will be + # essential for retrieving type metadata in the future. + return 64 + + @staticmethod + def _validate_alignment_patterns( + addresses: Iterable[int], + address_alignment: int, + ) -> bool: + """Check if the memory addresses meet our alignments patterns + + Args: + addresses: Iterable with the address values + address_alignment: Number of bytes for alignment validation + + Returns: + True if all the addresses meet the alignment + """ + return all(addr % address_alignment == 0 for addr in addresses) + + @classmethod + def get_hidden_modules( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + known_module_addresses: Set[int], + modules_memory_boundaries: Tuple, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Enumerate hidden modules by taking advantage of memory address alignment patterns + + This technique is much faster and uses less memory than the traditional scan method + in Volatility2, but it doesn't work with older kernels. + + From kernels 4.2 struct module allocation are aligned to the L1 cache line size. + In i386/amd64/arm64 this is typically 64 bytes. However, this can be changed in + the Linux kernel configuration via CONFIG_X86_L1_CACHE_SHIFT. The alignment can + also be obtained from the DWARF info i.e. DW_AT_alignment<64>, but dwarf2json + doesn't support this feature yet. + In kernels < 4.2, alignment attributes are absent in the struct module, meaning + alignment cannot be guaranteed. Therefore, for older kernels, it's better to use + the traditional scan technique. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + known_module_addresses: Set with known module addresses + modules_memory_boundaries: Minimum and maximum address boundaries for module allocation. + Yields: + module objects + """ + vmlinux = context.modules[vmlinux_module_name] + vmlinux_layer = context.layers[vmlinux.layer_name] + + module_addr_min, module_addr_max = modules_memory_boundaries + module_address_alignment = cls._get_module_address_alignment( + context, vmlinux_module_name + ) + if not cls._validate_alignment_patterns( + known_module_addresses, module_address_alignment + ): + vollog.warning( + f"Module addresses aren't aligned to {module_address_alignment} bytes. " + "Switching to 1 byte aligment scan method." + ) + module_address_alignment = 1 + + mkobj_offset = vmlinux.get_type("module").relative_child_offset("mkobj") + mod_offset = vmlinux.get_type("module_kobject").relative_child_offset("mod") + offset_to_mkobj_mod = mkobj_offset + mod_offset + mod_member_template = vmlinux.get_type("module_kobject").child_template("mod") + mod_size = mod_member_template.size + mod_member_data_format = mod_member_template.data_format + + for module_addr in range( + module_addr_min, module_addr_max, module_address_alignment + ): + if module_addr in known_module_addresses: + continue + + try: + # This is just a pre-filter. Module readability and consistency are verified in module.is_valid() + self_referential_bytes = vmlinux_layer.read( + module_addr + offset_to_mkobj_mod, mod_size + ) + self_referential = objects.convert_data_to_value( + self_referential_bytes, int, mod_member_data_format + ) + if self_referential != module_addr: + continue + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ): + continue + + module = vmlinux.object("module", offset=module_addr, absolute=True) + if module and module.is_valid(): + yield module + + @classmethod + def get_lsmod_module_addresses( + cls, + context: interfaces.context.ContextInterface, + vmlinux_module_name: str, + ) -> Set[int]: + """Obtain a set the known module addresses from linux.lsmod plugin + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + vmlinux_module_name: The name of the kernel module on which to operate + + Returns: + A set containing known kernel module addresses + """ + vmlinux = context.modules[vmlinux_module_name] + vmlinux_layer = context.layers[vmlinux.layer_name] + + known_module_addresses = { + vmlinux_layer.canonicalize(module.vol.offset) + for module in lsmod.Lsmod.list_modules(context, vmlinux_module_name) + } + return known_module_addresses + + def _generator(self): + vmlinux_module_name = self.config["kernel"] + known_module_addresses = self.get_lsmod_module_addresses( + self.context, vmlinux_module_name + ) + modules_memory_boundaries = self.get_modules_memory_boundaries( + self.context, vmlinux_module_name + ) + for module in self.get_hidden_modules( + self.context, + vmlinux_module_name, + known_module_addresses, + modules_memory_boundaries, + ): + module_addr = module.vol.offset + module_name = module.get_name() or renderers.NotAvailableValue() + fields = (format_hints.Hex(module_addr), module_name) + yield (0, fields) + + def run(self): + if self.context.symbol_space.verify_table_versions( + "dwarf2json", lambda version, _: (not version) or version < (0, 8, 0) + ): + raise exceptions.SymbolSpaceError( + "Invalid symbol table, please ensure the ISF table produced by dwarf2json was created with version 0.8.0 or later" + ) + + headers = [ + ("Address", format_hints.Hex), + ("Name", str), + ] + return renderers.TreeGrid(headers, self._generator()) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 1f2812c10..3289775b6 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -615,7 +615,7 @@ class IDStorage(ABC): return nodep - def _iter_node(self, nodep, height) -> int: + def _iter_node(self, nodep, height) -> Iterator[int]: node = self.nodep_to_node(nodep) node_slots = node.slots for off in range(self.CHUNK_SIZE): @@ -632,7 +632,7 @@ class IDStorage(ABC): for child_node in self._iter_node(nodep, height - 1): yield child_node - def get_entries(self, root: interfaces.objects.ObjectInterface) -> int: + def get_entries(self, root: interfaces.objects.ObjectInterface) -> Iterator[int]: """Walks the tree data structure Args: @@ -818,7 +818,7 @@ class PageCache(object): self._page_cache = page_cache self._idstorage = IDStorage.choose_id_storage(context, kernel_module_name) - def get_cached_pages(self) -> interfaces.objects.ObjectInterface: + def get_cached_pages(self) -> Iterator[interfaces.objects.ObjectInterface]: """Returns all page cache contents Yields: diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index a9127907e..0099b17e1 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -14,12 +14,7 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion -from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY -from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS -from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS -from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES -from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES -from volatility3.framework.constants.linux import CAPABILITIES, PT_FLAGS, NSEC_PER_SEC +from volatility3.framework.constants import linux as linux_constants from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed @@ -33,112 +28,140 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._mod_mem_type = None # Initialize _mod_mem_type to None for memoization + def is_valid(self): + """Determine whether it is a valid module object by verifying the self-referential + in module_kobject. This also confirms that the module is actively allocated and + not a remnant of freed memory or a failed module load attempt by verifying the + module memory section sizes. + """ + layer = self._context.layers[self.vol.layer_name] + # Make sure the entire module content is readable + if not layer.is_valid(self.vol.offset, self.vol.size): + return False - @property - def mod_mem_type(self): + core_size = self.get_core_size() + core_text_size = self.get_core_text_size() + init_size = self.get_init_size() + if not ( + 0 < core_text_size <= linux_constants.MODULE_MAXIMUM_CORE_TEXT_SIZE + and 0 < core_size <= linux_constants.MODULE_MAXIMUM_CORE_SIZE + and core_size + init_size >= linux_constants.MODULE_MINIMUM_SIZE + ): + return False + + if not ( + self.mkobj + and self.mkobj.mod + and self.mkobj.mod.is_readable() + and self.mkobj.mod == self.vol.offset + ): + return False + + return True + + @functools.cached_property + def mod_mem_type(self) -> Dict: """Return the mod_mem_type enum choices if available or an empty dict if not""" # mod_mem_type and module_memory were added in kernel 6.4 which replaces # module_layout for storing the information around core_layout etc. # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information + symbol_table_name = self.get_symbol_table_name() + mod_mem_type_symname = symbol_table_name + constants.BANG + "mod_mem_type" + symbol_space = self._context.symbol_space + try: + mod_mem_type = symbol_space.get_enumeration(mod_mem_type_symname).choices + except exceptions.SymbolError: + mod_mem_type = {} + vollog.debug( + "Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" + ) - if self._mod_mem_type is None: - try: - self._mod_mem_type = self._context.symbol_space.get_enumeration( - self.get_symbol_table_name() + constants.BANG + "mod_mem_type" - ).choices - except exceptions.SymbolError: - vollog.debug( - "Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" - ) - # set to empty dict to show that the enum was not found, and so shouldn't be searched for again - self._mod_mem_type = {} - return self._mod_mem_type + return mod_mem_type + + def _get_mem_type(self, mod_mem_type_name): + module_mem_index = self.mod_mem_type.get(mod_mem_type_name) + if module_mem_index is None: + raise AttributeError(f"Unknown module memory type '{mod_mem_type_name}'") + + if not (0 <= module_mem_index < self.mem.count): + raise AttributeError( + f"Invalid module memory type index '{module_mem_index}'" + ) + + return self.mem[module_mem_index] + + def _get_mem_size(self, mod_mem_type_name): + return self._get_mem_type(mod_mem_type_name).size + + def _get_mem_base(self, mod_mem_type_name): + return self._get_mem_type(mod_mem_type_name).base def get_module_base(self): if self.has_member("mem"): # kernels 6.4+ - try: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base - except KeyError: - raise AttributeError( - "module -> get_module_base: Unable to get module base. Cannot read base from MOD_TEXT." - ) + return self._get_mem_base("MOD_TEXT") elif self.has_member("core_layout"): return self.core_layout.base elif self.has_member("module_core"): return self.module_core - raise AttributeError("module -> get_module_base: Unable to get module base") + + raise AttributeError("Unable to get module base") def get_init_size(self): if self.has_member("mem"): # kernels 6.4+ - try: - return ( - self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size - + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size - ) - except KeyError: - raise AttributeError( - "module -> get_init_size: Unable to determine .init section size of module. Cannot read size of MOD_INIT_TEXT, MOD_INIT_DATA, and MOD_INIT_RODATA" - ) + return ( + self._get_mem_size("MOD_INIT_TEXT") + + self._get_mem_size("MOD_INIT_DATA") + + self._get_mem_size("MOD_INIT_RODATA") + ) elif self.has_member("init_layout"): return self.init_layout.size elif self.has_member("init_size"): return self.init_size - raise AttributeError( - "module -> get_init_size: Unable to determine .init section size of module" - ) + + raise AttributeError("Unable to determine .init section size of module") def get_core_size(self): if self.has_member("mem"): # kernels 6.4+ - try: - return ( - self.mem[self.mod_mem_type["MOD_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_DATA"]].size - + self.mem[self.mod_mem_type["MOD_RODATA"]].size - + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size - ) - except KeyError: - raise AttributeError( - "module -> get_core_size: Unable to determine core size of module. Cannot read size of MOD_TEXT, MOD_DATA, MOD_RODATA, and MOD_RO_AFTER_INIT." - ) + return ( + self._get_mem_size("MOD_TEXT") + + self._get_mem_size("MOD_DATA") + + self._get_mem_size("MOD_RODATA") + + self._get_mem_size("MOD_RO_AFTER_INIT") + ) elif self.has_member("core_layout"): return self.core_layout.size elif self.has_member("core_size"): return self.core_size - raise AttributeError( - "module -> get_core_size: Unable to determine core size of module" - ) + + raise AttributeError("Unable to determine core size of module") + + def get_core_text_size(self): + if self.has_member("mem"): # kernels 6.4+ + return self._get_mem_size("MOD_TEXT") + elif self.has_member("core_layout"): + return self.core_layout.text_size + elif self.has_member("core_text_size"): + return self.core_text_size + + raise AttributeError("Unable to determine core text size of module") def get_module_core(self): if self.has_member("mem"): # kernels 6.4+ - try: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base - except KeyError: - raise AttributeError( - "module -> get_module_core: Unable to get module core. Cannot read base from MOD_TEXT." - ) + return self._get_mem_base("MOD_TEXT") elif self.has_member("core_layout"): return self.core_layout.base elif self.has_member("module_core"): return self.module_core - raise AttributeError("module -> get_module_core: Unable to get module core") + raise AttributeError("Unable to get module core") def get_module_init(self): if self.has_member("mem"): # kernels 6.4+ - try: - return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base - except KeyError: - raise AttributeError( - "module -> get_module_core: Unable to get module init. Cannot read base from MOD_INIT_TEXT." - ) + return self._get_mem_base("MOD_INIT_TEXT") elif self.has_member("init_layout"): return self.init_layout.base elif self.has_member("module_init"): return self.module_init - raise AttributeError("module -> get_module_init: Unable to get module init") + raise AttributeError("Unable to get module init") def get_name(self): """Get the name of the module as a string""" @@ -340,7 +363,7 @@ class task_struct(generic.GenericIntelProcess): Returns: bool: True, if this task is a kernel thread. Otherwise, False. """ - return (self.flags & constants.linux.PF_KTHREAD) != 0 + return (self.flags & linux_constants.PF_KTHREAD) != 0 @property def is_thread_group_leader(self) -> bool: @@ -417,7 +440,11 @@ class task_struct(generic.GenericIntelProcess): def get_ptrace_tracee_flags(self) -> Optional[str]: """Returns a string with the ptrace flags""" - return PT_FLAGS(self.ptrace).flags if self.is_being_ptraced else None + return ( + linux_constants.PT_FLAGS(self.ptrace).flags + if self.is_being_ptraced + else None + ) def _get_task_start_time(self) -> datetime.timedelta: """Returns the task's monotonic start_time as a timedelta. @@ -1492,7 +1519,7 @@ class vfsmount(objects.StructType): bool: 'True' if the given argument points to the the same 'vfsmount' as 'self'. """ - if type(vfsmount_ptr) == objects.Pointer: + if isinstance(vfsmount_ptr, objects.Pointer): return self.vol.offset == vfsmount_ptr else: raise exceptions.VolatilityException( @@ -1715,18 +1742,18 @@ class socket(objects.StructType): def get_state(self): socket_state_idx = self.state - if 0 <= socket_state_idx < len(SOCKET_STATES): - return SOCKET_STATES[socket_state_idx] + if 0 <= socket_state_idx < len(linux_constants.SOCKET_STATES): + return linux_constants.SOCKET_STATES[socket_state_idx] class sock(objects.StructType): def get_family(self): family_idx = self.__sk_common.skc_family - if 0 <= family_idx < len(SOCK_FAMILY): - return SOCK_FAMILY[family_idx] + if 0 <= family_idx < len(linux_constants.SOCK_FAMILY): + return linux_constants.SOCK_FAMILY[family_idx] def get_type(self): - return SOCK_TYPES.get(self.sk_type, "") + return linux_constants.SOCK_TYPES.get(self.sk_type, "") def get_inode(self): if not self.sk_socket: @@ -1760,8 +1787,8 @@ class unix_sock(objects.StructType): # Unix socket states reuse (a subset) of the inet_sock states contants if self.sk.get_type() == "STREAM": state_idx = self.sk.__sk_common.skc_state - if 0 <= state_idx < len(TCP_STATES): - return TCP_STATES[state_idx] + if 0 <= state_idx < len(linux_constants.TCP_STATES): + return linux_constants.TCP_STATES[state_idx] else: # Return the generic socket state return self.sk.sk_socket.get_state() @@ -1773,15 +1800,15 @@ class unix_sock(objects.StructType): class inet_sock(objects.StructType): def get_family(self): family_idx = self.sk.__sk_common.skc_family - if 0 <= family_idx < len(SOCK_FAMILY): - return SOCK_FAMILY[family_idx] + if 0 <= family_idx < len(linux_constants.SOCK_FAMILY): + return linux_constants.SOCK_FAMILY[family_idx] def get_protocol(self): # If INET6 family and a proto is defined, we use that specific IPv6 protocol. # Otherwise, we use the standard IP protocol. - protocol = IP_PROTOCOLS.get(self.sk.sk_protocol) + protocol = linux_constants.IP_PROTOCOLS.get(self.sk.sk_protocol) if self.get_family() == "AF_INET6": - protocol = IPV6_PROTOCOLS.get(self.sk.sk_protocol, protocol) + protocol = linux_constants.IPV6_PROTOCOLS.get(self.sk.sk_protocol, protocol) return protocol def get_state(self): @@ -1789,8 +1816,8 @@ class inet_sock(objects.StructType): if self.sk.get_type() == "STREAM": state_idx = self.sk.__sk_common.skc_state - if 0 <= state_idx < len(TCP_STATES): - return TCP_STATES[state_idx] + if 0 <= state_idx < len(linux_constants.TCP_STATES): + return linux_constants.TCP_STATES[state_idx] else: # Return the generic socket state return self.sk.sk_socket.get_state() @@ -1873,8 +1900,8 @@ class inet_sock(objects.StructType): class netlink_sock(objects.StructType): def get_protocol(self): protocol_idx = self.sk.sk_protocol - if 0 <= protocol_idx < len(NETLINK_PROTOCOLS): - return NETLINK_PROTOCOLS[protocol_idx] + if 0 <= protocol_idx < len(linux_constants.NETLINK_PROTOCOLS): + return linux_constants.NETLINK_PROTOCOLS[protocol_idx] def get_state(self): # Return the generic socket state @@ -1916,8 +1943,8 @@ class packet_sock(objects.StructType): eth_proto = socket_module.htons(self.num) if eth_proto == 0: return None - elif eth_proto in ETH_PROTOCOLS: - return ETH_PROTOCOLS[eth_proto] + elif eth_proto in linux_constants.ETH_PROTOCOLS: + return linux_constants.ETH_PROTOCOLS[eth_proto] else: return f"0x{eth_proto:x}" @@ -1929,13 +1956,13 @@ class packet_sock(objects.StructType): class bt_sock(objects.StructType): def get_protocol(self): type_idx = self.sk.sk_protocol - if 0 <= type_idx < len(BLUETOOTH_PROTOCOLS): - return BLUETOOTH_PROTOCOLS[type_idx] + if 0 <= type_idx < len(linux_constants.BLUETOOTH_PROTOCOLS): + return linux_constants.BLUETOOTH_PROTOCOLS[type_idx] def get_state(self): state_idx = self.sk.__sk_common.skc_state - if 0 <= state_idx < len(BLUETOOTH_STATES): - return BLUETOOTH_STATES[state_idx] + if 0 <= state_idx < len(linux_constants.BLUETOOTH_STATES): + return linux_constants.BLUETOOTH_STATES[state_idx] class xdp_sock(objects.StructType): @@ -2053,7 +2080,7 @@ class kernel_cap_struct(objects.StructType): Returns: int: The latest capability ID supported by the framework. """ - return len(CAPABILITIES) - 1 + return len(linux_constants.CAPABILITIES) - 1 def get_kernel_cap_full(self) -> int: """Return the maximum value allowed for this kernel for a capability @@ -2082,7 +2109,7 @@ class kernel_cap_struct(objects.StructType): """ capabilities = [] - for bit, name in enumerate(CAPABILITIES): + for bit, name in enumerate(linux_constants.CAPABILITIES): if capabilities_bitfield & (1 << bit) != 0: capabilities.append(name) @@ -2143,10 +2170,10 @@ class kernel_cap_struct(objects.StructType): Returns: bool: "True" if the given capability is enabled. """ - if capability not in CAPABILITIES: + if capability not in linux_constants.CAPABILITIES: raise AttributeError(f"Unknown capability with name '{capability}'") - cap_value = 1 << CAPABILITIES.index(capability) + cap_value = 1 << linux_constants.CAPABILITIES.index(capability) return cap_value & self.get_capabilities() != 0 From c0fa2cfcd67d676dab100150daad2f93948c641b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 5 Nov 2024 20:02:55 +1100 Subject: [PATCH 535/979] Linux: Boottime API: User linux_constanst import --- .../symbols/linux/extensions/__init__.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0099b17e1..d05c34304 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2217,12 +2217,12 @@ class Timespec64Abstract(abc.ABC): # Based on ns_to_timespec64() if nsec > 0: - tv_sec = nsec // NSEC_PER_SEC - tv_nsec = nsec % NSEC_PER_SEC + tv_sec = nsec // linux_constants.NSEC_PER_SEC + tv_nsec = nsec % linux_constants.NSEC_PER_SEC elif nsec < 0: - tv_sec = -((-nsec - 1) // NSEC_PER_SEC) - 1 - rem = (-nsec - 1) % NSEC_PER_SEC - tv_nsec = NSEC_PER_SEC - rem - 1 + tv_sec = -((-nsec - 1) // linux_constants.NSEC_PER_SEC) - 1 + rem = (-nsec - 1) % linux_constants.NSEC_PER_SEC + tv_nsec = linux_constants.NSEC_PER_SEC - rem - 1 else: tv_sec = tv_nsec = 0 @@ -2233,13 +2233,15 @@ class Timespec64Abstract(abc.ABC): # pylint: disable=E1101 return conversion.unixtime_to_datetime( - self.tv_sec + self.tv_nsec / NSEC_PER_SEC + self.tv_sec + self.tv_nsec / linux_constants.NSEC_PER_SEC ) def to_timedelta(self) -> datetime.timedelta: """Converts this Timespec64Abstract subclass object to timedelta""" # pylint: disable=E1101 - return datetime.timedelta(seconds=self.tv_sec + self.tv_nsec / NSEC_PER_SEC) + return datetime.timedelta( + seconds=self.tv_sec + self.tv_nsec / linux_constants.NSEC_PER_SEC + ) def __add__(self, other) -> "Timespec64Concrete": """Returns a new Timespec64Concrete object that sums the current values with those @@ -2282,12 +2284,12 @@ class Timespec64Abstract(abc.ABC): # Based on kernel's set_normalized_timespec64() # pylint: disable=E1101 - while self.tv_nsec >= NSEC_PER_SEC: - self.tv_nsec -= NSEC_PER_SEC + while self.tv_nsec >= linux_constants.NSEC_PER_SEC: + self.tv_nsec -= linux_constants.NSEC_PER_SEC self.tv_sec += 1 while self.tv_nsec < 0: - self.tv_nsec += NSEC_PER_SEC + self.tv_nsec += linux_constants.NSEC_PER_SEC self.tv_sec -= 1 From 0ad8054f125c360afadcd25db70c1f67d8265e57 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 7 Nov 2024 20:26:26 +1100 Subject: [PATCH 536/979] intel layer: minor improve lru_cache argument --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 2b0df5372..33d5432cc 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -252,7 +252,7 @@ class Intel(linear.LinearlyMappedLayer): return entry, position - @functools.lru_cache(1025) + @functools.lru_cache(maxsize=1025) def _get_valid_table(self, base_address: int) -> Optional[bytes]: """Extracts the table, validates it and returns it if it's valid.""" table = self._context.layers.read( From 9ae7c2bb109681bbd8b6cffc5a1f5b7f1dfa71cf Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 7 Nov 2024 20:28:21 +1100 Subject: [PATCH 537/979] Data layer interface: Convert address_mask to a cached property --- volatility3/framework/interfaces/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index e2a68780a..78687d8d5 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -136,7 +136,7 @@ class DataLayerInterface( def minimum_address(self) -> int: """Returns the minimum valid address of the space.""" - @property + @functools.cached_property def address_mask(self) -> int: """Returns a mask which encapsulates all the active bits of an address for this layer.""" From 73d4f2fc88e15ac038f962cb5f5b11acf7855091 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 7 Nov 2024 20:31:20 +1100 Subject: [PATCH 538/979] Linux: Add support for PROT_NONE, Intel Side Channel Vulnerability L1TF changes and fix _maxphyaddr in x86-64 --- volatility3/framework/automagic/linux.py | 6 +- .../framework/constants/linux/__init__.py | 8 ++ volatility3/framework/layers/intel.py | 100 ++++++++++++++++-- 3 files changed, 105 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 52a73f45a..27e07e564 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -80,14 +80,14 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): context, table_name, layer_name, progress_callback=progress_callback ) - layer_class: Type = intel.Intel if "init_top_pgt" in table.symbols: - layer_class = intel.Intel32e + layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_top_pgt" elif "init_level4_pgt" in table.symbols: - layer_class = intel.Intel32e + layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" else: + layer_class = intel.LinuxIntel dtb_symbol_name = "swapper_pg_dir" dtb = cls.virtual_to_physical_address( diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 7c485d3c3..303c534ed 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -11,6 +11,14 @@ KERNEL_NAME = "__kernel__" """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" +# Translation Layer constants +PAGE_BIT_PRESENT = 0 +PAGE_BIT_PSE = 7 # Page Size Extension: 4 MB (or 2MB) page +PAGE_BIT_PROTNONE = 8 +PAGE_BIT_PAT_LARGE = 12 # 2MB or 1GB pages +PAGE_PRESENT = 1 << PAGE_BIT_PRESENT +PAGE_PROTNONE = 1 << PAGE_BIT_PROTNONE + # include/linux/sched.h PF_KTHREAD = 0x00200000 # I'm a kernel thread diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 33d5432cc..e6d20d992 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -13,6 +13,7 @@ from volatility3 import classproperty from volatility3.framework import exceptions, interfaces, constants from volatility3.framework.configuration import requirements from volatility3.framework.layers import linear +from volatility3.framework.constants import linux as linux_constants vollog = logging.getLogger(__name__) @@ -163,12 +164,17 @@ class Intel(linear.LinearlyMappedLayer): entry, f"Page Fault at entry {hex(entry)} in page entry", ) - page = self._mask(entry, self._maxphyaddr - 1, position + 1) | self._mask( - offset, position, 0 - ) + + pfn = self.pte_pfn(entry) + page_offset = self._mask(offset, position, 0) + page = pfn << self.page_shift | page_offset return page, 1 << (position + 1), self._base_layer + def pte_pfn(self, entry: int) -> int: + """Extracts the page frame number (PFN) from the page table entry (PTE) entry""" + return entry >> self.page_shift + def _translate_entry(self, offset: int) -> Tuple[int, int]: """Translates a specific offset based on paging tables. @@ -203,10 +209,10 @@ class Intel(linear.LinearlyMappedLayer): "Page Fault at entry " + hex(entry) + " in table " + name, ) # Check if we're a large page - if large_page and (entry & (1 << 7)): + if large_page and (entry & (1 << linux_constants.PAGE_BIT_PSE)): # Mask off the PAT bit - if entry & (1 << 12): - entry -= 1 << 12 + if entry & (1 << linux_constants.PAGE_BIT_PAT_LARGE): + entry -= 1 << linux_constants.PAGE_BIT_PAT_LARGE # We're a large page, the rest is finished below # If we want to implement PSE-36, it would need to be done here break @@ -501,3 +507,85 @@ class WindowsIntel32e(WindowsMixin, Intel32e): def _translate(self, offset: int) -> Tuple[int, int, str]: return self._translate_swap(self, offset, self._bits_per_register // 2) + + +class LinuxMixin(Intel): + @functools.cached_property + def register_mask(self) -> int: + return (1 << self._bits_per_register) - 1 + + @functools.cached_property + def physical_mask(self) -> int: + # From kernels 4.18 the physical mask is dynamic: See AMD SME, Intel Multi-Key Total + # Memory Encryption and CONFIG_DYNAMIC_PHYSICAL_MASK: 94d49eb30e854c84d1319095b5dd0405a7da9362 + physical_mask = (1 << self._maxphyaddr) - 1 + # TODO: Come back once SME support is available in the framework + return physical_mask + + @functools.cached_property + def page_mask(self) -> int: + # Note that within the Intel class it's a class method. However, since it uses + # complement operations and we are working in Python, it would be more careful to + # limit it to the architecture's pointer size. + return ~(self.page_size - 1) & self.register_mask + + @functools.cached_property + def physical_page_mask(self) -> int: + return self.page_mask & self.physical_mask + + @functools.cached_property + def pte_pfn_mask(self) -> int: + return self.physical_page_mask + + @functools.cached_property + def pte_flags_mask(self) -> int: + return ~self.pte_pfn_mask & self.register_mask + + def pte_flags(self, pte) -> int: + return pte & self.pte_flags_mask + + def is_pte_present(self, entry: int) -> bool: + return ( + self.pte_flags(entry) + & (linux_constants.PAGE_PRESENT | linux_constants.PAGE_PROTNONE) + ) != 0 + + def _page_is_valid(self, entry: int) -> bool: + # Overrides the Intel static method with the Linux-specific implementation + return self.is_pte_present(entry) + + def pte_needs_invert(self, entry) -> bool: + # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted + return not (entry & linux_constants.PAGE_PRESENT) + + def protnone_mask(self, entry: int) -> int: + """Gets a mask to XOR with the page table entry to get the correct PFN""" + return ~0 & self.register_mask if self.pte_needs_invert(entry) else 0 + + def pte_pfn(self, entry: int) -> int: + """Extracts the page frame number from the page table entry""" + pfn = entry ^ self.protnone_mask(entry) + return (pfn & self.pte_pfn_mask) >> self.page_shift + + +class LinuxIntel(LinuxMixin, Intel): + pass + + +class LinuxIntelPAE(LinuxMixin, IntelPAE): + pass + + +class LinuxIntel32e(LinuxMixin, Intel32e): + # In the Linux kernel, the __PHYSICAL_MASK_SHIFT is a mask used to extract the + # physical address from a PTE. In Volatility3, this is referred to as _maxphyaddr. + # + # Until kernel version 4.17, Linux x86-64 used a 46-bit mask. With commit + # b83ce5ee91471d19c403ff91227204fb37c95fb2, this was extended to 52 bits, + # applying to both 4 and 5-level page tables. + # + # We initially used 52 bits for all Intel 64-bit systems, but this produced incorrect + # results for PROT_NONE pages. Since the mask value is defined by a preprocessor macro, + # it's difficult to detect the exact bit shift used in the current kernel. + # Using 46 bits has proven reliable for our use case, as seen in tools like crashtool. + _maxphyaddr = 46 From a5253ac2767ca3e0051dcc3a78de7d9b6a7589b8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 7 Nov 2024 13:40:18 +0100 Subject: [PATCH 539/979] re-trigger black --- volatility3/framework/plugins/windows/malfind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 5ce2279cc..1df090b5b 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -140,7 +140,7 @@ class Malfind(interfaces.plugins.PluginInterface): dirty_page_check = True break except exceptions.InvalidAddressException: - # Abort as it is likely that other addresses in the same range will also fail + # Abort as it is likely that other addresses in the same range will also fail. break if not dirty_page_check: continue From 20317e0d13d049070201562a1617ea89256879dc Mon Sep 17 00:00:00 2001 From: Joren Vrancken Date: Thu, 7 Nov 2024 18:05:36 +0100 Subject: [PATCH 540/979] Improve windows.amcache plugin description --- volatility3/framework/plugins/windows/amcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/amcache.py b/volatility3/framework/plugins/windows/amcache.py index 8d0a4769e..1e918d61c 100644 --- a/volatility3/framework/plugins/windows/amcache.py +++ b/volatility3/framework/plugins/windows/amcache.py @@ -215,7 +215,7 @@ def _get_datetime_str_value( class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): - """Scans for windows services.""" + """Extract information on executed applications from the AmCache.""" _required_framework_version = (2, 0, 0) _version = (1, 0, 0) From c7259037356fde4cf6120acddecbae4af16c9ea0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 7 Nov 2024 18:08:58 +0100 Subject: [PATCH 541/979] introduce scatter-gather scatterlists --- .../framework/symbols/linux/__init__.py | 1 + .../symbols/linux/extensions/__init__.py | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3289775b6..e8a3d5240 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -43,6 +43,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.optional_set_type_class("bpf_prog_aux", extensions.bpf_prog_aux) self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) + self.optional_set_type_class("scatterlist", extensions.scatterlist) # kernels >= 4.18 self.optional_set_type_class("timespec64", extensions.timespec64) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index aa3e8c675..0dd657372 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2410,3 +2410,107 @@ class rb_root(objects.StructType): """ yield from self._walk_nodes(root_node=self.rb_node) + + +class scatterlist(objects.StructType): + SG_CHAIN = 0x01 + SG_END = 0x02 + SG_PAGE_LINK_MASK = SG_CHAIN | SG_END + + def _sg_flags(self) -> int: + return self.page_link & self.SG_PAGE_LINK_MASK + + def _sg_is_chain(self) -> int: + return self._sg_flags() & self.SG_CHAIN + + def _sg_is_last(self) -> int: + return self._sg_flags() & self.SG_END + + def _sg_chain_ptr(self) -> int: + """Clears the last two bits basically.""" + return self.page_link & ~self.SG_PAGE_LINK_MASK + + def _sg_dma_len(self) -> int: + # Depends on CONFIG_NEED_SG_DMA_LENGTH + if self.has_member("dma_length"): + return self.dma_length + return self.length + + def _get_sg_max_single_alloc(self) -> int: + """Based on kernel's SG_MAX_SINGLE_ALLOC. + + Doc. from kernel source : + * Maximum number of entries that will be allocated in one piece, if + * a list larger than this is required then chaining will be utilized. + """ + return self._context.layers[self.vol.layer_name].page_size // self.vol.size + + def _sg_next(self) -> interfaces.objects.ObjectInterface: + """Get the next scatterlist struct from the list. + Based on kernel's sg_next. + + Doc. from kernel source : + * Notes on SG table design. + * + * We use the unsigned long page_link field in the scatterlist struct to place + * the page pointer AND encode information about the sg table as well. The two + * lower bits are reserved for this information. + * + * If bit 0 is set, then the page_link contains a pointer to the next sg + * table list. Otherwise the next entry is at sg + 1. + * + * If bit 1 is set, then this sg entry is the last element in a list. + """ + if self._sg_is_last(): + return None + + if self._sg_is_chain(): + next_address = self._sg_chain_ptr() + else: + next_address = self.vol.offset + self.vol.size + + sg = self._context.object( + self.get_symbol_table_name() + constants.BANG + "scatterlist", + self.vol.layer_name, + next_address, + ) + return sg + + def for_each_sg(self) -> Iterator[interfaces.objects.ObjectInterface]: + """Iterate over each struct in the scatterlist.""" + sg = self + sg_max_single_alloc = self._get_sg_max_single_alloc() + + # Empty scatterlists protection + if sg.page_link == 0 and sg._sg_dma_len() == 0 and sg.dma_address == 0: + return None + else: + # Yield itself first + yield sg + + entries_count = 1 + # entries_count <= sg_max_single_alloc should always be true if the + # scatterlists were correctly chained. + while entries_count <= sg_max_single_alloc: + sg = sg._sg_next() + if sg is None: + break + # Points to a new scatterlist + elif sg._sg_is_chain(): + entries_count = 0 + else: + entries_count += 1 + yield sg + + def get_content( + self, + ) -> Iterator[bytes]: + """Traverse a scatterlist to gather content located at each + dma_address position. + + Returns: + An iterator of bytes + """ + physical_layer = self._context.layers["memory_layer"] + for sg in self.for_each_sg(): + yield from physical_layer.read(sg.dma_address, sg._sg_dma_len()) From 6494260d916838c8f1a854450e217049dff015bb Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 7 Nov 2024 23:33:52 +0000 Subject: [PATCH 542/979] Core: Limit capstone to compatible versions --- requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bd61edf14..e0d366391 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,8 @@ yara-python>=3.8.0 # This is required for several plugins that perform malware analysis and disassemble code. # It can also improve accuracy of Windows 8 and later memory samples. -capstone>=3.0.5 +# FIXME: Version 6.0.0 is incompatible (#1336) so we'll need an adaptor at some point +capstone>=3.0.5,<6.0.0 # This is required by plugins that decrypt passwords, password hashes, etc. pycryptodome @@ -19,4 +20,4 @@ leechcorepyc>=2.4.0; sys_platform != 'darwin' # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage gcsfs>=2023.1.0 -s3fs>=2023.1.0 \ No newline at end of file +s3fs>=2023.1.0 From 74cf394ad0adb65d9cc149dae0451d7bee3d76a8 Mon Sep 17 00:00:00 2001 From: Joren Vrancken Date: Fri, 8 Nov 2024 10:06:59 +0100 Subject: [PATCH 543/979] Print plugin description on plugin --help --- volatility3/cli/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index b4e993dac..75b62abf6 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -367,7 +367,9 @@ class CommandLine: ) for plugin in sorted(plugin_list): plugin_parser = subparser.add_parser( - plugin, help=plugin_list[plugin].__doc__ + plugin, + help=plugin_list[plugin].__doc__, + description=plugin_list[plugin].__doc__, ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) From b37ba26dd62a9cc85c89e174c20f1afe549eb753 Mon Sep 17 00:00:00 2001 From: Joren Vrancken Date: Fri, 8 Nov 2024 10:19:05 +0100 Subject: [PATCH 544/979] Fix typo in SuspiciousThreads --- volatility3/framework/plugins/windows/suspicious_threads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspicious_threads.py b/volatility3/framework/plugins/windows/suspicious_threads.py index 2679b191a..4bfb6baa5 100644 --- a/volatility3/framework/plugins/windows/suspicious_threads.py +++ b/volatility3/framework/plugins/windows/suspicious_threads.py @@ -13,11 +13,11 @@ from volatility3.plugins.windows import pslist, threads, vadinfo, thrdscan vollog = logging.getLogger(__name__) -class SupsiciousThreads(interfaces.plugins.PluginInterface): +class SuspiciousThreads(interfaces.plugins.PluginInterface): """Lists suspicious userland process threads""" _required_framework_version = (2, 4, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 1fd51772600dc14ad6893e76362c9b8f599be180 Mon Sep 17 00:00:00 2001 From: lesander <4174509+lesander@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:33:20 +0100 Subject: [PATCH 545/979] move dependency definitions to pyproject.toml and bump minimal cpython version to 3.9.0 since 3.8.0 is EOL. --- README.md | 48 +++++++++++++--------------------------- mypy.ini | 4 ---- pyproject.toml | 42 ++++++++++++++++++++++++++++++----- requirements-dev.txt | 9 -------- requirements-minimal.txt | 2 -- requirements.txt | 23 ------------------- setup.py | 24 -------------------- 7 files changed, 51 insertions(+), 101 deletions(-) delete mode 100644 mypy.ini delete mode 100644 requirements-dev.txt delete mode 100644 requirements-minimal.txt delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/README.md b/README.md index 1463c2bde..78eb7c491 100644 --- a/README.md +++ b/README.md @@ -18,62 +18,44 @@ the Volatility Software License (VSL). See the [LICENSE](https://www.volatilityfoundation.org/license/vsl-v1.0) file for more details. -## Requirements +## Installing -Volatility 3 requires Python 3.8.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.9.0 or later and is published on the [PyPi registry](https://pypi.org/project/volatility3). ```shell -pip3 install -r requirements-minimal.txt +pip install volatility3 ``` -Alternately, the minimal packages will be installed automatically when Volatility 3 is installed using pip. However, as noted in the Quick Start section below, Volatility 3 does not *need* to be installed prior to using it. +If you want to use the latest development version of Volatility 3 we recommend you manually clone this repository and install an editable version of the project. +We recommend you use a virtual environment to keep installed dependencies separate from system packages. -```shell -pip3 install . -``` - -To enable the full range of Volatility 3 functionality, use a command like the one below. For partial functionality, comment out any unnecessary packages in [requirements.txt](requirements.txt) prior to running the command. - -```shell -pip3 install -r requirements.txt -``` - -## Downloading Volatility - -The latest stable version of Volatility will always be the stable branch of the GitHub repository. You can get the latest version of the code using the following command: +The latest stable version of Volatility will always be the `stable` branch of the GitHub repository. The default branch is `develop`. ```shell git clone https://github.com/volatilityfoundation/volatility3.git +cd volatility3/ +python3 -m venv venv && . venv/bin/activate +pip install -e .[dev] ``` ## Quick Start -1. Clone the latest version of Volatility from GitHub: - - ```shell - git clone https://github.com/volatilityfoundation/volatility3.git - ``` +1. Install Volatility 3 as documented in the Installing section of the readme. 2. See available options: ```shell - python3 vol.py -h + vol -h ``` -3. To get more information on a Windows memory sample and to make sure -Volatility supports that sample type, run -`python3 vol.py -f windows.info` - - Example: +3. To get more information on a Windows memory sample and to make sure Volatility supports that sample type, run `vol -f windows.info`: ```shell - python3 vol.py -f /home/user/samples/stuxnet.vmem windows.info + vol -f /home/user/samples/stuxnet.vmem windows.info ``` -4. Run some other plugins. The `-f` or `--single-location` is not strictly -required, but most plugins expect a single sample. Some also -require/accept other options. Run `python3 vol.py -h` -for more information on a particular command. +4. Run some other plugins. The `-f` or `--single-location` is not strictly required, but most plugins expect a single sample. +Some also require/accept other options. Run `vol -h` for more information on a particular command. ## Symbol Tables diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 6fb9f9ed3..000000000 --- a/mypy.ini +++ /dev/null @@ -1,4 +0,0 @@ -[mypy] -mypy_path = ./stubs -show_traceback = True -ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 2e1636a43..91848e1fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,15 +6,39 @@ readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, ] -requires-python = ">=3.8.0" +requires-python = ">=3.9.0" license = { text = "VSL" } dynamic = ["dependencies", "optional-dependencies", "version"] +dependencies = [ + "pefile>=2024.8.26", +] + +[project.optional-dependencies] +full = [ + "yara-python>=4.5.1,<5", + "capstone>=5.0.3,<6", + "pycryptodome>=3.21.0,<4", + "leechcorepyc>=2.19.2,<3; sys_platform != 'darwin'", +] + +cloud = [ + "gcsfs>=2024.10.0", + "s3fs>=2024.10.0", +] + +dev = [ + "volatility3[full,cloud]", + "jsonschema>=4.23.0,<5", + "pyinstaller>=6.11.0,<7", + "pyinstaller-hooks-contrib>=2024.9", +] + [project.urls] -Homepage = "https://github.com/volatilityfoundation/volatility3/" -"Bug Tracker" = "https://github.com/volatilityfoundation/volatility3/issues" -Documentation = "https://volatility3.readthedocs.io/" -"Source Code" = "https://github.com/volatilityfoundation/volatility3" +homepage = "https://github.com/volatilityfoundation/volatility3/" +documentation = "https://volatility3.readthedocs.io/" +repository = "https://github.com/volatilityfoundation/volatility3" +issues = "https://github.com/volatilityfoundation/volatility3/issues" [project.scripts] vol = "volatility3.cli:main" @@ -22,11 +46,17 @@ volshell = "volatility3.cli.volshell:main" [tool.setuptools.dynamic] version = { attr = "volatility3.framework.constants._version.PACKAGE_VERSION" } -dependencies = { file = "requirements-minimal.txt" } [tool.setuptools.packages.find] include = ["volatility3*"] +[tool.mypy] +mypy_path = "./stubs" +show_traceback = true + +[tool.mypy.overrides] +ignore_missing_imports = true + [build-system] requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index ae3482290..000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ --r requirements.txt - -# This can improve error messages regarding improperly configured ISF files, -# but is only recommended for development -jsonschema>=2.3.0 - -# Used to build executable file -pyinstaller>=6.5.0 -pyinstaller-hooks-contrib>=2024.3 \ No newline at end of file diff --git a/requirements-minimal.txt b/requirements-minimal.txt deleted file mode 100644 index c030b332d..000000000 --- a/requirements-minimal.txt +++ /dev/null @@ -1,2 +0,0 @@ -# These packages are required for core functionality. -pefile>=2023.2.7 #foo \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e0d366391..000000000 --- a/requirements.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Include the minimal requirements --r requirements-minimal.txt - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 - -# This is required for several plugins that perform malware analysis and disassemble code. -# It can also improve accuracy of Windows 8 and later memory samples. -# FIXME: Version 6.0.0 is incompatible (#1336) so we'll need an adaptor at some point -capstone>=3.0.5,<6.0.0 - -# This is required by plugins that decrypt passwords, password hashes, etc. -pycryptodome - -# This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0; sys_platform != 'darwin' - -# This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage -gcsfs>=2023.1.0 -s3fs>=2023.1.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 3af033160..000000000 --- a/setup.py +++ /dev/null @@ -1,24 +0,0 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -import setuptools - - -def get_requires(filename): - requirements = [] - with open(filename, "r", encoding="utf-8") as fh: - for line in fh.readlines(): - stripped_line = line.strip() - if stripped_line == "" or stripped_line.startswith(("#", "-r")): - continue - requirements.append(stripped_line) - return requirements - - -setuptools.setup( - extras_require={ - "dev": get_requires("requirements-dev.txt"), - "full": get_requires("requirements.txt"), - }, -) From a04c04c4e3789008987ef18207ed242ff606ba1e Mon Sep 17 00:00:00 2001 From: lesander <4174509+lesander@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:03:08 +0100 Subject: [PATCH 546/979] fix ci --- .github/workflows/install.yml | 6 +----- .github/workflows/test.yaml | 6 ++---- .readthedocs.yml | 5 ++++- MANIFEST.in | 2 +- doc/requirements.txt | 9 --------- pyproject.toml | 15 ++++++++++++++- test/README.md | 6 ++---- test/requirements-testing.txt | 11 ----------- 8 files changed, 24 insertions(+), 36 deletions(-) delete mode 100644 doc/requirements.txt delete mode 100644 test/requirements-testing.txt diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 398ff8ae3..9b9cbed4d 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -20,12 +20,8 @@ jobs: - name: Setup python-pip run: python -m pip install --upgrade pip - - name: Install dependencies - run: | - pip install -r requirements.txt - - name: Install volatility3 run: pip install . - name: Run volatility3 - run: vol --help \ No newline at end of file + run: vol --help diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6358dd45d..930b68526 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,10 +16,8 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install Cmake - pip install build - pip install -r ./test/requirements-testing.txt + python -m pip install --upgrade pip Cmake build + pip install .[test] - name: Build PyPi packages run: | diff --git a/.readthedocs.yml b/.readthedocs.yml index e7c2b25d5..628e79ebf 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -20,4 +20,7 @@ build: # Optionally set the version of Python and requirements required to build your docs python: install: - - requirements: doc/requirements.txt + - method: pip + path: . + extra_requirements: + - docs diff --git a/MANIFEST.in b/MANIFEST.in index 1cec729f6..863621381 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ prune development include * .* -include doc/make.bat doc/Makefile doc/requirements.txt +include pyproject.toml doc/make.bat doc/Makefile recursive-include doc/source * recursive-include volatility3 *.json recursive-exclude doc/source volatility3.*.rst diff --git a/doc/requirements.txt b/doc/requirements.txt deleted file mode 100644 index d3ba51224..000000000 --- a/doc/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -# These packages are required for building the documentation. -sphinx>=4.0.0,<7 -sphinx_autodoc_typehints>=1.4.0 -sphinx-rtd-theme>=0.4.3 - -yara-python -yara-x -pycryptodome -pefile diff --git a/pyproject.toml b/pyproject.toml index 91848e1fd..a966e41b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ authors = [ ] requires-python = ">=3.9.0" license = { text = "VSL" } -dynamic = ["dependencies", "optional-dependencies", "version"] +dynamic = ["version"] dependencies = [ "pefile>=2024.8.26", @@ -34,6 +34,19 @@ dev = [ "pyinstaller-hooks-contrib>=2024.9", ] +test = [ + "volatility3[dev]", + "pytest>=8.3.3,<9", + "yara-x>=0.10.0,<1", +] + +docs = [ + "volatility3[dev]", + "sphinx>=8.0.0,<7", + "sphinx-autodoc-typehints>=2.5.0,<3", + "sphinx-rtd-theme>=3.0.1,<4", +] + [project.urls] homepage = "https://github.com/volatilityfoundation/volatility3/" documentation = "https://volatility3.readthedocs.io/" diff --git a/test/README.md b/test/README.md index dcbe289b0..5891d9508 100644 --- a/test/README.md +++ b/test/README.md @@ -2,14 +2,12 @@ ## Requirements -The Volatility 3 Testing Framework requires the same version of Python as Volatility3 itself. To install the current set of dependencies that the framework requires, use a command like this: +The Volatility 3 Testing Framework requires the same version of Python as Volatility 3 itself. To install the current set of dependencies that the framework requires, use a command like this: ```shell -pip3 install -r requirements-testing.txt +pip3 install -e .[test] ``` -NOTE: `requirements-testing.txt` can be found in this current `test/` directory. - ## Quick Start: Manual Testing 1. To test Volatility 3 on an image, first download one with a command such as: diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt deleted file mode 100644 index 51c8f602c..000000000 --- a/test/requirements-testing.txt +++ /dev/null @@ -1,11 +0,0 @@ -# These packages are required for core functionality. -pefile>=2017.8.1 #foo - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 -yara-x>=0.5.0 - -pytest>=7.0.0 From b739f8d067c024f84087bfdc2443b174cf559462 Mon Sep 17 00:00:00 2001 From: lesander <4174509+lesander@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:05:54 +0100 Subject: [PATCH 547/979] revert 3.8 to 3.9 soft bump --- README.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 78eb7c491..cc33d3cc4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Installing -Volatility 3 requires Python 3.9.0 or later and is published on the [PyPi registry](https://pypi.org/project/volatility3). +Volatility 3 requires Python 3.8.0 or later and is published on the [PyPi registry](https://pypi.org/project/volatility3). ```shell pip install volatility3 diff --git a/pyproject.toml b/pyproject.toml index a966e41b2..fc1ab96cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, ] -requires-python = ">=3.9.0" +requires-python = ">=3.8.0" license = { text = "VSL" } dynamic = ["version"] From 8f33aaf5b4ee859b12ca35347144597bec59ee1b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Nov 2024 17:45:53 +0100 Subject: [PATCH 548/979] Optional type hints --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0dd657372..e5a074a49 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2445,7 +2445,7 @@ class scatterlist(objects.StructType): """ return self._context.layers[self.vol.layer_name].page_size // self.vol.size - def _sg_next(self) -> interfaces.objects.ObjectInterface: + def _sg_next(self) -> Optional[interfaces.objects.ObjectInterface]: """Get the next scatterlist struct from the list. Based on kernel's sg_next. @@ -2476,7 +2476,7 @@ class scatterlist(objects.StructType): ) return sg - def for_each_sg(self) -> Iterator[interfaces.objects.ObjectInterface]: + def for_each_sg(self) -> Optional[Iterator[interfaces.objects.ObjectInterface]]: """Iterate over each struct in the scatterlist.""" sg = self sg_max_single_alloc = self._get_sg_max_single_alloc() @@ -2504,7 +2504,7 @@ class scatterlist(objects.StructType): def get_content( self, - ) -> Iterator[bytes]: + ) -> Optional[Iterator[bytes]]: """Traverse a scatterlist to gather content located at each dma_address position. From 485ef894e113cf68eb1acaef3c679b675639281d Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Nov 2024 17:49:08 +0100 Subject: [PATCH 549/979] remove taints_value overload attr --- .../framework/symbols/linux/extensions/__init__.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 42c1a470d..f9f72c161 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -288,7 +288,7 @@ class module(generic.GenericIntelProcess): """ taints_string = "" for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if taint_flag.module and self.taints_value & taint_flag.shift: + if taint_flag.module and self.taints & taint_flag.shift: taints_string += char return taints_string @@ -310,7 +310,7 @@ class module(generic.GenericIntelProcess): for i, taint_flag in enumerate(self.taint_flags_list): c_true = chr(taint_flag.c_true) c_false = chr(taint_flag.c_false) - if taint_flag.module and (self.taints_value & (1 << i)): + if taint_flag.module and (self.taints & (1 << i)): taints_string += c_true elif taint_flag.module and c_false != " ": taints_string += c_false @@ -378,10 +378,6 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("module -> strtab: Unable to get strtab") - @property - def taints_value(self) -> int: - return self.taints - @property def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) From dd3542b127b751ad083c91be4e8ffd373a1c74f7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Nov 2024 17:51:48 +0100 Subject: [PATCH 550/979] explicit loop iterator --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f9f72c161..402f8c9c6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -341,10 +341,10 @@ class module(generic.GenericIntelProcess): - module_flags_taint kernel function """ comprehensive_taints = [] - for c in self.get_taints_as_plain_string(): - taint_flag = linux_constants.TAINT_FLAGS.get(c) + for character in self.get_taints_as_plain_string(): + taint_flag = linux_constants.TAINT_FLAGS.get(character) if not taint_flag: - comprehensive_taints.append(f"") + comprehensive_taints.append(f"") elif taint_flag.when_present: comprehensive_taints.append(taint_flag.desc) From 106b3b3dd0849194c4b83b9e254365b0c8f6070c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 16:01:04 +1100 Subject: [PATCH 551/979] Linux: PageCache: Fix inode offset when logging an error --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 81e1f3601..005fc9acc 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -454,7 +454,7 @@ class InodePages(plugins.PluginInterface): if current_fp + len(page_bytes) > inode_size: vollog.error( "Page out of file bounds: inode 0x%x, inode size %d, page index %d", - inode.vol.object, + inode.vol.offset, inode_size, page_idx, ) From 3216700d8b6b0488f4a7431d6c1d940fc5299f24 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 16:05:59 +1100 Subject: [PATCH 552/979] Linux: module object extension: Fix recursion call issue --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d05c34304..bc8e25105 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -294,7 +294,7 @@ class module(generic.GenericIntelProcess): if self.has_member("kallsyms"): return int(self.kallsyms.num_symtab) elif self.has_member("num_symtab"): - return int(self.num_symtab) + return int(self.member("num_symtab")) raise AttributeError( "module -> num_symtab: Unable to determine number of symbols" ) From a2e99b2633a0099e19bbb5692d7919a9e5cd17cd Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 16:13:50 +1100 Subject: [PATCH 553/979] Linux: Object extensions: Fix type annotations for improved clarity and accuracy. --- .../framework/symbols/linux/extensions/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index bc8e25105..41d764f94 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -240,7 +240,7 @@ class module(generic.GenericIntelProcess): for sym in syms: yield sym - def get_symbols_names_and_addresses(self) -> Tuple[str, int]: + def get_symbols_names_and_addresses(self) -> Iterable[Tuple[str, int]]: """Get names and addresses for each symbol of the module Yields: @@ -1098,7 +1098,7 @@ class dentry(objects.StructType): current_dentry = current_dentry.d_parent return None - def get_subdirs(self) -> interfaces.objects.ObjectInterface: + def get_subdirs(self) -> Iterable[interfaces.objects.ObjectInterface]: """Walks dentry subdirs Yields: @@ -2435,7 +2435,7 @@ class inode(objects.StructType): """ return stat.filemode(self.i_mode) - def get_pages(self) -> interfaces.objects.ObjectInterface: + def get_pages(self) -> Iterable[interfaces.objects.ObjectInterface]: """Gets the inode's cached pages Yields: @@ -2643,7 +2643,7 @@ class IDR(objects.StructType): return idr_layer - def _old_kernel_get_entries(self) -> int: + def _old_kernel_get_entries(self) -> Iterable[int]: # Kernels < 4.11 cur = self.cur total = next_id = 0 @@ -2655,7 +2655,7 @@ class IDR(objects.StructType): next_id += 1 - def _new_kernel_get_entries(self) -> int: + def _new_kernel_get_entries(self) -> Iterable[int]: # Kernels >= 4.11 id_storage = linux.IDStorage.choose_id_storage( self._context, kernel_module_name="kernel" @@ -2663,7 +2663,7 @@ class IDR(objects.StructType): for page_addr in id_storage.get_entries(root=self.idr_rt): yield page_addr - def get_entries(self) -> int: + def get_entries(self) -> Iterable[int]: """Walks the IDR and yield a pointer associated with each element. Args: From 3a734fdeb159c8ff31b216b5ef1249d1dd4d3df7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 16:23:33 +1100 Subject: [PATCH 554/979] Linux: Object extensions: Remove redundant log message header; the logger handles this formatting. If needed, we can adjust the logger's format instead --- .../framework/symbols/linux/extensions/__init__.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 41d764f94..927f767e2 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -287,7 +287,7 @@ class module(generic.GenericIntelProcess): return self.kallsyms.symtab elif self.has_member("symtab"): return self.symtab - raise AttributeError("module -> symtab: Unable to get symtab") + raise AttributeError("Unable to get symtab") @property def num_symtab(self): @@ -295,9 +295,7 @@ class module(generic.GenericIntelProcess): return int(self.kallsyms.num_symtab) elif self.has_member("num_symtab"): return int(self.member("num_symtab")) - raise AttributeError( - "module -> num_symtab: Unable to determine number of symbols" - ) + raise AttributeError("Unable to determine number of symbols") @property def section_strtab(self): @@ -307,7 +305,7 @@ class module(generic.GenericIntelProcess): # Older kernels elif self.has_member("strtab"): return self.strtab - raise AttributeError("module -> strtab: Unable to get strtab") + raise AttributeError("Unable to get strtab") class task_struct(generic.GenericIntelProcess): From d90019657c26b2f6f723a5225ad99ad6c3fd012b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 18:14:36 +1100 Subject: [PATCH 555/979] Linux: Add kthreads plugin to enumerate kernel thread functions --- .../framework/plugins/linux/kthreads.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 volatility3/framework/plugins/linux/kthreads.py diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py new file mode 100644 index 000000000..639759e49 --- /dev/null +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -0,0 +1,109 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import List + +from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import linux +from volatility3.framework.constants import architectures +from volatility3.framework.objects import utility +from volatility3.plugins.linux import pslist, lsmod + +vollog = logging.getLogger(__name__) + + +class Kthreads(plugins.PluginInterface): + """Enumerates kthread functions""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) + ), + ] + + def _generator(self): + vmlinux = self.context.modules[self.config["kernel"]] + + modules = lsmod.Lsmod.list_modules(self.context, vmlinux.name) + handlers = linux.LinuxUtilities.generate_kernel_handler_info( + self.context, vmlinux.name, modules + ) + + kthread_type = vmlinux.get_type( + vmlinux.symbol_table_name + constants.BANG + "kthread" + ) + + if not kthread_type.has_member("threadfn"): + raise exceptions.VolatilityException( + "Unsupported kthread implementation. This plugin only works with kernels >= 5.8" + ) + + for task in pslist.PsList.list_tasks( + self.context, vmlinux.name, include_threads=True + ): + if not task.is_kernel_thread: + continue + + if task.has_member("worker_private"): + # kernels >= 5.17 e32cf5dfbe227b355776948b2c9b5691b84d1cbd + ktread_base_pointer = task.worker_private + else: + # 5.8 <= kernels < 5.17 in 52782c92ac85c4e393eb4a903a62e6c24afa633f threadfn + # was added to struct kthread. task.set_child_tid is safe on those versions. + ktread_base_pointer = task.set_child_tid + + if not ktread_base_pointer.is_readable(): + continue + + kthread = ktread_base_pointer.dereference().cast("kthread") + threadfn = kthread.threadfn + if not (threadfn and threadfn.is_readable()): + continue + + task_name = utility.array_to_string(task.comm) + + # kernels >= 5.17 in d6986ce24fc00b0638bd29efe8fb7ba7619ed2aa full_name was added to kthread + thread_name = ( + utility.pointer_to_string(kthread.full_name, count=255) + if kthread.has_member("full_name") + else task_name + ) + module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( + vmlinux, handlers, threadfn + ) + + fields = [ + task.pid, + thread_name, + format_hints.Hex(threadfn), + module_name, + symbol_name, + ] + yield 0, fields + + def run(self): + return renderers.TreeGrid( + [ + ("TID", int), + ("Thread Name", str), + ("Handler Address", format_hints.Hex), + ("Module", str), + ("Symbol", str), + ], + self._generator(), + ) From 780f9ab6f569af2c4e0095cb529736c2d5ef992d Mon Sep 17 00:00:00 2001 From: eve Date: Tue, 12 Nov 2024 07:22:45 +0000 Subject: [PATCH 556/979] Volshell: add regex_scan --- volatility3/cli/volshell/generic.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index a0477656e..9936244ee 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -14,7 +14,7 @@ from urllib import parse, request from volatility3.cli import text_renderer, volshell from volatility3.framework import exceptions, interfaces, objects, plugins, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.layers import intel, physical, resources +from volatility3.framework.layers import intel, physical, resources, scanners try: import capstone @@ -149,6 +149,7 @@ class Volshell(interfaces.plugins.PluginInterface): (["cc", "create_configurable"], self.create_configurable), (["lf", "load_file"], self.load_file), (["rs", "run_script"], self.run_script), + (["re", "regex_scan"], self.regex_scan), ] def _construct_locals_dict(self) -> Dict[str, Any]: @@ -288,6 +289,21 @@ class Volshell(interfaces.plugins.PluginInterface): remaining_data = self._read_data(offset, count=count, layer_name=layer_name) self._display_data(offset, remaining_data, format_string="H") + def regex_scan(self, pattern, count=128, layer_name=None): + """Scans for regex pattern in layer using RegExScanner.""" + if not isinstance(pattern, bytes): + raise TypeError("pattern must be bytes, e.g. re(b'pattern')") + layer_name_to_scan = layer_name or self.current_layer + for offset in self.context.layers[layer_name_to_scan].scan( + scanner=scanners.RegExScanner(pattern), + context=self.context, + ): + remaining_data = self._read_data( + offset, count=count, layer_name=layer_name_to_scan + ) + self._display_data(offset, remaining_data) + print("") + def disassemble(self, offset, count=128, layer_name=None, architecture=None): """Disassembles a number of instructions from the code at offset""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) From 8efe429719f50603e35a170b3eb37e39fc43f5fc Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 12 Nov 2024 18:32:26 +1100 Subject: [PATCH 557/979] Linux: kthreads plugin: Adjust required framework version --- volatility3/framework/plugins/linux/kthreads.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index 639759e49..263793495 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -19,7 +19,7 @@ vollog = logging.getLogger(__name__) class Kthreads(plugins.PluginInterface): """Enumerates kthread functions""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 11, 0) _version = (1, 0, 0) From 787e15ac2b51987db496f38b6d9fa17e4eb43fe9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 13 Nov 2024 09:50:08 +1100 Subject: [PATCH 558/979] Linux: kthreads plugin: Add missing requirements --- volatility3/framework/plugins/linux/kthreads.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index 263793495..2e51b4688 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -34,6 +34,12 @@ class Kthreads(plugins.PluginInterface): requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 3, 0) + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), ] def _generator(self): From 05e8f8ff1075adbb398076a7d91701ba256b8d8a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 13 Nov 2024 12:20:03 +1100 Subject: [PATCH 559/979] intel layer: Move constants to the Intel class --- .../framework/constants/linux/__init__.py | 8 ------- volatility3/framework/layers/intel.py | 24 ++++++++++++------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 303c534ed..7c485d3c3 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -11,14 +11,6 @@ KERNEL_NAME = "__kernel__" """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" -# Translation Layer constants -PAGE_BIT_PRESENT = 0 -PAGE_BIT_PSE = 7 # Page Size Extension: 4 MB (or 2MB) page -PAGE_BIT_PROTNONE = 8 -PAGE_BIT_PAT_LARGE = 12 # 2MB or 1GB pages -PAGE_PRESENT = 1 << PAGE_BIT_PRESENT -PAGE_PROTNONE = 1 << PAGE_BIT_PROTNONE - # include/linux/sched.h PF_KTHREAD = 0x00200000 # I'm a kernel thread diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index e6d20d992..25a98fc21 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -13,7 +13,6 @@ from volatility3 import classproperty from volatility3.framework import exceptions, interfaces, constants from volatility3.framework.configuration import requirements from volatility3.framework.layers import linear -from volatility3.framework.constants import linux as linux_constants vollog = logging.getLogger(__name__) @@ -23,6 +22,16 @@ INTEL_TRANSLATION_DEBUGGING = False class Intel(linear.LinearlyMappedLayer): """Translation Layer for the Intel IA32 memory mapping.""" + _PAGE_BIT_PRESENT = 0 + _PAGE_BIT_PSE = 7 # Page Size Extension: 4 MB (or 2MB) page + _PAGE_BIT_PROTNONE = 8 + _PAGE_BIT_PAT_LARGE = 12 # 2MB or 1GB pages + + _PAGE_PRESENT = 1 << _PAGE_BIT_PRESENT + _PAGE_PSE = 1 << _PAGE_BIT_PSE + _PAGE_PROTNONE = 1 << _PAGE_BIT_PROTNONE + _PAGE_PAT_LARGE = 1 << _PAGE_BIT_PAT_LARGE + _entry_format = " bool: - return ( - self.pte_flags(entry) - & (linux_constants.PAGE_PRESENT | linux_constants.PAGE_PROTNONE) - ) != 0 + return (self.pte_flags(entry) & (self._PAGE_PRESENT | self._PAGE_PROTNONE)) != 0 def _page_is_valid(self, entry: int) -> bool: # Overrides the Intel static method with the Linux-specific implementation @@ -556,7 +562,7 @@ class LinuxMixin(Intel): def pte_needs_invert(self, entry) -> bool: # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted - return not (entry & linux_constants.PAGE_PRESENT) + return not (entry & self._PAGE_PRESENT) def protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" From 36c19aa6d91b561ab57db34eb6c6a2ba7bbe2316 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 13 Nov 2024 12:21:03 +1100 Subject: [PATCH 560/979] core: Bump framework minor version --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index ce803a687..55ef19e4b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 11 # Number of changes that only add to the interface +VERSION_MINOR = 12 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 173e35105a6fc614e0ae329331e8e21e63f52f14 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 13 Nov 2024 12:30:11 +1100 Subject: [PATCH 561/979] LinuxMixin: Make new methods internal --- volatility3/framework/layers/intel.py | 46 ++++++++++++++------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 25a98fc21..dc88c20bf 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -174,13 +174,13 @@ class Intel(linear.LinearlyMappedLayer): f"Page Fault at entry {hex(entry)} in page entry", ) - pfn = self.pte_pfn(entry) + pfn = self._pte_pfn(entry) page_offset = self._mask(offset, position, 0) page = pfn << self.page_shift | page_offset return page, 1 << (position + 1), self._base_layer - def pte_pfn(self, entry: int) -> int: + def _pte_pfn(self, entry: int) -> int: """Extracts the page frame number (PFN) from the page table entry (PTE) entry""" return entry >> self.page_shift @@ -520,11 +520,11 @@ class WindowsIntel32e(WindowsMixin, Intel32e): class LinuxMixin(Intel): @functools.cached_property - def register_mask(self) -> int: + def _register_mask(self) -> int: return (1 << self._bits_per_register) - 1 @functools.cached_property - def physical_mask(self) -> int: + def _physical_mask(self) -> int: # From kernels 4.18 the physical mask is dynamic: See AMD SME, Intel Multi-Key Total # Memory Encryption and CONFIG_DYNAMIC_PHYSICAL_MASK: 94d49eb30e854c84d1319095b5dd0405a7da9362 physical_mask = (1 << self._maxphyaddr) - 1 @@ -536,42 +536,44 @@ class LinuxMixin(Intel): # Note that within the Intel class it's a class method. However, since it uses # complement operations and we are working in Python, it would be more careful to # limit it to the architecture's pointer size. - return ~(self.page_size - 1) & self.register_mask + return ~(self.page_size - 1) & self._register_mask @functools.cached_property - def physical_page_mask(self) -> int: - return self.page_mask & self.physical_mask + def _physical_page_mask(self) -> int: + return self.page_mask & self._physical_mask @functools.cached_property - def pte_pfn_mask(self) -> int: - return self.physical_page_mask + def _pte_pfn_mask(self) -> int: + return self._physical_page_mask @functools.cached_property - def pte_flags_mask(self) -> int: - return ~self.pte_pfn_mask & self.register_mask + def _pte_flags_mask(self) -> int: + return ~self._pte_pfn_mask & self._register_mask - def pte_flags(self, pte) -> int: - return pte & self.pte_flags_mask + def _pte_flags(self, pte) -> int: + return pte & self._pte_flags_mask - def is_pte_present(self, entry: int) -> bool: - return (self.pte_flags(entry) & (self._PAGE_PRESENT | self._PAGE_PROTNONE)) != 0 + def _is_pte_present(self, entry: int) -> bool: + return ( + self._pte_flags(entry) & (self._PAGE_PRESENT | self._PAGE_PROTNONE) + ) != 0 def _page_is_valid(self, entry: int) -> bool: # Overrides the Intel static method with the Linux-specific implementation - return self.is_pte_present(entry) + return self._is_pte_present(entry) - def pte_needs_invert(self, entry) -> bool: + def _pte_needs_invert(self, entry) -> bool: # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted return not (entry & self._PAGE_PRESENT) - def protnone_mask(self, entry: int) -> int: + def _protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" - return ~0 & self.register_mask if self.pte_needs_invert(entry) else 0 + return ~0 & self._register_mask if self._pte_needs_invert(entry) else 0 - def pte_pfn(self, entry: int) -> int: + def _pte_pfn(self, entry: int) -> int: """Extracts the page frame number from the page table entry""" - pfn = entry ^ self.protnone_mask(entry) - return (pfn & self.pte_pfn_mask) >> self.page_shift + pfn = entry ^ self._protnone_mask(entry) + return (pfn & self._pte_pfn_mask) >> self.page_shift class LinuxIntel(LinuxMixin, Intel): From af7a420cdc14689ee6d2ff72d5ac8eefad9e833d Mon Sep 17 00:00:00 2001 From: eve Date: Wed, 13 Nov 2024 08:56:58 +0000 Subject: [PATCH 562/979] Volshell: rename regex_scan short hand to rx so that it does not clash with the built in python re module --- volatility3/cli/volshell/generic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 9936244ee..b196360d5 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -149,7 +149,7 @@ class Volshell(interfaces.plugins.PluginInterface): (["cc", "create_configurable"], self.create_configurable), (["lf", "load_file"], self.load_file), (["rs", "run_script"], self.run_script), - (["re", "regex_scan"], self.regex_scan), + (["rx", "regex_scan"], self.regex_scan), ] def _construct_locals_dict(self) -> Dict[str, Any]: @@ -292,7 +292,7 @@ class Volshell(interfaces.plugins.PluginInterface): def regex_scan(self, pattern, count=128, layer_name=None): """Scans for regex pattern in layer using RegExScanner.""" if not isinstance(pattern, bytes): - raise TypeError("pattern must be bytes, e.g. re(b'pattern')") + raise TypeError("pattern must be bytes, e.g. rx(b'pattern')") layer_name_to_scan = layer_name or self.current_layer for offset in self.context.layers[layer_name_to_scan].scan( scanner=scanners.RegExScanner(pattern), From 5ab97564ac025978ab3baebe143a259111d6fbd3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 14 Nov 2024 16:37:05 +1100 Subject: [PATCH 563/979] linux: intel VMCoreInfo: Move to its own stacker... Maximize use of VMCOREINFO data without reliance on ISF symbols: - Obtain the DTB - Utilize OSRELEASE (the same as UTS_RELEASE used in the Linux banner and init_uts_ns/new_utsname) to prefilter the list of Linux banners, reducing search time for linux_banner in memory. - Find the correct layer using the VMCOREINFO data (including 32bit PAE). --- volatility3/framework/automagic/linux.py | 302 ++++++++++++++++------- 1 file changed, 212 insertions(+), 90 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 248bbf04d..f877d1cd0 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -122,7 +122,7 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): return None @classmethod - def find_aslr_classic( + def find_aslr( cls, context: interfaces.context.ContextInterface, symbol_table: str, @@ -198,101 +198,14 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): vollog.debug("Scanners could not determine any ASLR shifts, using 0 for both") return 0, 0 - @classmethod - def find_aslr_vmcoreinfo( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None, - ) -> Optional[Tuple[int, int]]: - """Determines the ASLR offsets using the VMCOREINFO ELF note - - Args: - context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The layer within the context in which the module exists - progress_callback: A function that takes a percentage (and an optional description) that will be called periodically - - Returns: - kaslr_shirt and aslr_shift - """ - - for ( - _vmcoreinfo_offset, - vmcoreinfo, - ) in linux.VMCoreInfo.search_vmcoreinfo_elf_note( - context=context, - layer_name=layer_name, - progress_callback=progress_callback, - ): - - phys_base_str = vmcoreinfo.get("NUMBER(phys_base)") - if phys_base_str is None: - # We are in kernel (x86) < 4.10 401721ecd1dcb0a428aa5d6832ee05ffbdbffbbe where it was SYMBOL(phys_base) - # It's the symbol address instead of the value itself, which is useless for calculating the physical address. - continue - - kerneloffset_str = vmcoreinfo.get("KERNELOFFSET") - if kerneloffset_str is None: - # KERNELOFFSET: (x86) kernels < 3.13 b6085a865762236bb84934161273cdac6dd11c2d - continue - - aslr_shift = int(kerneloffset_str, 16) - kaslr_shift = int(phys_base_str) + aslr_shift - - vollog.debug( - "Linux ASLR shift values found in VMCOREINFO ELF note: physical 0x%x virtual 0x%x", - kaslr_shift, - aslr_shift, - ) - - return kaslr_shift, aslr_shift - - vollog.debug("The vmcoreinfo scanner could not determine any ASLR shifts") - return None - - @classmethod - def virtual_to_physical_address(cls, addr: int) -> int: + @staticmethod + def virtual_to_physical_address(addr: int) -> int: """Converts a virtual linux address to a physical one (does not account of ASLR)""" if addr > 0xFFFFFFFF80000000: return addr - 0xFFFFFFFF80000000 return addr - 0xC0000000 - @classmethod - def find_aslr( - cls, - context: interfaces.context.ContextInterface, - symbol_table: str, - layer_name: str, - progress_callback: constants.ProgressCallback = None, - ) -> Tuple[int, int]: - """Determines the offset of the actual DTB in physical space and its - symbol offset. - Args: - context: The context to retrieve required elements (layers, symbol tables) from - symbol_table: The name of the kernel module on which to operate - layer_name: The layer within the context in which the module exists - progress_callback: A function that takes a percentage (and an optional description) that will be called periodically - - Returns: - kaslr_shirt and aslr_shift - """ - - aslr_shifts = cls.find_aslr_vmcoreinfo( - context, layer_name, progress_callback=progress_callback - ) - if aslr_shifts: - kaslr_shift, aslr_shift = aslr_shifts - else: - # Fallback to the traditional scanner method - kaslr_shift, aslr_shift = cls.find_aslr_classic( - context, - symbol_table, - layer_name, - progress_callback=progress_callback, - ) - return kaslr_shift, aslr_shift - class LinuxSymbolFinder(symbol_finder.SymbolFinder): """Linux symbol loader based on uname signature strings.""" @@ -302,3 +215,212 @@ class LinuxSymbolFinder(symbol_finder.SymbolFinder): symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] exclusion_list = ["mac", "windows"] + + +class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): + stack_order = 34 + exclusion_list = ["mac", "windows"] + + @staticmethod + def _check_versions() -> bool: + """Verify the versions of the required modules""" + + # Check SQlite cache version + sqlitecache_version_required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required( + sqlitecache_version_required, symbol_cache.SqliteCache.version + ): + vollog.info( + "SQLiteCache version not suitable: required %s found %s", + sqlitecache_version_required, + symbol_cache.SqliteCache.version, + ) + return False + + # Check VMCOREINFO API version + vmcoreinfo_version_required = (1, 0, 0) + if not requirements.VersionRequirement.matches_required( + vmcoreinfo_version_required, linux.VMCoreInfo._version + ): + vollog.info( + "VMCOREINFO version not suitable: required %s found %s", + vmcoreinfo_version_required, + linux.VMCoreInfo._version, + ) + return False + + return True + + @classmethod + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: + """Attempts to identify linux within this layer.""" + + # Verify the versions of the required modules + if not cls._check_versions(): + return None + + # Bail out by default unless we can stack properly + layer = context.layers[layer_name] + + # Never stack on top of an intel layer + # FIXME: Find a way to improve this check + if isinstance(layer, intel.Intel): + return None + + identifiers_path = os.path.join( + constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + ) + sqlite_cache = symbol_cache.SqliteCache(identifiers_path) + linux_banners = sqlite_cache.get_identifier_dictionary(operating_system="linux") + if not linux_banners: + # If we have no banners, don't bother scanning + vollog.info( + "No Linux banners found - if this is a linux plugin, please check your " + "symbol files location" + ) + return None + + vmcoreinfo_elf_notes_iter = linux.VMCoreInfo.search_vmcoreinfo_elf_note( + context=context, + layer_name=layer_name, + progress_callback=progress_callback, + ) + + # Iterate through each VMCOREINFO ELF note found, using the first one that is valid. + for _vmcoreinfo_offset, vmcoreinfo in vmcoreinfo_elf_notes_iter: + shifts = cls._vmcoreinfo_find_aslr(vmcoreinfo) + if not shifts: + # Let's try the next vmcoreinfo, in case this one isn't correct. + continue + + kaslr_shift, aslr_shift = shifts + + dtb = cls._vmcoreinfo_get_dtb(vmcoreinfo, aslr_shift, kaslr_shift) + + is_32bit, is_pae = cls._vmcoreinfo_is_32bit(vmcoreinfo) + if is_32bit: + layer_class = intel.IntelPAE if is_pae else intel.Intel + else: + layer_class = intel.Intel32e + + uts_release = vmcoreinfo["OSRELEASE"] + + # See how linux_banner constant is built in the linux kernel + linux_version_prefix = f"Linux version {uts_release} (".encode() + valid_banners = [ + x for x in linux_banners if x and x.startswith(linux_version_prefix) + ] + if not valid_banners: + # There's no banner matching this VMCOREINFO, keep trying with the next one + continue + + join = interfaces.configuration.path_join + mss = scanners.MultiStringScanner(valid_banners) + for _, banner in layer.scan( + context=context, scanner=mss, progress_callback=progress_callback + ): + isf_path = linux_banners.get(banner, None) + if not isf_path: + vollog.warning( + "Identified banner %r, but no matching ISF is available.", + banner, + ) + continue + + vollog.debug("Identified banner: %r", banner) + table_name = context.symbol_space.free_table_name("LintelStacker") + table = linux.LinuxKernelIntermedSymbols( + context, + f"temporary.{table_name}", + name=table_name, + isf_url=isf_path, + ) + context.symbol_space.append(table) + + # Build the new layer + new_layer_name = context.layers.free_layer_name("IntelLayer") + config_path = join("IntelHelper", new_layer_name) + kernel_banner = LinuxSymbolFinder.banner_config_key + banner_str = banner.decode(encoding="latin-1") + context.config[join(config_path, "memory_layer")] = layer_name + context.config[join(config_path, "page_map_offset")] = dtb + context.config[join(config_path, kernel_banner)] = banner_str + layer = layer_class( + context, + config_path=config_path, + name=new_layer_name, + metadata={"os": "Linux"}, + ) + layer.config["kernel_virtual_offset"] = aslr_shift + + if layer and dtb: + vollog.debug( + "Values found in VMCOREINFO: KASLR=0x%x, ASLR=0x%x, DTB=0x%x", + kaslr_shift, + aslr_shift, + dtb, + ) + + return layer + + vollog.debug("No suitable linux banner could be matched") + return None + + @staticmethod + def _vmcoreinfo_find_aslr(vmcoreinfo) -> Tuple[int, int]: + phys_base_str = vmcoreinfo.get("NUMBER(phys_base)") + if phys_base_str is None: + # In kernel < 4.10, there may be a SYMBOL(phys_base), but as noted in the + # c401721ecd1dcb0a428aa5d6832ee05ffbdbffbbe commit comment, this value + # isn't useful for calculating the physical address. + # There's nothing we can do here, so let's try with the next VMCOREINFO or + # the next Stacker. + return None + + # kernels 3.14 (b6085a865762236bb84934161273cdac6dd11c2d) KERNELOFFSET was added + kerneloffset_str = vmcoreinfo.get("KERNELOFFSET") + if kerneloffset_str is None: + # kernels < 3.14 if KERNELOFFSET is missing, KASLR might not be implemented. + # Oddly, NUMBER(phys_base) is present without it. To be safe, proceed only + # if both are present. + return None + + aslr_shift = int(kerneloffset_str, 16) + kaslr_shift = int(phys_base_str) + aslr_shift + + return kaslr_shift, aslr_shift + + @staticmethod + def _vmcoreinfo_get_dtb(vmcoreinfo, aslr_shift, kaslr_shift) -> int: + """Returns the page global directory address physical (a.k.a DTB or PGD)""" + # In x86-64, since kernels 2.5.22 swapper_pg_dir is a macro to the respective pgd. + # First, in e3ebadd95cb621e2c7436f3d3646447ac9d5c16d to init_level4_pgt, and later + # in kernels 4.13 in 65ade2f872b474fa8a04c2d397783350326634e6) to init_top_pgt. + # In x86-32, the pgd is swapper_pg_dir. So, in any case, for VMCOREINFO + # SYMBOL(swapper_pg_dir) will always have the right value. + dtb_vaddr = int(vmcoreinfo["SYMBOL(swapper_pg_dir)"], 16) + dtb_paddr = ( + LinuxIntelStacker.virtual_to_physical_address(dtb_vaddr) + - aslr_shift + + kaslr_shift + ) + + return dtb_paddr + + @staticmethod + def _vmcoreinfo_is_32bit(vmcoreinfo) -> Tuple[bool, bool]: + """Returns a tuple of booleans with is_32bit and is_pae values""" + is_pae = vmcoreinfo.get("CONFIG_X86_PAE", "n") == "y" + if is_pae: + is_32bit = True + else: + # Check the swapper_pg_dir virtual address size + dtb_vaddr = int(vmcoreinfo["SYMBOL(swapper_pg_dir)"], 16) + is_32bit = dtb_vaddr <= 2**32 + + return is_32bit, is_pae From 00b528aafe4205946e2b1ce11b4a203eaf467ec2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 14 Nov 2024 18:57:26 +1100 Subject: [PATCH 564/979] linux: intel VMCoreInfo: Fix docstring typo --- volatility3/framework/automagic/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f877d1cd0..556a02c1f 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -397,7 +397,7 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): @staticmethod def _vmcoreinfo_get_dtb(vmcoreinfo, aslr_shift, kaslr_shift) -> int: - """Returns the page global directory address physical (a.k.a DTB or PGD)""" + """Returns the page global directory physical address (a.k.a DTB or PGD)""" # In x86-64, since kernels 2.5.22 swapper_pg_dir is a macro to the respective pgd. # First, in e3ebadd95cb621e2c7436f3d3646447ac9d5c16d to init_level4_pgt, and later # in kernels 4.13 in 65ade2f872b474fa8a04c2d397783350326634e6) to init_top_pgt. From 41a93473e987bb43bca8e8095dddd890a1a1b351 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 14 Nov 2024 19:35:47 +1100 Subject: [PATCH 565/979] linux: intel VMCoreInfo: Improve layer config code --- volatility3/framework/automagic/linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 556a02c1f..c92fa05c4 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -347,16 +347,16 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): config_path = join("IntelHelper", new_layer_name) kernel_banner = LinuxSymbolFinder.banner_config_key banner_str = banner.decode(encoding="latin-1") + context.config[join(config_path, kernel_banner)] = banner_str context.config[join(config_path, "memory_layer")] = layer_name context.config[join(config_path, "page_map_offset")] = dtb - context.config[join(config_path, kernel_banner)] = banner_str + context.config[join(config_path, "kernel_virtual_offset")] = aslr_shift layer = layer_class( context, config_path=config_path, name=new_layer_name, metadata={"os": "Linux"}, ) - layer.config["kernel_virtual_offset"] = aslr_shift if layer and dtb: vollog.debug( From 357d8a914b4a69d231b4ed9de3aecdcf7179be7f Mon Sep 17 00:00:00 2001 From: eve Date: Thu, 14 Nov 2024 10:55:41 +0000 Subject: [PATCH 566/979] Volshell: Make the default number of bytes returned by db, dw, dd, dq, rx, and dis use the constant DEFAULT_NUM_DISPLAY_BYTES --- volatility3/cli/volshell/generic.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index b196360d5..82c470e1a 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -29,6 +29,8 @@ class Volshell(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) + DEFAULT_NUM_DISPLAY_BYTES = 128 + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__current_layer: Optional[str] = None @@ -269,27 +271,31 @@ class Volshell(interfaces.plugins.PluginInterface): self.__current_kernel_name = kernel_name print(f"Current kernel : {self.current_kernel_name}") - def display_bytes(self, offset, count=128, layer_name=None): + def display_bytes(self, offset, count=DEFAULT_NUM_DISPLAY_BYTES, layer_name=None): """Displays byte values and ASCII characters""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) self._display_data(offset, remaining_data) - def display_quadwords(self, offset, count=128, layer_name=None): + def display_quadwords( + self, offset, count=DEFAULT_NUM_DISPLAY_BYTES, layer_name=None + ): """Displays quad-word values (8 bytes) and corresponding ASCII characters""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) self._display_data(offset, remaining_data, format_string="Q") - def display_doublewords(self, offset, count=128, layer_name=None): + def display_doublewords( + self, offset, count=DEFAULT_NUM_DISPLAY_BYTES, layer_name=None + ): """Displays double-word values (4 bytes) and corresponding ASCII characters""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) self._display_data(offset, remaining_data, format_string="I") - def display_words(self, offset, count=128, layer_name=None): + def display_words(self, offset, count=DEFAULT_NUM_DISPLAY_BYTES, layer_name=None): """Displays word values (2 bytes) and corresponding ASCII characters""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) self._display_data(offset, remaining_data, format_string="H") - def regex_scan(self, pattern, count=128, layer_name=None): + def regex_scan(self, pattern, count=DEFAULT_NUM_DISPLAY_BYTES, layer_name=None): """Scans for regex pattern in layer using RegExScanner.""" if not isinstance(pattern, bytes): raise TypeError("pattern must be bytes, e.g. rx(b'pattern')") @@ -304,7 +310,13 @@ class Volshell(interfaces.plugins.PluginInterface): self._display_data(offset, remaining_data) print("") - def disassemble(self, offset, count=128, layer_name=None, architecture=None): + def disassemble( + self, + offset, + count=DEFAULT_NUM_DISPLAY_BYTES, + layer_name=None, + architecture=None, + ): """Disassembles a number of instructions from the code at offset""" remaining_data = self._read_data(offset, count=count, layer_name=layer_name) if not has_capstone: From bee3f001b66fa6a9ba9b1e325cb2fa2e273f8ff2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 15 Nov 2024 11:31:30 +1100 Subject: [PATCH 567/979] linux: netfilter plugin: Enhance docstring to provide a clearer explanation of 'hooked' --- volatility3/framework/plugins/linux/netfilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 66075f907..b371d5ad2 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -174,7 +174,7 @@ class AbstractNetfilter(ABC): priority [int]: Priority hook_ops_hook [int]: Hook address module_name [str]: Linux kernel module name - hooked [bool]: hooked? + hooked [bool]: "True" if the network stack has been hijacked """ for netns, net in self.get_net_namespaces(): for proto_idx, proto_name, hook_idx, hook_name in self._proto_hook_loop(): From c909fdd4e76af6f8743a5df9b2ab38c981e68003 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 15 Nov 2024 11:53:05 +1100 Subject: [PATCH 568/979] linux: netfilter plugin: Bump minor version --- volatility3/framework/plugins/linux/netfilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index b371d5ad2..73496dfd9 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -675,7 +675,7 @@ class Netfilter(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 1, 0) _required_linuxutils_version = (2, 1, 0) _required_lsmod_version = (2, 0, 0) From b802b16c62ff5a50ea2dbc659450dc6d1fd5e83e Mon Sep 17 00:00:00 2001 From: eve Date: Fri, 15 Nov 2024 13:03:36 +0000 Subject: [PATCH 569/979] Add first version of regex scanning plugins --- .../framework/plugins/linux/vmaregexscan.py | 127 +++++++++++++++++ volatility3/framework/plugins/regexscan.py | 77 +++++++++++ .../framework/plugins/windows/vadregexscan.py | 128 ++++++++++++++++++ 3 files changed, 332 insertions(+) create mode 100644 volatility3/framework/plugins/linux/vmaregexscan.py create mode 100644 volatility3/framework/plugins/regexscan.py create mode 100644 volatility3/framework/plugins/windows/vadregexscan.py diff --git a/volatility3/framework/plugins/linux/vmaregexscan.py b/volatility3/framework/plugins/linux/vmaregexscan.py new file mode 100644 index 000000000..83df5458f --- /dev/null +++ b/volatility3/framework/plugins/linux/vmaregexscan.py @@ -0,0 +1,127 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import re + +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.layers import scanners +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + + +class VmaRegExScan(plugins.PluginInterface): + """Scans all virtual memory areas for tasks using RegEx.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + MAXSIZE_DEFAULT = 128 + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.StringRequirement( + name="pattern", description="RegEx pattern", optional=False + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size in bytes for displayed context", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), + ] + + def _generator(self, regex_pattern, tasks): + regex_pattern = bytes(regex_pattern, "UTF-8") + vollog.debug(f"RegEx Pattern: {regex_pattern}") + + for task in tasks: + + if not task.mm: + continue + name = utility.array_to_string(task.comm) + + # attempt to create a process layer for each task and skip those + # that cannot (e.g. kernel threads) + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + continue + + # get the proc_layer object from the context + proc_layer = self.context.layers[proc_layer_name] + + # get process sections for scanning + sections = [ + (start, size) for (start, size) in task.get_process_memory_sections() + ] + + for offset in proc_layer.scan( + context=self.context, + scanner=scanners.RegExScanner(regex_pattern), + sections=sections, + progress_callback=self._progress_callback, + ): + result_data = proc_layer.read(offset, self.MAXSIZE_DEFAULT, pad=True) + + # reapply the regex in order to extact just the match + regex_result = re.match(regex_pattern, result_data) + + if regex_result: + # the match is within the results_data (e.g. it fits within MAXSIZE_DEFAULT) + # extract just the match itself + regex_match = regex_result.group(0) + text_result = str(regex_match, encoding="UTF-8", errors="replace") + bytes_result = regex_match + else: + # the match is not with the results_data (e.g. it doesn't fit within MAXSIZE_DEFAULT) + text_result = str(result_data, encoding="UTF-8", errors="replace") + bytes_result = result_data + + user_pid = task.tgid + yield 0, ( + user_pid, + name, + format_hints.Hex(offset), + text_result, + bytes_result, + ) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Offset", format_hints.Hex), + ("Text", str), + ("Hex", bytes), + ], + self._generator( + self.config.get("pattern"), + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ), + ), + ) diff --git a/volatility3/framework/plugins/regexscan.py b/volatility3/framework/plugins/regexscan.py new file mode 100644 index 000000000..d47a16407 --- /dev/null +++ b/volatility3/framework/plugins/regexscan.py @@ -0,0 +1,77 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import re + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.layers import scanners +from volatility3.framework.renderers import format_hints + +vollog = logging.getLogger(__name__) + + +class RegExScan(plugins.PluginInterface): + """Scans kernel memory using RegEx patterns.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + MAXSIZE_DEFAULT = 128 + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.StringRequirement( + name="pattern", description="RegEx pattern", optional=False + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size in bytes for displayed context", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), + ] + + def _generator(self, regex_pattern): + regex_pattern = bytes(regex_pattern, "UTF-8") + vollog.debug(f"RegEx Pattern: {regex_pattern}") + + layer = self.context.layers[self.config["primary"]] + for offset in layer.scan( + context=self.context, scanner=scanners.RegExScanner(regex_pattern) + ): + result_data = layer.read(offset, self.MAXSIZE_DEFAULT, pad=True) + + # reapply the regex in order to extact just the match + regex_result = re.match(regex_pattern, result_data) + + if regex_result: + # the match is within the results_data (e.g. it fits within MAXSIZE_DEFAULT) + # extract just the match itself + regex_match = regex_result.group(0) + text_result = str(regex_match, encoding="UTF-8", errors="replace") + bytes_result = regex_match + else: + # the match is not with the results_data (e.g. it doesn't fit within MAXSIZE_DEFAULT) + text_result = str(result_data, encoding="UTF-8", errors="replace") + bytes_result = result_data + + yield 0, (format_hints.Hex(offset), text_result, bytes_result) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Text", str), + ("Hex", bytes), + ], + self._generator(self.config.get("pattern")), + ) diff --git a/volatility3/framework/plugins/windows/vadregexscan.py b/volatility3/framework/plugins/windows/vadregexscan.py new file mode 100644 index 000000000..742b00ace --- /dev/null +++ b/volatility3/framework/plugins/windows/vadregexscan.py @@ -0,0 +1,128 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import re + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.layers import scanners +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist, vadyarascan + +vollog = logging.getLogger(__name__) + + +class VadRegExScan(plugins.PluginInterface): + """Scans all virtual memory areas for tasks using RegEx.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + MAXSIZE_DEFAULT = 128 + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="vadyarascan", plugin=vadyarascan.VadYaraScan, version=(1, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.StringRequirement( + name="pattern", description="RegEx pattern", optional=False + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size in bytes for displayed context", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), + ] + + def _generator(self, regex_pattern, procs): + regex_pattern = bytes(regex_pattern, "UTF-8") + vollog.debug(f"RegEx Pattern: {regex_pattern}") + + for proc in procs: + + # attempt to create a process layer for each proc + proc_layer_name = proc.add_process_layer() + if not proc_layer_name: + continue + + # get the proc_layer object from the context + proc_layer = self.context.layers[proc_layer_name] + + # get process sections for scanning + sections = sections = vadyarascan.VadYaraScan.get_vad_maps(proc) + + for offset in proc_layer.scan( + context=self.context, + scanner=scanners.RegExScanner(regex_pattern), + sections=sections, + progress_callback=self._progress_callback, + ): + result_data = proc_layer.read(offset, self.MAXSIZE_DEFAULT, pad=True) + + # reapply the regex in order to extact just the match + regex_result = re.match(regex_pattern, result_data) + + if regex_result: + # the match is within the results_data (e.g. it fits within MAXSIZE_DEFAULT) + # extract just the match itself + regex_match = regex_result.group(0) + text_result = str(regex_match, encoding="UTF-8", errors="replace") + bytes_result = regex_match + else: + # the match is not with the results_data (e.g. it doesn't fit within MAXSIZE_DEFAULT) + text_result = str(result_data, encoding="UTF-8", errors="replace") + bytes_result = result_data + + proc_id = proc.UniqueProcessId + process_name = proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ) + yield 0, ( + proc_id, + process_name, + format_hints.Hex(offset), + text_result, + bytes_result, + ) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] + procs = pslist.PsList.list_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=filter_func, + ) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Offset", format_hints.Hex), + ("Text", str), + ("Hex", bytes), + ], + self._generator(self.config.get("pattern"), procs), + ) From c2058e744b9a0f1e213857f4712ba1d9fdf85964 Mon Sep 17 00:00:00 2001 From: eve Date: Fri, 15 Nov 2024 13:12:03 +0000 Subject: [PATCH 570/979] Fix missing type imports for regex plugins --- volatility3/framework/plugins/linux/vmaregexscan.py | 5 +++-- volatility3/framework/plugins/regexscan.py | 1 + volatility3/framework/plugins/windows/vadregexscan.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmaregexscan.py b/volatility3/framework/plugins/linux/vmaregexscan.py index 83df5458f..77ab29814 100644 --- a/volatility3/framework/plugins/linux/vmaregexscan.py +++ b/volatility3/framework/plugins/linux/vmaregexscan.py @@ -4,8 +4,9 @@ import logging import re +from typing import List -from volatility3.framework import renderers +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import scanners @@ -24,7 +25,7 @@ class VmaRegExScan(plugins.PluginInterface): MAXSIZE_DEFAULT = 128 @classmethod - def get_requirements(cls): + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement( diff --git a/volatility3/framework/plugins/regexscan.py b/volatility3/framework/plugins/regexscan.py index d47a16407..c526b1697 100644 --- a/volatility3/framework/plugins/regexscan.py +++ b/volatility3/framework/plugins/regexscan.py @@ -4,6 +4,7 @@ import logging import re +from typing import List from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements diff --git a/volatility3/framework/plugins/windows/vadregexscan.py b/volatility3/framework/plugins/windows/vadregexscan.py index 742b00ace..141508283 100644 --- a/volatility3/framework/plugins/windows/vadregexscan.py +++ b/volatility3/framework/plugins/windows/vadregexscan.py @@ -4,6 +4,7 @@ import logging import re +from typing import List from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -23,7 +24,7 @@ class VadRegExScan(plugins.PluginInterface): MAXSIZE_DEFAULT = 128 @classmethod - def get_requirements(cls): + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement( From aa93b57f0dde01b97bc94025447b242afa4c36b2 Mon Sep 17 00:00:00 2001 From: eve Date: Fri, 15 Nov 2024 13:15:40 +0000 Subject: [PATCH 571/979] Fix windows.vadregexscan sections --- volatility3/framework/plugins/windows/vadregexscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadregexscan.py b/volatility3/framework/plugins/windows/vadregexscan.py index 141508283..8d17c469c 100644 --- a/volatility3/framework/plugins/windows/vadregexscan.py +++ b/volatility3/framework/plugins/windows/vadregexscan.py @@ -70,7 +70,7 @@ class VadRegExScan(plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] # get process sections for scanning - sections = sections = vadyarascan.VadYaraScan.get_vad_maps(proc) + sections = vadyarascan.VadYaraScan.get_vad_maps(proc) for offset in proc_layer.scan( context=self.context, From 9f145ddcf0ddb84cf7ae45585ce7e9da0d204a17 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:04:33 +0100 Subject: [PATCH 572/979] pillow dependency --- requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requirements.txt b/requirements.txt index e0d366391..21c8f9a76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,9 @@ leechcorepyc>=2.4.0; sys_platform != 'darwin' # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage gcsfs>=2023.1.0 s3fs>=2023.1.0 + +# This is required by plugins that manipulate pixels and images. +# https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst +# 10.0.0 dropped support for Python3.7 +# 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 +pillow>=10.0.0,<11.0.0 \ No newline at end of file From 36a18f405b8ba7605ca957ca47d540a5b7c520d7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:05:23 +0100 Subject: [PATCH 573/979] fourcc code converter helper --- volatility3/framework/symbols/linux/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3289775b6..573bc66d8 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -483,6 +483,22 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return kernel + @classmethod + def convert_fourcc_code(cls, code: int) -> str: + """Convert a fourcc integer back to its fourcc string representation. + + Args: + code: the numerical representation of the fourcc + + Returns: + The fourcc code string. + """ + + code_bytes_length = (code.bit_length() + 7) // 8 + return "".join( + [chr((code >> (i * 8)) & 0xFF) for i in range(code_bytes_length)] + ) + class IDStorage(ABC): """Abstraction to support both XArray and RadixTree""" From a7620cd6f659dfa685a445536a240182225a778c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:09:21 +0100 Subject: [PATCH 574/979] linux fbdev subsystem api plugin --- .../framework/plugins/linux/graphics/fbdev.py | 314 ++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 volatility3/framework/plugins/linux/graphics/fbdev.py diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py new file mode 100644 index 000000000..4bde6f62f --- /dev/null +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -0,0 +1,314 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +import io + +# Image manipulation functions are kept in the plugin, +# to prevent a general exit on missing PIL (pillow) dependency. +from PIL import Image +from dataclasses import dataclass +from typing import Type, List, Dict, Tuple +from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.objects import utility +from volatility3.framework.constants import architectures +from volatility3.framework.symbols import linux + +vollog = logging.getLogger(__name__) + + +@dataclass +class Framebuffer: + """Framebuffer object internal representation. This is useful to unify an framebuffer with precalculated + properties and pass it through functions conveniently.""" + + id: str + xres_virtual: int + yres_virtual: int + line_length: int + bpp: int + """Bits Per Pixel""" + size: int + color_fields: Dict[str, Tuple[int, int, int]] + fb_info: interfaces.objects.ObjectInterface + + +class Fbdev(interfaces.plugins.PluginInterface): + """Extract framebuffers from the fbdev graphics subsystem""" + + _version = (1, 0, 0) + _required_framework_version = (2, 11, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.BooleanRequirement( + name="dump", + description="Dump framebuffers", + default=False, + optional=True, + ), + ] + + @classmethod + def parse_fb_pixel_bitfields( + cls, fb_var_screeninfo: interfaces.objects.ObjectInterface + ) -> Dict[str, Tuple[int, int, int]]: + """Organize a framebuffer pixel format into a dictionary. + This is needed to know the position and bitlength of a color inside + a pixel. + + Args: + fb_var_screeninfo: a fb_var_screeninfo kernel object instance + + Returns: + The color fields mappings + + Documentation: + include/uapi/linux/fb.h: + struct fb_bitfield { + __u32 offset; /* beginning of bitfield */ + __u32 length; /* length of bitfield */ + __u32 msb_right; /* != 0 : Most significant bit is right */ + }; + """ + # Naturally order by RGBA + color_mappings = [ + ("R", fb_var_screeninfo.red), + ("G", fb_var_screeninfo.green), + ("B", fb_var_screeninfo.blue), + ("A", fb_var_screeninfo.transp), + ] + color_fields = {} + for color_code, fb_bitfield in color_mappings: + color_fields[color_code] = ( + int(fb_bitfield.offset), + int(fb_bitfield.length), + int(fb_bitfield.msb_right), + ) + return color_fields + + @classmethod + def convert_fb_raw_buffer_to_image( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + fb: Framebuffer, + ) -> Image.Image: + """Convert raw framebuffer pixels to an image. + + Args: + fb: the relevant Framebuffer object + + Returns: + A PIL Image object + + Documentation: + include/uapi/linux/fb.h: + /* Interpretation of offset for color fields: All offsets are from the right, + * inside a "pixel" value, which is exactly 'bits_per_pixel' wide (means: you + * can use the offset as right argument to <<). A pixel afterwards is a bit + * stream and is written to video memory as that unmodified. + """ + kernel = context.modules[kernel_name] + kernel_layer = context.layers[kernel.layer_name] + + raw_pixels = io.BytesIO(kernel_layer.read(fb.fb_info.screen_base, fb.size)) + bytes_per_pixel = fb.bpp // 8 + image = Image.new("RGBA", (fb.xres_virtual, fb.yres_virtual)) + + # This is not designed to be extremely fast (numpy isn't available), + # but convenient and dynamic for any color field layout. + for y in range(fb.yres_virtual): + for x in range(fb.xres_virtual): + raw_pixel = int.from_bytes(raw_pixels.read(bytes_per_pixel), "little") + pixel = [0, 0, 0, 255] + # The framebuffer is expected to have been correctly constructed, + # especially by parse_fb_pixel_bitfields, to get the needed RGBA mappings. + for i, color_code in enumerate(["R", "G", "B", "A"]): + offset, length, msb_right = fb.color_fields[color_code] + if length == 0: + continue + color_value = (raw_pixel >> offset) & (2**length - 1) + if msb_right: + # Reverse bit order + color_value = int( + "{:0{length}b}".format(color_value, length=length)[::-1], 2 + ) + pixel[i] = color_value + image.putpixel((x, y), tuple(pixel)) + + return image + + @classmethod + def dump_fb( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], + fb: Framebuffer, + convert_to_image: bool, + image_format: str = "PNG", + ) -> str: + """Dump a Framebuffer raw buffer to disk. + + Args: + fb: the relevant Framebuffer object + convert_to_image: a boolean specifying if the buffer should be converted to an image + image_format: the target PIL image format (defaults to PNG) + + Returns: + The filename of the dumped buffer. + """ + kernel = context.modules[kernel_name] + kernel_layer = context.layers[kernel.layer_name] + base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" + if convert_to_image: + image = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) + output = io.BytesIO() + image.save(output, image_format) + file_handle = open_method(f"{base_filename}.{image_format.lower()}") + file_handle.write(output.getvalue()) + else: + raw_pixels = kernel_layer.read(fb.fb_info.screen_base, fb.size) + file_handle = open_method(f"{base_filename}.raw") + file_handle.write(raw_pixels) + + file_handle.close() + return file_handle.preferred_filename + + @classmethod + def parse_fb_info( + cls, + fb_info: interfaces.objects.ObjectInterface, + ) -> Framebuffer: + """Parse an fb_info struct + Args: + fb_info: an fb_info kernel object live instance + + Returns: + A Framebuffer object + + Documentation: + https://docs.kernel.org/fb/api.html: + - struct fb_fix_screeninfo stores device independent unchangeable information about the frame buffer device and the current format. + Those information can't be directly modified by applications, but can be changed by the driver when an application modifies the format. + - struct fb_var_screeninfo stores device independent changeable information about a frame buffer device, its current format and video mode, + as well as other miscellaneous parameters. + """ + # NotAvailableValue() messes with the filename output on disk + id = utility.array_to_string(fb_info.fix.id) or "N-A" + color_fields = None + + # 0 = color, 1 = grayscale, >1 = FOURCC + if fb_info.var.grayscale in [0, 1]: + color_fields = cls.parse_fb_pixel_bitfields(fb_info.var) + + # There a lot of tricky pixel formats used by drivers and vendors in include/uapi/linux/videodev2.h. + # As Volatility3 is not a video format converter, it is best to play it safe and let the user parse + # the raw data manually (with ffmpeg for example). + elif fb_info.var.grayscale > 1: + fourcc = linux.LinuxUtilities.convert_fourcc_code(fb_info.var.grayscale) + warn_msg = f"""Framebuffer "{id}" uses a FOURCC pixel format "{fourcc}" that isn't natively supported. +You can try using ffmpeg to decode the raw buffer. Example usage: +"ffmpeg -pix_fmts" to list supported formats, then +"ffmpeg -f rawvideo -video_size {fb_info.var.xres_virtual}x{fb_info.var.yres_virtual} -i .raw -pix_fmt output.png".""" + vollog.warning(warn_msg) + + # Prefer using the virtual resolution, instead of the visible one. + # This prevents missing non-visible data stored in the framebuffer. + fb = Framebuffer( + id, + xres_virtual=fb_info.var.xres_virtual, + yres_virtual=fb_info.var.yres_virtual, + line_length=fb_info.fix.line_length, + bpp=fb_info.var.bits_per_pixel, + size=fb_info.var.yres_virtual * fb_info.fix.line_length, + color_fields=color_fields, + fb_info=fb_info, + ) + + return fb + + def _generator(self): + kernel_name = self.config["kernel"] + kernel = self.context.modules[kernel_name] + + if not kernel.has_symbol("num_registered_fb"): + raise exceptions.SymbolError( + "num_registered_fb", + kernel.symbol_table_name, + "The provided symbol does not exist in the symbol table. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.", + ) + + num_registered_fb = kernel.object_from_symbol("num_registered_fb") + if num_registered_fb < 1: + vollog.info("No registered framebuffer in the fbdev API.") + return None + + registered_fb = kernel.object_from_symbol("registered_fb") + fb_info_list = utility.array_of_pointers( + registered_fb, + num_registered_fb, + kernel.symbol_table_name + constants.BANG + "fb_info", + self.context, + ) + + for fb_info in fb_info_list: + fb = self.parse_fb_info(fb_info) + file_output = "Disabled" + if self.config["dump"]: + try: + file_output = self.dump_fb( + self.context, kernel_name, self.open, fb, bool(fb.color_fields) + ) + except exceptions.InvalidAddressException as excp: + vollog.error( + f'Layer {excp.layer_name} failed to read address {hex(excp.invalid_address)} when dumping framebuffer "{fb.id}".' + ) + file_output = "Error" + + try: + fb_device_name = utility.pointer_to_string( + fb.fb_info.dev.kobj.name, 256 + ) + except exceptions.InvalidAddressException: + fb_device_name = NotAvailableValue() + + yield ( + 0, + ( + format_hints.Hex(fb.fb_info.screen_base), + fb_device_name, + fb.id, + fb.size, + f"{fb.xres_virtual}x{fb.yres_virtual}", + fb.bpp, + "RUNNING" if fb.fb_info.state == 0 else "SUSPENDED", + str(file_output), + ), + ) + + def run(self): + columns = [ + ("Address", format_hints.Hex), + ("Device", str), + ("ID", str), + ("Size", int), + ("Virtual resolution", str), + ("BPP", int), + ("State", str), + ("Filename", str), + ] + + return TreeGrid( + columns, + self._generator(), + ) From f192437a944f56ec3b8eae186d61f3049e691e80 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:11:11 +0100 Subject: [PATCH 575/979] typo --- volatility3/framework/plugins/linux/graphics/fbdev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 4bde6f62f..60e00d033 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -21,7 +21,7 @@ vollog = logging.getLogger(__name__) @dataclass class Framebuffer: - """Framebuffer object internal representation. This is useful to unify an framebuffer with precalculated + """Framebuffer object internal representation. This is useful to unify a framebuffer with precalculated properties and pass it through functions conveniently.""" id: str From 4c0ddca7962ade96747e16b7c2db3bd126e1af6a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 18 Nov 2024 15:20:38 +1100 Subject: [PATCH 576/979] linux: intel VMCoreInfo: BugFix. Return layer if both values are present --- volatility3/framework/automagic/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index c92fa05c4..adecc36d0 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -366,7 +366,7 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): dtb, ) - return layer + return layer vollog.debug("No suitable linux banner could be matched") return None From 4a88a7450b8fb33b29c7bdf1ee2a9703c67b5446 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 18 Nov 2024 16:47:45 +1100 Subject: [PATCH 577/979] linux: VMCoreInfo API: Integrate support for value parsing within the VMCoreInfo API --- volatility3/framework/automagic/linux.py | 20 +++++++++++-------- .../framework/plugins/linux/vmcoreinfo.py | 5 +++++ .../framework/symbols/linux/__init__.py | 17 ++++++++++++++-- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index adecc36d0..ff6bbbfe4 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -373,8 +373,8 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): @staticmethod def _vmcoreinfo_find_aslr(vmcoreinfo) -> Tuple[int, int]: - phys_base_str = vmcoreinfo.get("NUMBER(phys_base)") - if phys_base_str is None: + phys_base = vmcoreinfo.get("NUMBER(phys_base)") + if phys_base is None: # In kernel < 4.10, there may be a SYMBOL(phys_base), but as noted in the # c401721ecd1dcb0a428aa5d6832ee05ffbdbffbbe commit comment, this value # isn't useful for calculating the physical address. @@ -383,15 +383,15 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): return None # kernels 3.14 (b6085a865762236bb84934161273cdac6dd11c2d) KERNELOFFSET was added - kerneloffset_str = vmcoreinfo.get("KERNELOFFSET") - if kerneloffset_str is None: + kerneloffset = vmcoreinfo.get("KERNELOFFSET") + if kerneloffset is None: # kernels < 3.14 if KERNELOFFSET is missing, KASLR might not be implemented. # Oddly, NUMBER(phys_base) is present without it. To be safe, proceed only # if both are present. return None - aslr_shift = int(kerneloffset_str, 16) - kaslr_shift = int(phys_base_str) + aslr_shift + aslr_shift = kerneloffset + kaslr_shift = phys_base + aslr_shift return kaslr_shift, aslr_shift @@ -403,7 +403,11 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): # in kernels 4.13 in 65ade2f872b474fa8a04c2d397783350326634e6) to init_top_pgt. # In x86-32, the pgd is swapper_pg_dir. So, in any case, for VMCOREINFO # SYMBOL(swapper_pg_dir) will always have the right value. - dtb_vaddr = int(vmcoreinfo["SYMBOL(swapper_pg_dir)"], 16) + dtb_vaddr = vmcoreinfo.get("SYMBOL(swapper_pg_dir)") + if dtb_vaddr is None: + # Abort, it should be present + return None + dtb_paddr = ( LinuxIntelStacker.virtual_to_physical_address(dtb_vaddr) - aslr_shift @@ -420,7 +424,7 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): is_32bit = True else: # Check the swapper_pg_dir virtual address size - dtb_vaddr = int(vmcoreinfo["SYMBOL(swapper_pg_dir)"], 16) + dtb_vaddr = vmcoreinfo["SYMBOL(swapper_pg_dir)"] is_32bit = dtb_vaddr <= 2**32 return is_32bit, is_pae diff --git a/volatility3/framework/plugins/linux/vmcoreinfo.py b/volatility3/framework/plugins/linux/vmcoreinfo.py index ea30520e3..0f07f8589 100644 --- a/volatility3/framework/plugins/linux/vmcoreinfo.py +++ b/volatility3/framework/plugins/linux/vmcoreinfo.py @@ -38,6 +38,11 @@ class VMCoreInfo(plugins.PluginInterface): layer_name=layer_name, ): for key, value in vmcoreinfo.items(): + if key.startswith("SYMBOL(") or key == "KERNELOFFSET": + value = f"0x{value:x}" + else: + value = str(value) + yield 0, (format_hints.Hex(vmcoreinfo_offset), key, value) def run(self): diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index b0b6b7325..b8a2056d5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -842,8 +842,9 @@ class VMCoreInfo(interfaces.configuration.VersionableInterface): _version = (1, 0, 0) - @staticmethod + @classmethod def _vmcoreinfo_data_to_dict( + cls, vmcoreinfo_data, ) -> Optional[Dict[str, str]]: """Converts the input VMCoreInfo data buffer into a dictionary""" @@ -859,10 +860,22 @@ class VMCoreInfo(interfaces.configuration.VersionableInterface): break key, value = line.split("=", 1) - vmcoreinfo_dict[key] = value + vmcoreinfo_dict[key] = cls._parse_value(key, value) return vmcoreinfo_dict + @classmethod + def _parse_value(cls, key, value): + if key.startswith("SYMBOL(") or key == "KERNELOFFSET": + return int(value, 16) + elif key.startswith(("NUMBER(", "LENGTH(", "SIZE(", "OFFSET(")): + return int(value) + elif key == "PAGESIZE": + return int(value) + + # Default, as string + return value + @classmethod def search_vmcoreinfo_elf_note( cls, From f2a2f8aa679e2ec314049bdb721ad13376209514 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 18 Nov 2024 17:00:00 +1100 Subject: [PATCH 578/979] linux: intel VMCoreInfo: immediately abort processing the current VMCOREINFO if DTB is not found --- volatility3/framework/automagic/linux.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index ff6bbbfe4..64365bd83 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -295,12 +295,15 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): for _vmcoreinfo_offset, vmcoreinfo in vmcoreinfo_elf_notes_iter: shifts = cls._vmcoreinfo_find_aslr(vmcoreinfo) if not shifts: - # Let's try the next vmcoreinfo, in case this one isn't correct. + # Let's try the next VMCOREINFO, in case this one isn't correct. continue kaslr_shift, aslr_shift = shifts dtb = cls._vmcoreinfo_get_dtb(vmcoreinfo, aslr_shift, kaslr_shift) + if dtb is None: + # Discard this VMCOREINFO immediately + continue is_32bit, is_pae = cls._vmcoreinfo_is_32bit(vmcoreinfo) if is_32bit: @@ -358,7 +361,7 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): metadata={"os": "Linux"}, ) - if layer and dtb: + if layer: vollog.debug( "Values found in VMCOREINFO: KASLR=0x%x, ASLR=0x%x, DTB=0x%x", kaslr_shift, From 6d366f16ee84844bec5c49ee94a10655995ea3ed Mon Sep 17 00:00:00 2001 From: eve Date: Mon, 18 Nov 2024 06:38:06 +0000 Subject: [PATCH 579/979] Windows.vadregexscan: update imports --- volatility3/framework/plugins/windows/vadyarascan.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index efcc70d07..fd76bbc48 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -5,7 +5,8 @@ import logging from typing import Iterable, List, Tuple -from volatility3.framework import interfaces, renderers +from volatility3.framework import renderers +from volatility3.framework.interfaces import plugins, configuration, objects from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan @@ -14,14 +15,14 @@ from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) -class VadYaraScan(interfaces.plugins.PluginInterface): +class VadYaraScan(plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) _version = (1, 1, 1) @classmethod - def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + def get_requirements(cls) -> List[configuration.RequirementInterface]: # create a list of requirements for vadyarascan vadyarascan_requirements = [ requirements.ModuleRequirement( @@ -112,7 +113,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): @staticmethod def get_vad_maps( - task: interfaces.objects.ObjectInterface, + task: objects.ObjectInterface, ) -> Iterable[Tuple[int, int]]: """Creates a map of start/end addresses within a virtual address descriptor tree. From d80ef553255acf29ee16d884e864ec7c546a2be0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 18 Nov 2024 17:49:30 +1100 Subject: [PATCH 580/979] linux: intel VMCoreInfo: Select the fastest scanner for each scenario --- volatility3/framework/automagic/linux.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 64365bd83..81f5f5661 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -321,12 +321,20 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): if not valid_banners: # There's no banner matching this VMCOREINFO, keep trying with the next one continue + elif len(valid_banners) == 1: + # Usually, we narrow down the Linux banner list to a single element. + # Using BytesScanner here is slightly faster than MultiStringScanner. + scanner = scanners.BytesScanner(valid_banners[0]) + else: + scanner = scanners.MultiStringScanner(valid_banners) join = interfaces.configuration.path_join - mss = scanners.MultiStringScanner(valid_banners) - for _, banner in layer.scan( - context=context, scanner=mss, progress_callback=progress_callback + for match in layer.scan( + context=context, scanner=scanner, progress_callback=progress_callback ): + # Unfortunately, the scanners do not maintain a consistent interface + banner = match[1] if isinstance(match, Tuple) else valid_banners[0] + isf_path = linux_banners.get(banner, None) if not isf_path: vollog.warning( From 13729ac25f699d72c280b999ddbe0e9d02fcede6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 18 Nov 2024 18:18:14 +1100 Subject: [PATCH 581/979] linux: VMCoreInfo API: AARCH64 uses NUMBER() with hex values. Fortunately, includes the 0x prefix, allowing the parser to still handle these cases correctly --- volatility3/framework/symbols/linux/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index b8a2056d5..cd4d16649 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -869,9 +869,9 @@ class VMCoreInfo(interfaces.configuration.VersionableInterface): if key.startswith("SYMBOL(") or key == "KERNELOFFSET": return int(value, 16) elif key.startswith(("NUMBER(", "LENGTH(", "SIZE(", "OFFSET(")): - return int(value) + return int(value, 0) elif key == "PAGESIZE": - return int(value) + return int(value, 0) # Default, as string return value From 07701fc4cf9385422f9d06b89cf64640e6a76b0d Mon Sep 17 00:00:00 2001 From: eve Date: Mon, 18 Nov 2024 18:32:51 +0000 Subject: [PATCH 582/979] windows.vadregexscan: remove dependency on vadyarascan --- volatility3/framework/plugins/windows/vadregexscan.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadregexscan.py b/volatility3/framework/plugins/windows/vadregexscan.py index 8d17c469c..206c9faae 100644 --- a/volatility3/framework/plugins/windows/vadregexscan.py +++ b/volatility3/framework/plugins/windows/vadregexscan.py @@ -11,7 +11,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist, vadyarascan +from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) @@ -35,9 +35,6 @@ class VadRegExScan(plugins.PluginInterface): requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), - requirements.PluginRequirement( - name="vadyarascan", plugin=vadyarascan.VadYaraScan, version=(1, 0, 0) - ), requirements.ListRequirement( name="pid", description="Filter on specific process IDs", @@ -70,7 +67,11 @@ class VadRegExScan(plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] # get process sections for scanning - sections = vadyarascan.VadYaraScan.get_vad_maps(proc) + sections = [] + for vad in proc.get_vad_root().traverse(): + base = vad.get_start() + if vad.get_size(): + sections.append((base, vad.get_size())) for offset in proc_layer.scan( context=self.context, From e374ca96472279944bd5c0922f64e87f0b30e7e3 Mon Sep 17 00:00:00 2001 From: eve Date: Mon, 18 Nov 2024 21:15:06 +0000 Subject: [PATCH 583/979] Windows.vadregexscan: update imports, revert vadyarascan changes --- volatility3/framework/plugins/windows/vadregexscan.py | 6 +++--- volatility3/framework/plugins/windows/vadyarascan.py | 9 ++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/vadregexscan.py b/volatility3/framework/plugins/windows/vadregexscan.py index 206c9faae..0d35cd658 100644 --- a/volatility3/framework/plugins/windows/vadregexscan.py +++ b/volatility3/framework/plugins/windows/vadregexscan.py @@ -6,9 +6,9 @@ import logging import re from typing import List -from volatility3.framework import interfaces, renderers +from volatility3.framework import renderers from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins +from volatility3.framework.interfaces import plugins, configuration from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import pslist @@ -24,7 +24,7 @@ class VadRegExScan(plugins.PluginInterface): MAXSIZE_DEFAULT = 128 @classmethod - def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + def get_requirements(cls) -> List[configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ requirements.ModuleRequirement( diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index fd76bbc48..efcc70d07 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -5,8 +5,7 @@ import logging from typing import Iterable, List, Tuple -from volatility3.framework import renderers -from volatility3.framework.interfaces import plugins, configuration, objects +from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan @@ -15,14 +14,14 @@ from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) -class VadYaraScan(plugins.PluginInterface): +class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) _version = (1, 1, 1) @classmethod - def get_requirements(cls) -> List[configuration.RequirementInterface]: + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # create a list of requirements for vadyarascan vadyarascan_requirements = [ requirements.ModuleRequirement( @@ -113,7 +112,7 @@ class VadYaraScan(plugins.PluginInterface): @staticmethod def get_vad_maps( - task: objects.ObjectInterface, + task: interfaces.objects.ObjectInterface, ) -> Iterable[Tuple[int, int]]: """Creates a map of start/end addresses within a virtual address descriptor tree. From 1ba5b04b7d3ef1b9a8911eae973da10f4d33d7e5 Mon Sep 17 00:00:00 2001 From: eve Date: Mon, 18 Nov 2024 21:20:04 +0000 Subject: [PATCH 584/979] Windows: Remove dep on vadyarascan from svcscan plugin --- volatility3/framework/plugins/windows/svcscan.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 52ed5e759..ca390561f 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -20,7 +20,7 @@ from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions from volatility3.framework.symbols.windows.extensions import services as services_types -from volatility3.plugins.windows import poolscanner, pslist, vadyarascan +from volatility3.plugins.windows import poolscanner, pslist from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) @@ -39,7 +39,7 @@ class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) + _version = (3, 0, 1) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -60,9 +60,6 @@ class SvcScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) ), - requirements.PluginRequirement( - name="vadyarascan", plugin=vadyarascan.VadYaraScan, version=(1, 0, 0) - ), requirements.PluginRequirement( name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) ), @@ -317,10 +314,17 @@ class SvcScan(interfaces.plugins.PluginInterface): layer = context.layers[proc_layer_name] + # get process sections for scanning + sections = [] + for vad in task.get_vad_root().traverse(): + base = vad.get_start() + if vad.get_size(): + sections.append((base, vad.get_size())) + for offset in layer.scan( context=context, scanner=scanners.BytesScanner(needle=service_tag), - sections=vadyarascan.VadYaraScan.get_vad_maps(task), + sections=sections, ): if not is_vista_or_later: service_record = context.object( From 96f5fdae2f48b6a87f735daa3fc56e7aa2902c61 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 19 Nov 2024 11:25:52 +1100 Subject: [PATCH 585/979] linux: intel VMCoreInfo: Add the vmcoreinfo to the layer metadata --- volatility3/framework/automagic/linux.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 81f5f5661..2b7cdf4cc 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -2,8 +2,9 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import logging import os +import logging +import collections from typing import Optional, Tuple, Type from volatility3.framework import constants, interfaces @@ -107,11 +108,13 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): join(config_path, LinuxSymbolFinder.banner_config_key) ] = str(banner, "latin-1") + # Set an empty vmcoreinfo entry to prevent using the wrong one in the layer stack. + layer_metadata = dict(os="Linux", vmcoreinfo={}) layer = layer_class( context, config_path=config_path, name=new_layer_name, - metadata={"os": "Linux"}, + metadata=layer_metadata, ) layer.config["kernel_virtual_offset"] = aslr_shift @@ -376,6 +379,10 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): aslr_shift, dtb, ) + # Add the vmcoreinfo dict to the layer metadata + layer._direct_metadata = collections.ChainMap( + {"vmcoreinfo": vmcoreinfo}, layer._direct_metadata + ) return layer From 9f6d1949800981e7d5cc6416b661880fe8041a38 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 19 Nov 2024 12:47:56 +1100 Subject: [PATCH 586/979] linux: intel VMCoreInfo: The _direct_metadata class attribute needs to be modified; otherwise, it will only exist within the instance namespace. --- volatility3/framework/automagic/linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2b7cdf4cc..8c4ccd8e3 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -379,8 +379,8 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): aslr_shift, dtb, ) - # Add the vmcoreinfo dict to the layer metadata - layer._direct_metadata = collections.ChainMap( + # Add the vmcoreinfo dict to the layer class metadata + layer.__class__._direct_metadata = collections.ChainMap( {"vmcoreinfo": vmcoreinfo}, layer._direct_metadata ) From 41d2e924d147aed9adfa2181aa6191e26fa90568 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 20 Nov 2024 11:47:37 +1100 Subject: [PATCH 587/979] linux: intel VMCoreInfo: Revert changes associated with adding vmcoreinfo dict to the layer metadata --- volatility3/framework/automagic/linux.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 8c4ccd8e3..b8ed47220 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -4,7 +4,6 @@ import os import logging -import collections from typing import Optional, Tuple, Type from volatility3.framework import constants, interfaces @@ -108,13 +107,11 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): join(config_path, LinuxSymbolFinder.banner_config_key) ] = str(banner, "latin-1") - # Set an empty vmcoreinfo entry to prevent using the wrong one in the layer stack. - layer_metadata = dict(os="Linux", vmcoreinfo={}) layer = layer_class( context, config_path=config_path, name=new_layer_name, - metadata=layer_metadata, + metadata={os: "Linux"}, ) layer.config["kernel_virtual_offset"] = aslr_shift @@ -379,10 +376,6 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): aslr_shift, dtb, ) - # Add the vmcoreinfo dict to the layer class metadata - layer.__class__._direct_metadata = collections.ChainMap( - {"vmcoreinfo": vmcoreinfo}, layer._direct_metadata - ) return layer From 1abb4cc881d052b4f76df1e52bc4d0d9521c8c0c Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 11:13:21 +0100 Subject: [PATCH 588/979] comply with xdg base directory spec by using XDG_CACHE_HOME if its set --- volatility3/framework/constants/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 27fae4ba1..84c9c22ce 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -6,6 +6,7 @@ Stores all the constant values that are generally fixed throughout volatility This includes default scanning block sizes, etc. """ + import enum import os.path import sys @@ -65,7 +66,10 @@ LOGLEVEL_VVV = 7 LOGLEVEL_VVVV = 6 """Logging level for four levels of detail: -vvvvvv""" -CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") + +CACHE_PATH = os.path.join( + os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache"), "volatility3" +) """Default path to store cached data""" SQLITE_CACHE_PERIOD = "-3 days" From b93d5b7c13298ed2c1a24a951569a8e93f444c26 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 11:23:53 +0100 Subject: [PATCH 589/979] update docs --- doc/source/symbol-tables.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index b7c26e046..722f9e468 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -9,7 +9,7 @@ How Volatility finds symbol tables All files are stored as JSON data, they can be in pure JSON files as ``.json``, or compressed as ``.json.gz`` or ``.json.xz``. Volatility will automatically decompress them on use. It will also cache their contents (compressed) when used, located -under the user's home directory, in :file:`.cache/volatility3`, along with other useful data. The cache directory currently +under the user's home directory, in :file:`.cache/volatility3` or when `XDG_CACHE_HOME` is set in :file:`${XDG_CACHE_HOME}/volatility3`, along with other useful data. The cache directory currently cannot be altered. Symbol table JSON files live, by default, under the :file:`volatility3/symbols` directory. The symbols directory is From a2b96c0dc0b2989e87e4eb4a9c2a274ef4ed6e7f Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 11:46:59 +0100 Subject: [PATCH 590/979] fix: dont use `/` for compat --- volatility3/framework/constants/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 84c9c22ce..8bdf84730 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -68,7 +68,8 @@ LOGLEVEL_VVVV = 6 CACHE_PATH = os.path.join( - os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache"), "volatility3" + os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache"), + "volatility3", ) """Default path to store cached data""" From bbd2f9bc3770d798c7e225586f5f78e22400970d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 22 Nov 2024 22:40:53 +1100 Subject: [PATCH 591/979] linux: intel VMCoreInfo: Fix bug introduced in last commit reverting the layer metadata dict --- volatility3/framework/automagic/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index b8ed47220..90384b50d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -111,7 +111,7 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): context, config_path=config_path, name=new_layer_name, - metadata={os: "Linux"}, + metadata={"os": "Linux"}, ) layer.config["kernel_virtual_offset"] = aslr_shift From 76ecfc08830e4b7def60520ab423a8d439e8fdbe Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 22 Nov 2024 12:16:14 +0000 Subject: [PATCH 592/979] Core: Add in generic cache-manager chooser function --- volatility3/framework/automagic/linux.py | 17 ++--------------- volatility3/framework/automagic/mac.py | 17 ++--------------- volatility3/framework/automagic/symbol_cache.py | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 30 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 52a73f45a..6fe18a4a9 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -27,16 +27,6 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): progress_callback: constants.ProgressCallback = None, ) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify linux within this layer.""" - # Version check the SQlite cache - required = (1, 0, 0) - if not requirements.VersionRequirement.matches_required( - required, symbol_cache.SqliteCache.version - ): - vollog.info( - f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}" - ) - return None - # Bail out by default unless we can stack properly layer = context.layers[layer_name] join = interfaces.configuration.path_join @@ -46,12 +36,9 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - identifiers_path = os.path.join( - constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + linux_banners = symbol_cache.load_cache_manager().get_identifier_dictionary( + operating_system="linux" ) - linux_banners = symbol_cache.SqliteCache( - identifiers_path - ).get_identifier_dictionary(operating_system="linux") # If we have no banners, don't bother scanning if not linux_banners: vollog.info( diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index e51753139..7c478b521 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -28,16 +28,6 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): progress_callback: constants.ProgressCallback = None, ) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify mac within this layer.""" - # Version check the SQlite cache - required = (1, 0, 0) - if not requirements.VersionRequirement.matches_required( - required, symbol_cache.SqliteCache.version - ): - vollog.info( - f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}" - ) - return None - # Bail out by default unless we can stack properly layer = context.layers[layer_name] new_layer = None @@ -48,12 +38,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - identifiers_path = os.path.join( - constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + mac_banners = symbol_cache.load_cache_manager().get_identifier_dictionary( + operating_system="mac" ) - mac_banners = symbol_cache.SqliteCache( - identifiers_path - ).get_identifier_dictionary(operating_system="mac") # If we have no banners, don't bother scanning if not mac_banners: vollog.info( diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 22f1c94f3..e38771f79 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -492,6 +492,21 @@ class SqliteCache(CacheManagerInterface): return output +def load_cache_manager(cache_file: Optional[str] = None) -> CacheManagerInterface: + """Loads a cache manager based on a specific cache file""" + if cache_file is None: + cache_file = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + # Different implementations of cache + if not os.path.exists(cache_file): + raise ValueError("Non-existant cache file provided") + with open(cache_file, "rb") as fp: + header = fp.read(4) + if header not in [b"SQLi"]: + raise ValueError("Identifier file not in recognized format") + # Currently only one choice, so use that + return SqliteCache(cache_file) + + ### Automagic From 4bc6e1001df8b60586309e56abb4ed7892b98c5d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 22 Nov 2024 19:45:15 +0000 Subject: [PATCH 593/979] Windows: Improve logging of slowscan to show possible PDB entries --- volatility3/framework/automagic/pdbscan.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 7f38a23e1..1d5bf55ea 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -270,6 +270,10 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): progress_callback=progress_callback, ) for kernel in kernels: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1)} with MZ offset at {kernel.get('mz_offset', -1)}", + ) valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel) if valid_kernel is not None: break From 1a0d1a238da02db679c0bc5dfa10191abe25b6b7 Mon Sep 17 00:00:00 2001 From: lesander <4174509+lesander@users.noreply.github.com> Date: Sun, 24 Nov 2024 09:28:41 +0000 Subject: [PATCH 594/979] CLI: Partial changes by @lesander contributed in #1343 --- .github/workflows/test.yaml | 4 ++-- volatility3/cli/__init__.py | 5 +++-- volatility3/cli/volshell/__init__.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6358dd45d..73bf342b6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -41,8 +41,8 @@ jobs: - name: Testing... run: | - py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v - py.test ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v + pytest ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v + pytest ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v - name: Clean up post-test run: | diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 75b62abf6..5b2aa2838 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -88,7 +88,7 @@ class MuteProgress(PrintedProgress): class CommandLine: """Constructs a command-line interface object for users to run plugins.""" - CLI_NAME = "volatility" + CLI_NAME = os.path.basename(sys.argv[0]) # vol or volatility def __init__(self): self.setup_logging() @@ -364,6 +364,7 @@ class CommandLine: self.CLI_NAME ), action=volargparse.HelpfulSubparserAction, + metavar="PLUGIN", ) for plugin in sorted(plugin_list): plugin_parser = subparser.add_parser( @@ -385,7 +386,7 @@ class CommandLine: argcomplete.autocomplete(parser) args = parser.parse_args() if args.plugin is None: - parser.error("Please select a plugin to run") + parser.error(f"Please select a plugin to run (see '{self.CLI_NAME} --help' for options") vollog.log( constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}" diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 5172c5363..559adc12b 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -49,7 +49,7 @@ class VolShell(cli.CommandLine): python terminal with all the volatility support calls available. """ - CLI_NAME = "volshell" + CLI_NAME = os.path.basename(sys.argv[0]) # volshell def __init__(self): super().__init__() From 523a6e92fc75a972fe5d9a73c6dc65d4322a7f46 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 24 Nov 2024 09:33:30 +0000 Subject: [PATCH 595/979] CLI: Apply black to recent changes --- volatility3/cli/__init__.py | 6 ++++-- volatility3/cli/volshell/__init__.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 5b2aa2838..901f299a8 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -88,7 +88,7 @@ class MuteProgress(PrintedProgress): class CommandLine: """Constructs a command-line interface object for users to run plugins.""" - CLI_NAME = os.path.basename(sys.argv[0]) # vol or volatility + CLI_NAME = os.path.basename(sys.argv[0]) # vol or volatility def __init__(self): self.setup_logging() @@ -386,7 +386,9 @@ class CommandLine: argcomplete.autocomplete(parser) args = parser.parse_args() if args.plugin is None: - parser.error(f"Please select a plugin to run (see '{self.CLI_NAME} --help' for options") + parser.error( + f"Please select a plugin to run (see '{self.CLI_NAME} --help' for options" + ) vollog.log( constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}" diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 559adc12b..e9d3fda08 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -49,7 +49,7 @@ class VolShell(cli.CommandLine): python terminal with all the volatility support calls available. """ - CLI_NAME = os.path.basename(sys.argv[0]) # volshell + CLI_NAME = os.path.basename(sys.argv[0]) # volshell def __init__(self): super().__init__() From 641e008caf34b93b3c12360115066e1e291500ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 27 Nov 2024 15:06:33 +1100 Subject: [PATCH 596/979] Linux: intel: The non-present mapping shouldn't be inverted --- volatility3/framework/layers/intel.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index dc88c20bf..6d0aa6746 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -563,8 +563,9 @@ class LinuxMixin(Intel): return self._is_pte_present(entry) def _pte_needs_invert(self, entry) -> bool: - # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted - return not (entry & self._PAGE_PRESENT) + # Entries that were set to PROT_NONE (PAGE_PRESENT) are inverted + # A clear PTE shouldn't be inverted. See f19f5c4 + return entry and not (entry & self._PAGE_PRESENT) def _protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" From eff2a529c8e2e264553aca0550a5cd9ce3c9c298 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 27 Nov 2024 18:10:14 +1100 Subject: [PATCH 597/979] Linux: intel: Remove the bitwise zero complement to simplify the protnone mask calculation --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 6d0aa6746..d762b41a8 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -569,7 +569,7 @@ class LinuxMixin(Intel): def _protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" - return ~0 & self._register_mask if self._pte_needs_invert(entry) else 0 + return self._register_mask if self._pte_needs_invert(entry) else 0 def _pte_pfn(self, entry: int) -> int: """Extracts the page frame number from the page table entry""" From 7c559c53f4bad23557056fec14a633628e1d5fe2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 12:11:14 +1100 Subject: [PATCH 598/979] test_cases: underscore unused variables --- test/test_volatility.py | 91 ++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 41 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 847be88d9..2f3ba3c63 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -61,7 +61,7 @@ def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]) def test_windows_pslist(image, volatility, python): - rc, out, err = runvol_plugin("windows.pslist.PsList", image, volatility, python) + rc, out, _err = runvol_plugin("windows.pslist.PsList", image, volatility, python) out = out.lower() assert out.find(b"system") != -1 assert out.find(b"csrss.exe") != -1 @@ -69,7 +69,7 @@ def test_windows_pslist(image, volatility, python): assert out.count(b"\n") > 10 assert rc == 0 - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"] ) out = out.lower() @@ -79,7 +79,7 @@ def test_windows_pslist(image, volatility, python): def test_windows_psscan(image, volatility, python): - rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) + rc, out, _err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) out = out.lower() assert out.find(b"system") != -1 assert out.find(b"csrss.exe") != -1 @@ -89,21 +89,21 @@ def test_windows_psscan(image, volatility, python): def test_windows_dlllist(image, volatility, python): - rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) + rc, out, _err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 def test_windows_modules(image, volatility, python): - rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) + rc, out, _err = runvol_plugin("windows.modules.Modules", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 def test_windows_hivelist(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.registry.hivelist.HiveList", image, volatility, python ) out = out.lower() @@ -136,7 +136,7 @@ def test_windows_dumpfiles(image, volatility, python): path = tempfile.mkdtemp() - rc, out, err = runvol_plugin( + rc, _out, _err = runvol_plugin( "windows.dumpfiles.DumpFiles", image, volatility, @@ -166,7 +166,7 @@ def test_windows_dumpfiles(image, volatility, python): def test_windows_handles(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.handles.Handles", image, volatility, python, pluginargs=["--pid", "4"] ) @@ -183,7 +183,7 @@ def test_windows_handles(image, volatility, python): def test_windows_svcscan(image, volatility, python): - rc, out, err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) + rc, out, _err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) assert out.find(b"Microsoft ACPI Driver") != -1 assert out.count(b"\n") > 250 @@ -191,17 +191,19 @@ def test_windows_svcscan(image, volatility, python): def test_windows_thrdscan(image, volatility, python): - rc, out, err = runvol_plugin("windows.thrdscan.ThrdScan", image, volatility, python) + rc, out, _err = runvol_plugin( + "windows.thrdscan.ThrdScan", image, volatility, python + ) # find pid 4 (of system process) which starts with lowest tids assert out.find(b"\t4\t8") != -1 assert out.find(b"\t4\t12") != -1 assert out.find(b"\t4\t16") != -1 - #assert out.find(b"this raieses AssertionError") != -1 + # assert out.find(b"this raieses AssertionError") != -1 assert rc == 0 def test_windows_privileges(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"] ) @@ -213,7 +215,7 @@ def test_windows_privileges(image, volatility, python): def test_windows_getsids(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.getsids.GetSIDs", image, volatility, python, pluginargs=["--pid", "4"] ) @@ -225,7 +227,7 @@ def test_windows_getsids(image, volatility, python): def test_windows_envars(image, volatility, python): - rc, out, err = runvol_plugin("windows.envars.Envars", image, volatility, python) + rc, out, _err = runvol_plugin("windows.envars.Envars", image, volatility, python) assert out.find(b"PATH") != -1 assert out.find(b"PROCESSOR_ARCHITECTURE") != -1 @@ -237,7 +239,7 @@ def test_windows_envars(image, volatility, python): def test_windows_callbacks(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.callbacks.Callbacks", image, volatility, python ) @@ -249,7 +251,7 @@ def test_windows_callbacks(image, volatility, python): def test_windows_vadwalk(image, volatility, python): - rc, out, err = runvol_plugin("windows.vadwalk.VadWalk", image, volatility, python) + rc, out, _err = runvol_plugin("windows.vadwalk.VadWalk", image, volatility, python) assert out.find(b"Vad") != -1 assert out.find(b"VadS") != -1 @@ -260,7 +262,7 @@ def test_windows_vadwalk(image, volatility, python): def test_windows_devicetree(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.devicetree.DeviceTree", image, volatility, python ) @@ -277,7 +279,7 @@ def test_windows_devicetree(image, volatility, python): def test_linux_pslist(image, volatility, python): - rc, out, err = runvol_plugin("linux.pslist.PsList", image, volatility, python) + rc, out, _err = runvol_plugin("linux.pslist.PsList", image, volatility, python) out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) @@ -287,7 +289,9 @@ def test_linux_pslist(image, volatility, python): def test_linux_check_idt(image, volatility, python): - rc, out, err = runvol_plugin("linux.check_idt.Check_idt", image, volatility, python) + rc, out, _err = runvol_plugin( + "linux.check_idt.Check_idt", image, volatility, python + ) out = out.lower() assert out.count(b"__kernel__") >= 10 @@ -296,7 +300,7 @@ def test_linux_check_idt(image, volatility, python): def test_linux_check_syscall(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "linux.check_syscall.Check_syscall", image, volatility, python ) out = out.lower() @@ -308,7 +312,7 @@ def test_linux_check_syscall(image, volatility, python): def test_linux_lsmod(image, volatility, python): - rc, out, err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) + rc, out, _err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 @@ -316,7 +320,7 @@ def test_linux_lsmod(image, volatility, python): def test_linux_lsof(image, volatility, python): - rc, out, err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) + rc, out, _err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) out = out.lower() assert out.count(b"socket:") >= 10 @@ -325,7 +329,7 @@ def test_linux_lsof(image, volatility, python): def test_linux_proc_maps(image, volatility, python): - rc, out, err = runvol_plugin("linux.proc.Maps", image, volatility, python) + rc, out, _err = runvol_plugin("linux.proc.Maps", image, volatility, python) out = out.lower() assert out.count(b"anonymous mapping") >= 10 @@ -334,15 +338,18 @@ def test_linux_proc_maps(image, volatility, python): def test_linux_tty_check(image, volatility, python): - rc, out, err = runvol_plugin("linux.tty_check.tty_check", image, volatility, python) + rc, out, _err = runvol_plugin( + "linux.tty_check.tty_check", image, volatility, python + ) out = out.lower() assert out.find(b"__kernel__") != -1 assert out.count(b"\n") >= 5 assert rc == 0 + def test_linux_sockstat(image, volatility, python): - rc, out, err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) + rc, out, _err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) assert out.count(b"AF_UNIX") >= 354 assert out.count(b"AF_BLUETOOTH") >= 5 @@ -354,7 +361,7 @@ def test_linux_sockstat(image, volatility, python): def test_linux_library_list(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "linux.library_list.LibraryList", image, volatility, python ) @@ -383,7 +390,7 @@ def test_linux_library_list(image, volatility, python): def test_mac_pslist(image, volatility, python): - rc, out, err = runvol_plugin("mac.pslist.PsList", image, volatility, python) + rc, out, _err = runvol_plugin("mac.pslist.PsList", image, volatility, python) out = out.lower() assert (out.find(b"kernel_task") != -1) or (out.find(b"launchd") != -1) @@ -392,7 +399,7 @@ def test_mac_pslist(image, volatility, python): def test_mac_check_syscall(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.check_syscall.Check_syscall", image, volatility, python ) out = out.lower() @@ -405,7 +412,7 @@ def test_mac_check_syscall(image, volatility, python): def test_mac_check_sysctl(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.check_sysctl.Check_sysctl", image, volatility, python ) out = out.lower() @@ -416,7 +423,7 @@ def test_mac_check_sysctl(image, volatility, python): def test_mac_check_trap_table(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.check_trap_table.Check_trap_table", image, volatility, python ) out = out.lower() @@ -427,7 +434,7 @@ def test_mac_check_trap_table(image, volatility, python): def test_mac_ifconfig(image, volatility, python): - rc, out, err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) + rc, out, _err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) out = out.lower() assert out.find(b"127.0.0.1") != -1 @@ -437,7 +444,7 @@ def test_mac_ifconfig(image, volatility, python): def test_mac_lsmod(image, volatility, python): - rc, out, err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) + rc, out, _err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) out = out.lower() assert out.find(b"com.apple") != -1 @@ -446,7 +453,7 @@ def test_mac_lsmod(image, volatility, python): def test_mac_lsof(image, volatility, python): - rc, out, err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) + rc, out, _err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) out = out.lower() assert out.count(b"\n") > 50 @@ -454,7 +461,7 @@ def test_mac_lsof(image, volatility, python): def test_mac_malfind(image, volatility, python): - rc, out, err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) + rc, out, _err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) out = out.lower() assert out.count(b"\n") > 20 @@ -462,7 +469,7 @@ def test_mac_malfind(image, volatility, python): def test_mac_mount(image, volatility, python): - rc, out, err = runvol_plugin("mac.mount.Mount", image, volatility, python) + rc, out, _err = runvol_plugin("mac.mount.Mount", image, volatility, python) out = out.lower() assert out.find(b"/dev") != -1 @@ -471,7 +478,7 @@ def test_mac_mount(image, volatility, python): def test_mac_netstat(image, volatility, python): - rc, out, err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) + rc, out, _err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) assert out.find(b"TCP") != -1 assert out.find(b"UDP") != -1 @@ -481,7 +488,7 @@ def test_mac_netstat(image, volatility, python): def test_mac_proc_maps(image, volatility, python): - rc, out, err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) + rc, out, _err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) out = out.lower() assert out.find(b"[heap]") != -1 @@ -490,7 +497,7 @@ def test_mac_proc_maps(image, volatility, python): def test_mac_psaux(image, volatility, python): - rc, out, err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) + rc, out, _err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) out = out.lower() assert out.find(b"executable_path") != -1 @@ -499,7 +506,7 @@ def test_mac_psaux(image, volatility, python): def test_mac_socket_filters(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.socket_filters.Socket_filters", image, volatility, python ) out = out.lower() @@ -509,7 +516,7 @@ def test_mac_socket_filters(image, volatility, python): def test_mac_timers(image, volatility, python): - rc, out, err = runvol_plugin("mac.timers.Timers", image, volatility, python) + rc, out, _err = runvol_plugin("mac.timers.Timers", image, volatility, python) out = out.lower() assert out.count(b"\n") > 6 @@ -517,7 +524,9 @@ def test_mac_timers(image, volatility, python): def test_mac_trustedbsd(image, volatility, python): - rc, out, err = runvol_plugin("mac.trustedbsd.Trustedbsd", image, volatility, python) + rc, out, _err = runvol_plugin( + "mac.trustedbsd.Trustedbsd", image, volatility, python + ) out = out.lower() assert out.count(b"\n") > 10 From 97534f02de61202475c5c3aa9418488d886f744c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 12:19:56 +1100 Subject: [PATCH 599/979] linux: sockstat: It should import and verify the pslist version directly, instead of relying on lsof --- volatility3/framework/plugins/linux/sockstat.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 0ddd3e26d..e5cf48d16 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -12,6 +12,7 @@ from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.framework.symbols import linux from volatility3.plugins.linux import lsof +from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) @@ -21,7 +22,6 @@ class SockHandlers(interfaces.configuration.VersionableInterface): """Handles several socket families extracting the sockets information.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) def __init__(self, vmlinux, task, *args, **kwargs): @@ -438,8 +438,7 @@ class Sockstat(plugins.PluginInterface): """Lists all network connections for all processes.""" _required_framework_version = (2, 0, 0) - - _version = (3, 0, 0) + _version = (3, 0, 1) @classmethod def get_requirements(cls): @@ -455,6 +454,9 @@ class Sockstat(plugins.PluginInterface): requirements.PluginRequirement( name="lsof", plugin=lsof.Lsof, version=(2, 0, 0) ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) ), @@ -591,7 +593,7 @@ class Sockstat(plugins.PluginInterface): tasks: String with a list of tasks and FDs using a socket. It can also have extended information such as socket filters, bpf info, etc. """ - filter_func = lsof.pslist.PsList.create_pid_filter(pids) + filter_func = pslist.PsList.create_pid_filter(pids) socket_generator = self.list_sockets( self.context, symbol_table, filter_func=filter_func ) From fa060646352d41cf6d818e6cb8c40deaa91e7e7f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 13:16:33 +1100 Subject: [PATCH 600/979] linux plugins: Update pslist major version and its 18 dependent plugins --- volatility3/framework/plugins/linux/bash.py | 3 ++- volatility3/framework/plugins/linux/boottime.py | 4 ++-- .../framework/plugins/linux/capabilities.py | 7 +++---- .../framework/plugins/linux/check_creds.py | 5 ++--- volatility3/framework/plugins/linux/elfs.py | 4 ++-- volatility3/framework/plugins/linux/envars.py | 3 ++- volatility3/framework/plugins/linux/kthreads.py | 5 ++--- .../framework/plugins/linux/library_list.py | 5 ++--- volatility3/framework/plugins/linux/lsof.py | 4 ++-- volatility3/framework/plugins/linux/malfind.py | 5 +++-- volatility3/framework/plugins/linux/mountinfo.py | 5 ++--- .../framework/plugins/linux/pidhashtable.py | 9 ++++----- volatility3/framework/plugins/linux/proc.py | 5 +++-- volatility3/framework/plugins/linux/psaux.py | 3 ++- volatility3/framework/plugins/linux/pslist.py | 3 +-- volatility3/framework/plugins/linux/pstree.py | 15 +++++++-------- volatility3/framework/plugins/linux/ptrace.py | 4 ++-- .../framework/plugins/linux/vmaregexscan.py | 5 +++-- .../framework/plugins/linux/vmayarascan.py | 4 ++-- 19 files changed, 48 insertions(+), 50 deletions(-) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index ce4567ca6..77a433a3b 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -22,6 +22,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): """Recovers bash command history from memory.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -32,7 +33,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py index 8f63ee7f8..3df5cb3ac 100644 --- a/volatility3/framework/plugins/linux/boottime.py +++ b/volatility3/framework/plugins/linux/boottime.py @@ -16,7 +16,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) _required_framework_version = (2, 11, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -27,7 +27,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 3, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index bfdb69aba..a8a8fb1fa 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -50,8 +50,7 @@ class Capabilities(plugins.PluginInterface): """Lists process capabilities""" _required_framework_version = (2, 0, 0) - - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -62,7 +61,7 @@ class Capabilities(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pids", @@ -87,7 +86,7 @@ class Capabilities(plugins.PluginInterface): try: kernel_cap_last_cap = vmlinux.object_from_symbol(symbol_name="cap_last_cap") except exceptions.SymbolError: - # It should be a kernel < 3.2 + # It should be a kernel < 3.2 See 73efc0394e148d0e15583e13712637831f926720 return None vol2_last_cap = extensions.kernel_cap_struct.get_last_cap_value() diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index b7f73c3eb..0857576d5 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -12,8 +12,7 @@ class Check_creds(interfaces.plugins.PluginInterface): """Checks if any processes are sharing credential structures""" _required_framework_version = (2, 0, 0) - - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls): @@ -24,7 +23,7 @@ class Check_creds(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 22e39d127..9f3bd274b 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -25,7 +25,7 @@ class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (2, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -36,7 +36,7 @@ class Elfs(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 5cbf0f502..22aba6408 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -17,6 +17,7 @@ class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -28,7 +29,7 @@ class Envars(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index 2e51b4688..b9ced73f3 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -20,8 +20,7 @@ class Kthreads(plugins.PluginInterface): """Enumerates kthread functions""" _required_framework_version = (2, 11, 0) - - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -35,7 +34,7 @@ class Kthreads(plugins.PluginInterface): name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 3, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py index 062ed078e..7ec1f7f7f 100644 --- a/volatility3/framework/plugins/linux/library_list.py +++ b/volatility3/framework/plugins/linux/library_list.py @@ -21,8 +21,7 @@ class LibraryList(interfaces.plugins.PluginInterface): """Enumerate libraries loaded into processes""" _required_framework_version = (2, 0, 0) - - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -33,7 +32,7 @@ class LibraryList(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pids", diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 42b447dfb..802954f43 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -110,7 +110,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists open files for each processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -121,7 +121,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 18f3dcd56..0b10e60c6 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -4,7 +4,7 @@ from typing import List import logging -from volatility3.framework import constants, interfaces +from volatility3.framework import interfaces from volatility3.framework import renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility @@ -18,6 +18,7 @@ class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -28,7 +29,7 @@ class Malfind(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 2499f009e..65775c4aa 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -36,8 +36,7 @@ class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 2, 0) - - _version = (1, 2, 1) + _version = (1, 2, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -48,7 +47,7 @@ class MountInfo(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index edafe97e0..2d210c233 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -19,8 +19,7 @@ class PIDHashTable(plugins.PluginInterface): """Enumerates processes through the PID hash table""" _required_framework_version = (2, 0, 0) - - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -31,7 +30,7 @@ class PIDHashTable(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) @@ -236,8 +235,8 @@ class PIDHashTable(plugins.PluginInterface): self, decorate_comm: bool = False ) -> interfaces.objects.ObjectInterface: for task in self.get_tasks(): - offset, pid, tid, ppid, name = pslist.PsList.get_task_fields( - task, decorate_comm + offset, pid, tid, ppid, name, _creation_time = ( + pslist.PsList.get_task_fields(task, decorate_comm) ) fields = format_hints.Hex(offset), pid, tid, ppid, name yield 0, fields diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index e7d38b107..00832140a 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -21,7 +21,8 @@ class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) + MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @classmethod @@ -34,7 +35,7 @@ class Maps(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index a4a23498f..5467c3b4c 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -15,6 +15,7 @@ class PsAux(plugins.PluginInterface): """Lists processes with their command line arguments""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -26,7 +27,7 @@ class PsAux(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index b05d69c7a..6460462a7 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -18,8 +18,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 0, 0) - - _version = (2, 3, 0) + _version = (3, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index efe5223df..9dc5ea3cc 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -13,6 +13,7 @@ class PsTree(interfaces.plugins.PluginInterface): ID.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -24,7 +25,7 @@ class PsTree(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", @@ -100,13 +101,11 @@ class PsTree(interfaces.plugins.PluginInterface): def yield_processes(pid): task = self._tasks[pid] - row = pslist.PsList.get_task_fields(task, decorate_comm) - # update the first element, the offset, in the row tuple to use format_hints.Hex - # as a simple int is returned from get_task_fields. - row = (format_hints.Hex(row[0]),) + row[1:] - - tid = task.pid - yield (self._levels[tid] - 1, row) + offset, pid, tid, ppid, name, _creation_time = ( + pslist.PsList.get_task_fields(task, decorate_comm) + ) + fields = format_hints.Hex(offset), pid, tid, ppid, name + yield (self._levels[tid] - 1, fields) for child_pid in sorted(self._children.get(tid, [])): yield from yield_processes(child_pid) diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index e467ee644..271c0e75e 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -19,7 +19,7 @@ class Ptrace(plugins.PluginInterface): """Enumerates ptrace's tracer and tracee tasks""" _required_framework_version = (2, 10, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -30,7 +30,7 @@ class Ptrace(plugins.PluginInterface): architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/vmaregexscan.py b/volatility3/framework/plugins/linux/vmaregexscan.py index 77ab29814..4446fc550 100644 --- a/volatility3/framework/plugins/linux/vmaregexscan.py +++ b/volatility3/framework/plugins/linux/vmaregexscan.py @@ -21,7 +21,8 @@ class VmaRegExScan(plugins.PluginInterface): """Scans all virtual memory areas for tasks using RegEx.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) + MAXSIZE_DEFAULT = 128 @classmethod @@ -34,7 +35,7 @@ class VmaRegExScan(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 9fe06b0c8..38d6cbaac 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -15,7 +15,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -28,7 +28,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): optional=True, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(3, 0, 0) ), requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) From 63ac0abe642b4ba7fe93edfd2be0d6dd32e0483d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 13:25:08 +1100 Subject: [PATCH 601/979] linux: Add 15 new test cases for plugins dependent on pslist --- test/test_volatility.py | 165 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index 2f3ba3c63..4e948a2bd 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -386,6 +386,171 @@ def test_linux_library_list(image, volatility, python): assert rc == 0 +def test_linux_pstree(image, volatility, python): + rc, out, _err = runvol_plugin("linux.pstree.PsTree", image, volatility, python) + out = out.lower() + + assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_pidhashtable(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.pidhashtable.PIDHashTable", image, volatility, python + ) + out = out.lower() + + assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_bash(image, volatility, python): + rc, out, _err = runvol_plugin("linux.bash.Bash", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_boottime(image, volatility, python): + rc, out, _err = runvol_plugin("linux.boottime.Boottime", image, volatility, python) + out = out.lower() + + assert out.count(b"utc") >= 1 + assert rc == 0 + + +def test_linux_capabilities(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.capabilities.Capabilities", + image, + volatility, + python, + globalargs=["-vvv"], + ) + if rc != 0 and err.count(b"Unsupported kernel capabilities implementation") > 0: + # The linux-sample-1.bin kernel implementation isn't supported. + # However, we can still check that the plugin requirements are met. + return None + + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_check_creds(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.check_creds.Check_creds", image, volatility, python + ) + out = out.lower() + + # linux-sample-1.bin has no processes sharing credentials. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + + +def test_linux_elfs(image, volatility, python): + rc, out, _err = runvol_plugin("linux.elfs.Elfs", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_envars(image, volatility, python): + rc, out, _err = runvol_plugin("linux.envars.Envars", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_kthreads(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.kthreads.Kthreads", + image, + volatility, + python, + globalargs=["-vvv"], + ) + out = out.lower() + + if rc != 0 and err.count(b"Unsupported kthread implementation") > 0: + # The linux-sample-1.bin kernel implementation isn't supported. + # However, we can still check that the plugin requirements are met. + return None + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_malfind(image, volatility, python): + rc, out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) + out = out.lower() + + # linux-sample-1.bin has no process memory ranges with potential injected code. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + + +def test_linux_mountinfo(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.mountinfo.MountInfo", image, volatility, python + ) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_psaux(image, volatility, python): + rc, out, _err = runvol_plugin("linux.psaux.PsAux", image, volatility, python) + out = out.lower() + + assert out.count(b"\n") > 50 + assert rc == 0 + + +def test_linux_ptrace(image, volatility, python): + rc, out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) + out = out.lower() + + # linux-sample-1.bin has no processes being ptreaced. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + + +def test_linux_vmaregexscan(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.vmaregexscan.VmaRegExScan", + image, + volatility, + python, + pluginargs=["--pid", "1", "--pattern", "\\x7fELF"], + ) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + +def test_linux_vmayarascan(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.vmayarascan.VmaYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "1", "--yara-string", "ELF"], + ) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + # MAC From fb4ab9d4778b61e3a7af7761acac29f3bfe17ffd Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 13:45:26 +1100 Subject: [PATCH 602/979] linux: test cases: remove unused variables --- test/test_volatility.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 4e948a2bd..c0cc11cd3 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -442,10 +442,9 @@ def test_linux_capabilities(image, volatility, python): def test_linux_check_creds(image, volatility, python): - rc, out, _err = runvol_plugin( + rc, _out, _err = runvol_plugin( "linux.check_creds.Check_creds", image, volatility, python ) - out = out.lower() # linux-sample-1.bin has no processes sharing credentials. # This validates that plugin requirements are met and exceptions are not raised. @@ -488,8 +487,7 @@ def test_linux_kthreads(image, volatility, python): def test_linux_malfind(image, volatility, python): - rc, out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) - out = out.lower() + rc, _out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) # linux-sample-1.bin has no process memory ranges with potential injected code. # This validates that plugin requirements are met and exceptions are not raised. @@ -515,8 +513,7 @@ def test_linux_psaux(image, volatility, python): def test_linux_ptrace(image, volatility, python): - rc, out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) - out = out.lower() + rc, _out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) # linux-sample-1.bin has no processes being ptreaced. # This validates that plugin requirements are met and exceptions are not raised. From 9048cd1ab8683edac7d2ccfeb7fdcbee13d447df Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 13:47:03 +1100 Subject: [PATCH 603/979] linux: boottime: Minor, removed unnecessary blank line --- volatility3/framework/plugins/linux/boottime.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py index 3df5cb3ac..56de52883 100644 --- a/volatility3/framework/plugins/linux/boottime.py +++ b/volatility3/framework/plugins/linux/boottime.py @@ -15,7 +15,6 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """Shows the time the system was started""" _required_framework_version = (2, 11, 0) - _version = (1, 0, 1) @classmethod From aaeec80fdf4650be6a9b1f6c25ef5ba4aea03e29 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 13:53:11 +1100 Subject: [PATCH 604/979] linux: library_list testcase: Optimize testing performance by limiting the number of processes to just one. This change will reduce execution time and speed up the test --- test/test_volatility.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index c0cc11cd3..eb365ccf3 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -362,27 +362,19 @@ def test_linux_sockstat(image, volatility, python): def test_linux_library_list(image, volatility, python): rc, out, _err = runvol_plugin( - "linux.library_list.LibraryList", image, volatility, python + "linux.library_list.LibraryList", + image, + volatility, + python, + pluginargs=["--pids", "2363"], ) assert re.search( rb"NetworkManager\s2363\s0x7f52cdda0000\s/lib/x86_64-linux-gnu/libnss_files.so.2", out, ) - assert re.search( - rb"gnome-settings-\s3807\s0x7f7e660b5000\s/lib/x86_64-linux-gnu/libbz2.so.1.0", - out, - ) - assert re.search( - rb"gdu-notificatio\s3878\s0x7f25ce33e000\s/usr/lib/x86_64-linux-gnu/libXau.so.6", - out, - ) - assert re.search( - rb"bash\s8600\s0x7fe78a85f000\s/lib/x86_64-linux-gnu/libnss_files.so.2", - out, - ) - assert out.count(b"\n") >= 2677 + assert out.count(b"\n") > 10 assert rc == 0 From 8ecd7e2ddddb018164d3ef734899b91a93899d09 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 28 Nov 2024 15:17:22 +1100 Subject: [PATCH 605/979] Linux/Mac: Log producer information --- .../framework/automagic/symbol_finder.py | 31 ++++++++++++++++--- volatility3/framework/symbols/intermed.py | 15 ++++++--- volatility3/framework/symbols/metadata.py | 13 +++++++- 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 21e594549..55e2ad6f5 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -142,11 +142,11 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): ) for _, banner in banner_list: - vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = self.banners.get(banner, None) - if symbol_files: - isf_path = symbol_files - vollog.debug(f"Using symbol library: {symbol_files}") + vollog.debug(f"Identified banner: {banner!r}") + symbols_file = self.banners.get(banner, None) + if symbols_file: + isf_path = symbols_file + vollog.debug(f"Using symbol library: {symbols_file}") clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join @@ -160,8 +160,29 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): path_join(config_path, requirement.name, "symbol_mask") ] = layer.address_mask + # Keep track of the existing table names so we know which ones were added + old_table_names = set(context.symbol_space._dict) + # Construct the appropriate symbol table requirement.construct(context, config_path) + + new_table_names = context.symbol_space._dict.keys() - old_table_names + # It should add only one symbol table. Ignore the next steps if it doesn't + if len(new_table_names) == 1: + new_table_name = new_table_names.pop() + symbol_table = context.symbol_space._dict[new_table_name] + producer = symbol_table.producer + vollog.debug( + f"producer_name: {producer.name}, producer_version: {producer.version_string}" + ) + for category in symbol_table.metadata._json_data: + vollog.debug(f"{category}:") + for subkey in symbol_table.metadata._json_data[category]: + subkey_item = ", ".join( + f"{key}: '{value}'" for key, value in subkey.items() + ) + vollog.debug(f"\t{subkey_item}") + break else: vollog.debug(f"Symbol library path not found for: {banner}") diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 751f88e39..5f558bf12 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -738,10 +738,17 @@ class Version6Format(Version5Format): @property def metadata(self) -> Optional[interfaces.symbols.MetadataInterface]: """Returns a MetadataInterface object.""" - if self._json_object.get("metadata", {}).get("windows"): - return metadata.WindowsMetadata(self._json_object["metadata"]["windows"]) - if self._json_object.get("metadata", {}).get("linux"): - return metadata.LinuxMetadata(self._json_object["metadata"]["linux"]) + if "metadata" not in self._json_object: + return None + + json_metadata = self._json_object["metadata"] + if "windows" in json_metadata: + return metadata.WindowsMetadata(json_metadata["windows"]) + if "linux" in json_metadata: + return metadata.LinuxMetadata(json_metadata["linux"]) + if "mac" in json_metadata: + return metadata.MacMetadata(json_metadata["mac"]) + return None diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 95f542f07..39ddd6544 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -18,10 +18,17 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): def name(self) -> Optional[str]: return self._json_data.get("name", None) + @property + def version_string(self) -> str: + """Returns the ISF file producer's version as a string. + If no version is present, an empty string is returned. + """ + return self._json_data.get("version", "") + @property def version(self) -> Optional[Tuple[int]]: """Returns the version of the ISF file producer""" - version = self._json_data.get("version", None) + version = self.version_string() if not version: return None if all(x in "0123456789." for x in version): @@ -81,3 +88,7 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): class LinuxMetadata(interfaces.symbols.MetadataInterface): """Class to handle the metadata from a Linux symbol table.""" + + +class MacMetadata(interfaces.symbols.MetadataInterface): + """Class to handle the metadata from a Mac symbol table.""" From 3748cb89b8e2493fd8792315a9b3419d9a26dfb3 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 28 Nov 2024 10:22:45 +0000 Subject: [PATCH 606/979] Windows: Improve debugging output for pdbscan --- volatility3/framework/automagic/pdbscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 1d5bf55ea..3751b383e 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -272,7 +272,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): for kernel in kernels: vollog.log( constants.LOGLEVEL_VVVV, - f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1)} with MZ offset at {kernel.get('mz_offset', -1)}", + f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1):x} with MZ offset at {kernel.get('mz_offset', -1):x}", ) valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel) if valid_kernel is not None: From bec6bc659475b7fc016f1b05093e7783ca70d904 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 28 Nov 2024 16:59:52 +0000 Subject: [PATCH 607/979] Fix up vmayarascan and vadyarascan to use yarascan properly --- .../framework/plugins/linux/vmayarascan.py | 58 +++++++++------ .../framework/plugins/windows/vadyarascan.py | 71 ++++++++----------- 2 files changed, 64 insertions(+), 65 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 9fe06b0c8..3d8eb603b 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import logging from typing import Iterable, List, Tuple from volatility3.framework import interfaces, renderers @@ -10,6 +11,8 @@ from volatility3.framework.renderers import format_hints from volatility3.plugins import yarascan from volatility3.plugins.linux import pslist +vollog = logging.getLogger(__name__) + class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" @@ -50,6 +53,8 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): # use yarascan to parse the yara options provided and create the rules rules = yarascan.YaraScan.process_yara_options(dict(self.config)) + sanity_check = 1024 * 1024 * 1024 # 1 GB + # filter based on the pid option if provided filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) for task in pslist.PsList.list_tasks( @@ -66,29 +71,36 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): # get the proc_layer object from the context proc_layer = self.context.layers[proc_layer_name] - for start, end in self.get_vma_maps(task): - for match in rules.match( - data=proc_layer.read(start, end - start, True) - ): - if yarascan.YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: - yield 0, ( - format_hints.Hex(instance.offset + start), - task.UniqueProcessId, - match.rule, - match_string.identifier, - instance.matched_data, - ) - else: - for offset, name, value in match.strings: - yield 0, ( - format_hints.Hex(offset + start), - task.tgid, - match.rule, - name, - value, - ) + vma_maps = list(self.get_vma_maps(task)) + insane_vma_maps = [ + start for (start, size) in vma_maps if size > sanity_check + ] + for start in insane_vma_maps: + vollog.debug(f"VMA at 0x{start:x} over sanity-check size, not scanning") + + if not vma_maps: + vollog.warning(f"No VMAs were found for task {task.pid}, aborting") + continue + + max_vma_size: int = max( + [size for (start, size) in vma_maps if size <= sanity_check] + ) + scanner = yarascan.YaraScanner(rules=rules) + scanner.chunk_size = max_vma_size + + # scan the process layer with the yarascanner + for offset, rule_name, name, value in proc_layer.scan( + context=self.context, + scanner=scanner, + sections=vma_maps, + ): + yield 0, ( + format_hints.Hex(offset), + task.tgid, + rule_name, + name, + value, + ) @staticmethod def get_vma_maps( diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index efcc70d07..65006bdc2 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -32,6 +32,9 @@ class VadYaraScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), @@ -66,49 +69,33 @@ class VadYaraScan(interfaces.plugins.PluginInterface): ): layer_name = task.add_process_layer() layer = self.context.layers[layer_name] - for start, size in self.get_vad_maps(task): - if size > sanity_check: - vollog.debug( - f"VAD at 0x{start:x} over sanity-check size, not scanning" - ) - continue - data = layer.read(start, size, True) - if not yarascan.YaraScan._yara_x: - for match in rules.match(data=data): - if yarascan.YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: - yield 0, ( - format_hints.Hex(instance.offset + start), - task.UniqueProcessId, - match.rule, - match_string.identifier, - instance.matched_data, - ) - else: - for offset, name, value in match.strings: - yield 0, ( - format_hints.Hex(offset + start), - task.UniqueProcessId, - match.rule, - name, - value, - ) - else: - for match in rules.scan(data).matching_rules: - for match_string in match.patterns: - for instance in match_string.matches: - yield 0, ( - format_hints.Hex(instance.offset + start), - task.UniqueProcessId, - f"{match.namespace}.{match.identifier}", - match_string.identifier, - data[ - instance.offset : instance.offset - + instance.length - ], - ) + vad_maps = list(self.get_vad_maps(task)) + insane_vad_maps = [ + start for (start, size) in vad_maps if size > sanity_check + ] + for start in insane_vad_maps: + vollog.debug(f"VAD at 0x{start:x} over sanity-check size, not scanning") + + max_vad_size: int = max( + [size for (start, size) in vad_maps if size <= sanity_check] + ) + scanner = yarascan.YaraScanner(rules=rules) + scanner.chunk_size = max_vad_size + + # scan the process layer with the yarascanner + for offset, rule_name, name, value in layer.scan( + context=self.context, + scanner=scanner, + sections=vad_maps, + ): + yield 0, ( + format_hints.Hex(offset), + task.UniqueProcessId, + rule_name, + name, + value, + ) @staticmethod def get_vad_maps( From b8023f0c97ae97253ab9b8eae99e1dc4cba1eb79 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 29 Nov 2024 19:05:27 +1100 Subject: [PATCH 608/979] Linux/Mac: Address code review suggestions - Add getters for Linux/Mac ISF sources - Avoid using internal attributes - Use the dict repr instead of walking the dict to simplify code --- .../framework/automagic/symbol_finder.py | 26 ++++++++++--------- volatility3/framework/symbols/metadata.py | 19 +++++++++++--- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 55e2ad6f5..6d689e194 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -161,27 +161,29 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): ] = layer.address_mask # Keep track of the existing table names so we know which ones were added - old_table_names = set(context.symbol_space._dict) + old_table_names = set(context.symbol_space) # Construct the appropriate symbol table requirement.construct(context, config_path) - new_table_names = context.symbol_space._dict.keys() - old_table_names + new_table_names = set(context.symbol_space) - old_table_names # It should add only one symbol table. Ignore the next steps if it doesn't if len(new_table_names) == 1: new_table_name = new_table_names.pop() - symbol_table = context.symbol_space._dict[new_table_name] - producer = symbol_table.producer + symbol_table = context.symbol_space[new_table_name] + producer_metadata = symbol_table.producer vollog.debug( - f"producer_name: {producer.name}, producer_version: {producer.version_string}" + f"producer_name: {producer_metadata.name}, producer_version: {producer_metadata.version_string}" ) - for category in symbol_table.metadata._json_data: - vollog.debug(f"{category}:") - for subkey in symbol_table.metadata._json_data[category]: - subkey_item = ", ".join( - f"{key}: '{value}'" for key, value in subkey.items() - ) - vollog.debug(f"\t{subkey_item}") + + symbol_metadata = symbol_table.metadata + vollog.debug("Types:") + for types_source_dict in symbol_metadata.get_types_sources(): + vollog.debug(f"\t{types_source_dict}") + + vollog.debug("Symbols:") + for symbol_source_dict in symbol_metadata.get_symbols_sources(): + vollog.debug(f"\t{symbol_source_dict}") break else: diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 39ddd6544..02e9cc489 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -4,8 +4,7 @@ import datetime import logging -from typing import Optional, Tuple, Union - +from typing import Optional, Tuple, Union, List, Dict from volatility3.framework import constants, interfaces vollog = logging.getLogger(__name__) @@ -86,9 +85,21 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): return self._json_data.get("pdb", {}).get("age", None) -class LinuxMetadata(interfaces.symbols.MetadataInterface): +class DwarfMetadata(interfaces.symbols.MetadataInterface): + """Base class to handle metadata of DWARF-based ISF sources""" + + def get_types_sources(self) -> List[Optional[Dict]]: + """Returns the types sources metadata""" + return self._json_data.get("types", []) + + def get_symbols_sources(self) -> List[Optional[Dict]]: + """Returns the symbols sources metadata""" + return self._json_data.get("symbols", []) + + +class LinuxMetadata(DwarfMetadata): """Class to handle the metadata from a Linux symbol table.""" -class MacMetadata(interfaces.symbols.MetadataInterface): +class MacMetadata(DwarfMetadata): """Class to handle the metadata from a Mac symbol table.""" From 77778ee6f6cfaa9a9af1d73a225c67a8836727c7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 29 Nov 2024 19:35:18 +1100 Subject: [PATCH 609/979] Linux/Mac: ISF metadata: Rename s/DWARF/POSIX/, as I'm not happy with the generic name. BTF source could potentially generate the same keys --- volatility3/framework/symbols/metadata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 02e9cc489..73ad2cf21 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -85,8 +85,8 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface): return self._json_data.get("pdb", {}).get("age", None) -class DwarfMetadata(interfaces.symbols.MetadataInterface): - """Base class to handle metadata of DWARF-based ISF sources""" +class PosixMetadata(interfaces.symbols.MetadataInterface): + """Base class to handle metadata of Posix-based ISF sources""" def get_types_sources(self) -> List[Optional[Dict]]: """Returns the types sources metadata""" @@ -97,9 +97,9 @@ class DwarfMetadata(interfaces.symbols.MetadataInterface): return self._json_data.get("symbols", []) -class LinuxMetadata(DwarfMetadata): +class LinuxMetadata(PosixMetadata): """Class to handle the metadata from a Linux symbol table.""" -class MacMetadata(DwarfMetadata): +class MacMetadata(PosixMetadata): """Class to handle the metadata from a Mac symbol table.""" From 0030129ff8b440f8f3e43870f87d697b3847baa6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 29 Nov 2024 08:43:04 +0000 Subject: [PATCH 610/979] Make suggested fixes to reduce loops and ignore insane sections --- .../framework/plugins/linux/vmayarascan.py | 25 +++++++++-------- .../framework/plugins/windows/vadyarascan.py | 28 ++++++++++++------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 3d8eb603b..89210be69 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -71,20 +71,21 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): # get the proc_layer object from the context proc_layer = self.context.layers[proc_layer_name] - vma_maps = list(self.get_vma_maps(task)) - insane_vma_maps = [ - start for (start, size) in vma_maps if size > sanity_check - ] - for start in insane_vma_maps: - vollog.debug(f"VMA at 0x{start:x} over sanity-check size, not scanning") + max_vma_size = 0 + vma_maps_to_scan = [] + for start, size in self.get_vma_maps(task): + if size > sanity_check: + vollog.debug( + f"VMA at 0x{start:x} over sanity-check size, not scanning" + ) + continue + max_vma_size = max(max_vma_size, size) + vma_maps_to_scan.append((start, size)) - if not vma_maps: - vollog.warning(f"No VMAs were found for task {task.pid}, aborting") + if not vma_maps_to_scan: + vollog.warning(f"No VMAs were found for task {task.tgid}, not scanning") continue - max_vma_size: int = max( - [size for (start, size) in vma_maps if size <= sanity_check] - ) scanner = yarascan.YaraScanner(rules=rules) scanner.chunk_size = max_vma_size @@ -92,7 +93,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): for offset, rule_name, name, value in proc_layer.scan( context=self.context, scanner=scanner, - sections=vma_maps, + sections=vma_maps_to_scan, ): yield 0, ( format_hints.Hex(offset), diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 65006bdc2..a67b8dc0b 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -70,16 +70,24 @@ class VadYaraScan(interfaces.plugins.PluginInterface): layer_name = task.add_process_layer() layer = self.context.layers[layer_name] - vad_maps = list(self.get_vad_maps(task)) - insane_vad_maps = [ - start for (start, size) in vad_maps if size > sanity_check - ] - for start in insane_vad_maps: - vollog.debug(f"VAD at 0x{start:x} over sanity-check size, not scanning") + max_vad_size = 0 + vad_maps_to_scan = [] + + for start, size in self.get_vad_maps(task): + if size > sanity_check: + vollog.debug( + f"VAD at 0x{start:x} over sanity-check size, not scanning" + ) + continue + max_vad_size = max(max_vad_size, size) + vad_maps_to_scan.append((start, size)) + + if not vad_maps_to_scan: + vollog.warning( + f"No VADs were found for task {task.UniqueProcessID}, not scanning" + ) + continue - max_vad_size: int = max( - [size for (start, size) in vad_maps if size <= sanity_check] - ) scanner = yarascan.YaraScanner(rules=rules) scanner.chunk_size = max_vad_size @@ -87,7 +95,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): for offset, rule_name, name, value in layer.scan( context=self.context, scanner=scanner, - sections=vad_maps, + sections=vad_maps_to_scan, ): yield 0, ( format_hints.Hex(offset), From 393db1050f7df14d39c6f71bc0782e5422ed3188 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 29 Nov 2024 08:51:06 +0000 Subject: [PATCH 611/979] Windows: protect again mz_offsets being None --- volatility3/framework/automagic/pdbscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 3751b383e..0b4f6c73a 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -272,7 +272,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): for kernel in kernels: vollog.log( constants.LOGLEVEL_VVVV, - f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1):x} with MZ offset at {kernel.get('mz_offset', -1):x}", + f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1):x} with MZ offset at {(kernel.get('mz_offset', -1) or -1):x}", ) valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel) if valid_kernel is not None: From 3df385369cbd2489f8e057b616e98a83b79cf397 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 30 Nov 2024 11:28:09 +0000 Subject: [PATCH 612/979] Ensure the VAD/VMA gets scanned in a single block --- .../framework/plugins/linux/vmayarascan.py | 28 ++++++++++--------- .../framework/plugins/windows/vadyarascan.py | 25 ++++++++--------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 89210be69..8dd64404d 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -36,6 +36,9 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), requirements.ModuleRequirement( name="kernel", description="Linux kernel", @@ -89,19 +92,18 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): scanner = yarascan.YaraScanner(rules=rules) scanner.chunk_size = max_vma_size - # scan the process layer with the yarascanner - for offset, rule_name, name, value in proc_layer.scan( - context=self.context, - scanner=scanner, - sections=vma_maps_to_scan, - ): - yield 0, ( - format_hints.Hex(offset), - task.tgid, - rule_name, - name, - value, - ) + # scan the VMA data (in one contiguous block) with the yarascanner + for start, size in vma_maps_to_scan: + for offset, rule_name, name, value in scanner( + proc_layer.read(start, size, pad=True), start + ): + yield 0, ( + format_hints.Hex(offset), + task.tgid, + rule_name, + name, + value, + ) @staticmethod def get_vma_maps( diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index a67b8dc0b..2e9cc44ea 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -91,19 +91,18 @@ class VadYaraScan(interfaces.plugins.PluginInterface): scanner = yarascan.YaraScanner(rules=rules) scanner.chunk_size = max_vad_size - # scan the process layer with the yarascanner - for offset, rule_name, name, value in layer.scan( - context=self.context, - scanner=scanner, - sections=vad_maps_to_scan, - ): - yield 0, ( - format_hints.Hex(offset), - task.UniqueProcessId, - rule_name, - name, - value, - ) + # scan the VAD data (in one contiguous block) with the yarascanner + for start, size in vad_maps_to_scan: + for offset, rule_name, name, value in scanner( + layer.read(start, size, pad=True), start + ): + yield 0, ( + format_hints.Hex(offset), + task.UniqueProcessId, + rule_name, + name, + value, + ) @staticmethod def get_vad_maps( From 56f6ef0add6d73ecebeb41837a69627f960e8b0d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 30 Nov 2024 11:52:16 +0000 Subject: [PATCH 613/979] Include a test developed by @gcmoreira and @eve-mem --- test/test_volatility.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index 847be88d9..ea9ad8211 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -14,6 +14,7 @@ import tempfile import hashlib import ntpath import json +import contextlib # # HELPER FUNCTIONS @@ -378,6 +379,42 @@ def test_linux_library_list(image, volatility, python): assert out.count(b"\n") >= 2677 assert rc == 0 +def test_linux_vmayarascan_yara_rule(image, volatility, python): + yara_rule_01 = r""" + rule fullvmayarascan + { + strings: + $s1 = "_nss_files_parse_grent" + $s2 = "/lib64/ld-linux-x86-64.so.2" + $s3 = "(bufferend - (char *) 0) % sizeof (char *) == 0" + condition: + all of them + } + """ + + # FIXME: When the minimum Python version includes 3.12, replace the following with: + # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... + fd, filename = tempfile.mkstemp(suffix=".yar") + try: + with os.fdopen(fd, "w") as f: + f.write(yara_rule_01) + + rc, out, _err = runvol_plugin( + "linux.vmayarascan.VmaYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "8600", "--yara-file", filename], + ) + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + out = out.lower() + assert out.count(b"\n") > 4 + assert rc == 0 + + # MAC From 2dc168628936ffcc334c1de37c024dc3d0fd7ff6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 30 Nov 2024 13:36:14 +0000 Subject: [PATCH 614/979] Windows: Protect the SERICE_RECORD is_valid function a little more The request to .Order could fail depending on where the structure lies in memory. --- .../symbols/windows/extensions/services.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/services.py b/volatility3/framework/symbols/windows/extensions/services.py index e14de761d..0a2194e07 100644 --- a/volatility3/framework/symbols/windows/extensions/services.py +++ b/volatility3/framework/symbols/windows/extensions/services.py @@ -14,13 +14,16 @@ class SERVICE_RECORD(objects.StructType): def is_valid(self) -> bool: """Determine if the structure is valid.""" - if self.Order < 0 or self.Order > 0xFFFF: - return False - try: - _ = self.State.description - _ = self.Start.description - except ValueError: + if self.Order < 0 or self.Order > 0xFFFF: + return False + + try: + _ = self.State.description + _ = self.Start.description + except ValueError: + return False + except exceptions.InvalidAddressException: return False return True From d404747de5ce622ac1907199ed48b4f9905f2bbc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 1 Dec 2024 00:04:46 +0000 Subject: [PATCH 615/979] Tests: Add in vadyarascan tests --- test/test_volatility.py | 58 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index ea9ad8211..371dd281c 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -197,7 +197,7 @@ def test_windows_thrdscan(image, volatility, python): assert out.find(b"\t4\t8") != -1 assert out.find(b"\t4\t12") != -1 assert out.find(b"\t4\t16") != -1 - #assert out.find(b"this raieses AssertionError") != -1 + # assert out.find(b"this raieses AssertionError") != -1 assert rc == 0 @@ -274,6 +274,59 @@ def test_windows_devicetree(image, volatility, python): assert rc == 0 +def test_windows_vadyarascan_yara_rule(image, volatility, python): + yara_rule_01 = r""" + rule fullvadyarascan + { + strings: + $s1 = "!This program cannot be run in DOS mode." + $s2 = "Qw))Pw" + $s3 = "W_wD)Pw" + $s4 = "1Xw+2Xw" + $s5 = "xd`wh``w" + $s6 = "0g`w0g`w8g`w8g`w@g`w@g`wHg`wHg`wPg`wPg`wXg`wXg`w`g`w`g`whg`whg`wpg`wpg`wxg`wxg`w" + condition: + all of them + } + """ + + # FIXME: When the minimum Python version includes 3.12, replace the following with: + # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... + fd, filename = tempfile.mkstemp(suffix=".yar") + try: + with os.fdopen(fd, "w") as f: + f.write(yara_rule_01) + + rc, out, _err = runvol_plugin( + "windows.vadyarascan.VadYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "4012", "--yara-file", filename], + ) + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + out = out.lower() + assert out.count(b"\n") > 4 + assert rc == 0 + + +def test_windows_vadyarascan(image, volatility, python): + rc, out, _err = runvol_plugin( + "windows.vadyarascan.VadYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "4012", "--yara-string", "MZ"], + ) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + # LINUX @@ -342,6 +395,7 @@ def test_linux_tty_check(image, volatility, python): assert out.count(b"\n") >= 5 assert rc == 0 + def test_linux_sockstat(image, volatility, python): rc, out, err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) @@ -379,6 +433,7 @@ def test_linux_library_list(image, volatility, python): assert out.count(b"\n") >= 2677 assert rc == 0 + def test_linux_vmayarascan_yara_rule(image, volatility, python): yara_rule_01 = r""" rule fullvmayarascan @@ -415,7 +470,6 @@ def test_linux_vmayarascan_yara_rule(image, volatility, python): assert rc == 0 - # MAC From 3c20e46902a3cc11ed00fcdf122f375ed1ca4b6c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 1 Dec 2024 12:36:35 +1100 Subject: [PATCH 616/979] renderers: Fix HexBytes formatter to apply padding also at the end of the string, ensuring proper output alignment when the pretty renderer justifies each line to the right --- volatility3/cli/text_renderer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 96970d3ce..31307f67e 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -51,8 +51,10 @@ def hex_bytes_as_text(value: bytes, width: int = 16) -> str: # Handle leftovers when the lenght is not mutiple of width if printables: - output += " " * (width - len(printables)) + padding = width - len(printables) + output += " " * (padding) output += printables + output += " " * (padding) return output From c19a54cbd982fc29c7d937d90c830c1dda3bfd03 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 1 Dec 2024 14:08:53 +1100 Subject: [PATCH 617/979] testcases: Minor cleanup: Renaming and reordering functions to align with the Linux/Windows test cases --- test/test_volatility.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 97098d1ca..f7cb23e93 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -315,7 +315,7 @@ def test_windows_vadyarascan_yara_rule(image, volatility, python): assert rc == 0 -def test_windows_vadyarascan(image, volatility, python): +def test_windows_vadyarascan_yara_string(image, volatility, python): rc, out, _err = runvol_plugin( "windows.vadyarascan.VadYaraScan", image, @@ -476,6 +476,7 @@ def test_linux_capabilities(image, volatility, python): python, globalargs=["-vvv"], ) + if rc != 0 and err.count(b"Unsupported kernel capabilities implementation") > 0: # The linux-sample-1.bin kernel implementation isn't supported. # However, we can still check that the plugin requirements are met. @@ -521,13 +522,14 @@ def test_linux_kthreads(image, volatility, python): python, globalargs=["-vvv"], ) - out = out.lower() if rc != 0 and err.count(b"Unsupported kthread implementation") > 0: # The linux-sample-1.bin kernel implementation isn't supported. # However, we can still check that the plugin requirements are met. return None + out = out.lower() + assert out.count(b"\n") > 10 assert rc == 0 @@ -580,20 +582,6 @@ def test_linux_vmaregexscan(image, volatility, python): assert rc == 0 -def test_linux_vmayarascan(image, volatility, python): - rc, out, _err = runvol_plugin( - "linux.vmayarascan.VmaYaraScan", - image, - volatility, - python, - pluginargs=["--pid", "1", "--yara-string", "ELF"], - ) - out = out.lower() - - assert out.count(b"\n") > 10 - assert rc == 0 - - def test_linux_vmayarascan_yara_rule(image, volatility, python): yara_rule_01 = r""" rule fullvmayarascan @@ -630,6 +618,20 @@ def test_linux_vmayarascan_yara_rule(image, volatility, python): assert rc == 0 +def test_linux_vmayarascan_yara_string(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.vmayarascan.VmaYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "1", "--yara-string", "ELF"], + ) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + # MAC From 535ce3a22a575c14100ca30627f3d2889fd86c9c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Sun, 1 Dec 2024 19:34:35 +1100 Subject: [PATCH 618/979] testing: Enable automatic selection of the OS image based on the test and filename prefix, addressing an issue in the development environment. For example, when using VSCode with pytest, test autodiscovery triggers pytest_generate_tests(), adding all images to each test case. This causes issues, as Linux tests end up being executed with Windows and Mac images, and vice versa. --- .github/workflows/test.yaml | 10 ++++++---- test/conftest.py | 33 ++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 084cce295..dfc42499d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,10 +25,13 @@ jobs: - name: Download images run: | + mkdir test_images + cd test_images curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/linux-sample-1.bin.gz" gunzip linux-sample-1.bin.gz curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" gunzip win-xp-laptop-2005-06-25.img.gz + cd - - name: Download and Extract symbols run: | @@ -39,13 +42,12 @@ jobs: - name: Testing... run: | - pytest ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v - pytest ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_windows -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_linux -v - name: Clean up post-test run: | - rm -rf *.bin - rm -rf *.img + rm -rf test_images cd volatility3/symbols rm -rf linux rm -rf linux.zip diff --git a/test/conftest.py b/test/conftest.py index 4ad63065b..0115fade9 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -35,16 +35,39 @@ def pytest_addoption(parser): def pytest_generate_tests(metafunc): """Parameterize tests based on image names""" - images = metafunc.config.getoption("image") + images = metafunc.config.getoption("image").copy() for image_dir in metafunc.config.getoption("image_dir"): - images = images + [ - os.path.join(image_dir, dir) for dir in os.listdir(image_dir) + images += [ + os.path.join(image_dir, dir_name) for dir_name in os.listdir(image_dir) ] - # tests with "image" parameter are run against images + # tests with "image" parameter are run against image if "image" in metafunc.fixturenames: + filtered_images = [] + ids = [] + for image in images: + image_base = os.path.basename(image) + test_name = metafunc.definition.originalname + if test_name.startswith("test_windows_") and not image_base.startswith( + "win-" + ): + continue + elif test_name.startswith("test_linux_") and not image_base.startswith( + "linux-" + ): + continue + elif test_name.startswith("test_mac_") and not image_base.startswith( + "mac-" + ): + continue + + filtered_images.append(image) + ids.append(image_base) + metafunc.parametrize( - "image", images, ids=[os.path.basename(image) for image in images] + "image", + filtered_images, + ids=ids, ) From 20f15d3591a5e6340ecd2d3628406c01aef71924 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 2 Dec 2024 11:34:49 +0100 Subject: [PATCH 619/979] modular physical_layer access --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e5a074a49..2d77b8562 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2511,6 +2511,10 @@ class scatterlist(objects.StructType): Returns: An iterator of bytes """ - physical_layer = self._context.layers["memory_layer"] + # Either "physical" is layer-1 because this is a module layer, either "physical" is the current layer + physical_layer_name = self._context.layers[self.vol.layer_name].config.get( + "memory_layer", self.vol.layer_name + ) + physical_layer = self._context.layers[physical_layer_name] for sg in self.for_each_sg(): yield from physical_layer.read(sg.dma_address, sg._sg_dma_len()) From 3ff304ceed2501df6d9291259f9bc6b3c587c303 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 3 Dec 2024 19:58:19 +0000 Subject: [PATCH 620/979] Layers: Fix intel bug introduced in commit 73d4f2f The patch failed to mask the incoming address to the maximum physical address. This allowed non-canonical addresses (potentially within the page table) to be looked up incorrectly. Fixes #1374. Thanks to @the-rectifier for quickly identifying the issue! --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index d762b41a8..c30ae48a8 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -182,7 +182,7 @@ class Intel(linear.LinearlyMappedLayer): def _pte_pfn(self, entry: int) -> int: """Extracts the page frame number (PFN) from the page table entry (PTE) entry""" - return entry >> self.page_shift + return self._mask(entry, self._maxphyaddr - 1, 0) >> self.page_shift def _translate_entry(self, offset: int) -> Tuple[int, int]: """Translates a specific offset based on paging tables. From 5acf8858d95e413d32b7342fe5f388a68597201e Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 5 Dec 2024 18:31:04 +0000 Subject: [PATCH 621/979] =?UTF-8?q?PEP=20488=20=E2=80=93=20Elimination=20o?= =?UTF-8?q?f=20PYO=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python 3.5 implememted PEP 488, eliminating .pyo files. --- volatility3/framework/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 51310bfa2..4c09b4dae 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -164,7 +164,6 @@ def _filter_files(filename: str): return ( filename.endswith(".py") or filename.endswith(".pyc") - or filename.endswith(".pyo") ) and not filename.startswith("__") From f11ef06c27bbaaeb25cb83c3eadfba192531b52b Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 5 Dec 2024 21:30:20 +0000 Subject: [PATCH 622/979] =?UTF-8?q?PEP=20488=20=E2=80=93=20Elimination=20o?= =?UTF-8?q?f=20PYO=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python 3.5 implememted PEP 488, eliminating .pyo files. --- volatility3/framework/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 4c09b4dae..23ea745de 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -162,8 +162,7 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]: def _filter_files(filename: str): """Ensures that a filename traversed is an importable python file""" return ( - filename.endswith(".py") - or filename.endswith(".pyc") + filename.endswith(".py") or filename.endswith(".pyc") ) and not filename.startswith("__") From 846403115a8f14a5b6c47ee6fb53bb20f4d6dc78 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:00:03 +0000 Subject: [PATCH 623/979] Small documention changes --- doc/source/basics.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/basics.rst b/doc/source/basics.rst index 1b8e64780..91a45fbbf 100644 --- a/doc/source/basics.rst +++ b/doc/source/basics.rst @@ -14,7 +14,7 @@ Memory layers ------------- A memory layer is a body of data that can be accessed by requesting data at a specific address. At its lowest level -this data is stored on a phyiscal medium (RAM) and very early computers addresses locations in memory directly. However, +this data is stored on a phyiscal medium (RAM) and very early computers addressed locations in memory directly. However, as the size of memory increased and it became more difficult to manage memory most architectures moved to a "paged" model of memory, where the available memory is cut into specific fixed-sized pages. To help further, programs can ask for any address and the processor will look up their (virtual) address in a map, to find out where the (physical) address that it lives at is, @@ -25,8 +25,8 @@ address `9`). The automagic that runs at the start of every volatility session a kernel virtual layer, which allows for kernel addresses to be looked up and the correct data returned. There can, however, be several maps, and in general there is a different map for each process (although a portion of the operating system's memory is usually mapped to the same location across all processes). The maps may take the same address but point to a different part of -physical memory. It also means that two processes could theoretically share memory, but having an virtual address mapped to the -same physical address as another process. See the worked example below for more information. +physical memory. It also means that two processes could theoretically share memory, both having a virtual address mapped to the +same physical address. See the worked example below for more information. To translate an address on a layer, call :py:meth:`layer.mapping(offset, length, ignore_errors) ` and it will return a list of chunks without overlap, in order, for the requested range. If a portion cannot be mapped, an exception will be thrown unless `ignore_errors` is true. Each @@ -61,7 +61,7 @@ mean they each see something different: 4 -> 2 16 - Free In this example, part of the operating system is visible across all processes (although not all processes can write to the memory, there -is a permissions model for intel addressing which is not discussed further here).) +is a permissions model for Intel addressing which is not discussed further here). In Volatility 3 mappings are represented by a directed graph of layers, whose end nodes are :py:class:`DataLayers ` and whose internal nodes are :py:class:`TranslationLayers `. @@ -69,13 +69,13 @@ In this way, a raw memory image in the LiME file format and a page file can be c memory layer. When requesting addresses from the Intel layer, it will use the Intel memory mapping algorithm, along with the address of the directory table base or page table map, to translate that address into a physical address, which will then either be directed towards the swap layer or the LiME layer. Should it -be directed towards the LiME layer, the LiME file format algorithm will be translate the new address to determine where +be directed towards the LiME layer, the LiME file format algorithm will translate the new address to determine where within the file the data is stored. When the :py:meth:`layer.read() ` method is called, the translation is done automatically and the correct data gathered and combined. .. note:: Volatility 2 had a similar concept, called address spaces, but these could only stack linearly one on top of another. -The list of layers supported by volatility can be determined by running the `frameworkinfo` plugin. +The list of layers supported by Volatility can be determined by running the `frameworkinfo` plugin. Templates and Objects --------------------- From 93a47e811b94c4683db119f81ab4496c9e008736 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 6 Dec 2024 21:52:51 +0000 Subject: [PATCH 624/979] Small documentation changes --- doc/source/basics.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/basics.rst b/doc/source/basics.rst index 91a45fbbf..278ef4d73 100644 --- a/doc/source/basics.rst +++ b/doc/source/basics.rst @@ -167,8 +167,7 @@ There are certain setup tasks that establish the context in a way favorable to a several tasks that are repetitive and also easy to get wrong. These are called :py:class:`Automagic `, since they do things like magically taking a raw memory image and automatically providing the plugin with an appropriate Intel translation layer and an -accurate symbol table without either the plugin or the calling program having to specify all the necessary details. +accurate symbol table without either the plugin or the calling program having to specify all the necessary details. Automagics are a core component which consumers of the library can call or not at their discretion. .. note:: Volatility 2 used to do this as well, but it wasn't a particularly modular mechanism, and was used only for stacking address spaces (rather than identifying profiles), and it couldn't really be disabled/configured easily. - Automagics in Volatility 3 are a core component which consumers of the library can call or not at their discretion. From d29c23e922d22b5ccce0f4bc2ac439acd695a31c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 7 Dec 2024 15:51:35 +0000 Subject: [PATCH 625/979] Windows: Protect against missing _MM_SESSION_SPACE symbol --- volatility3/framework/symbols/windows/extensions/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ecfc2f163..793e506c3 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -825,6 +825,11 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): constants.LOGLEVEL_VVV, f"Cannot access _EPROCESS.Session.SessionId at {self.vol.offset:#x}", ) + except exceptions.SymbolError: + vollog.log( + constants.LOGLEVEL_VVV, + "Could not lookup _MM_SESSION_SPACE in symbol table", + ) return renderers.UnreadableValue() From 9ebc0bd90d544faeec98ab4d4dd46723aa0c3ed2 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:20:45 +0000 Subject: [PATCH 626/979] Cosmetic changes to documentation --- doc/source/symbol-tables.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 722f9e468..7f0b4153d 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -25,9 +25,9 @@ as long as the symbol files stay in the same location. Windows symbol tables --------------------- -For Windows systems, Volatility accepts a string made up of the GUID and Age of the required PDB file. It then +For Windows systems, Volatility accepts a string made up of the GUID and age of the required PDB file. It then searches all files under the configured symbol directories under the windows subdirectory. Any that contain metadata -which matches the pdb name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then +which matches the PDB name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then the associated PDB file will be downloaded from Microsoft's Symbol Server and converted into the appropriate JSON format, and will be saved in the correct location. @@ -35,7 +35,7 @@ Windows symbol tables can be manually constructed from an appropriate PDB file. is built into Volatility 3, called :file:`pdbconv.py`. It can be run from the top-level Volatility path, using the following command: -:command:`PYTHONPATH="." python volatility3/framework/symbols/windows/pdbconv.py` +:command:`PYTHONPATH="."; python volatility3/framework/symbols/windows/pdbconv.py` The :envvar:`PYTHONPATH` environment variable is not required if the Volatility library is installed in the system's library path or a virtual environment. @@ -54,8 +54,8 @@ most Volatility plugins. Note that in most linux distributions, the standard ke and the kernel with debugging information is stored in a package that must be acquired separately. A generic table isn't guaranteed to produce accurate results, and would reduce the number of structures -that all plugins could rely on. As such, and because linux kernels with different configurations can produce different structures, -volatility 3 requires that the banners in the JSON file match the banners found in the image *exactly*, not just the version +that all plugins could rely on. As such, and because Linux kernels with different configurations can produce different structures, +Volatility 3 requires that the banners in the JSON file match the banners found in the image *exactly*, not just the version number. This can include elements such as the compilation time and even the version of gcc used for the compilation. The exact match is required to ensure that the results volatility returns are accurate, therefore there is no simple means provided to get the wrong JSON ISF file to easily match. @@ -63,8 +63,8 @@ provided to get the wrong JSON ISF file to easily match. To determine the string for a particular memory image, use the `banners` plugin. Once the specific banner is known, try to locate that exact kernel debugging package for the operating system. Unfortunately each distribution provides its debugging packages under different package names and there are so many that the distribution may not keep all old -versions of the debugging symbols, and therefore **it may not be possible to find the right symbols to analyze a linux -memory image with volatility**. With Macs there are far fewer kernels and only one distribution, making it easier to +versions of the debugging symbols, and therefore **it may not be possible to find the right symbols to analyze a Linux +memory image with Volatility**. With Macs there are far fewer kernels and only one distribution, making it easier to ensure that the right symbols can be found. Once a kernel with debugging symbols/appropriate DWARF file has been located, `dwarf2json `_ will convert it into an @@ -75,7 +75,7 @@ symbol offsets within the DWARF data, which dwarf2json can extract into the JSON The banners available for volatility to use can be found using the `isfinfo` plugin, but this will potentially take a long time to run depending on the number of JSON files available. This will list all the JSON (ISF) files that -volatility3 is aware of, and for linux/mac systems what banner string they search for. For volatility to use the JSON +Volatility 3 is aware of, and for linux/mac systems what banner string they search for. For volatility to use the JSON file, the banners must match exactly (down to the compilation date). .. note:: From 97698cc5edc4a4d699e452353307c624ccb58a88 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:29:58 +0000 Subject: [PATCH 627/979] Cosmetic changes to documentation --- doc/source/vol2to3.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/source/vol2to3.rst b/doc/source/vol2to3.rst index e768df0c2..2520562a6 100644 --- a/doc/source/vol2to3.rst +++ b/doc/source/vol2to3.rst @@ -27,7 +27,7 @@ The object model has changed as well, objects now inherit directly from their Py object is actually a Python integer (and has all the associated methods, and can be used wherever a normal int could). In Volatility 2, a complex proxy object was constructed which tried to emulate all the methods of the host object, but ultimately it was a different type and could not be used in the same places (critically, it could make the ordering of -operations important, since a + b might not work, but b + a might work fine). +operations important, since x + y might not work, but y + x might work fine). Volatility 3 has also had significant speed improvements, where Volatility 2 was designed to allow access to live memory images and situations in which the underlying data could change during the run of the plugin, in Volatility 3 the data @@ -56,15 +56,14 @@ Volatility 2 were strictly limited to a stack, one on top of one other. In Vola Automagic --------- -In Volatility 2, we often tried to make this simpler for both users and developers. This resulted in something was -referred to as automagic, in that it was magic that happened automatically. We've now codified that more, so that the +In Volatility 2, we often tried to make this simpler for both users and developers. This resulted in something referred to as automagic, in that it was magic that happened automatically. We've now codified that more, so that the automagic processes are clearly defined and can be enabled or disabled as necessary for any particular run. We also included a stacker automagic to emulate the most common feature of Volatility 2, automatically stacking address spaces (now translation layers) on top of each other. -By default the automagic chosen to be run are determined based on the plugin requested, so that linux plugins get linux -specific automagic and windows plugins get windows specific automagic. This should reduce unnecessarily searching for -linux kernels in a windows image, for example. At the moment this is not user configurableS. +By default the automagic chosen to be run are determined based on the plugin requested, so that Linux plugins get Linux +specific automagic and Windows plugins get Windows specific automagic. This should reduce unnecessarily searching for +Linux kernels in a Windows image, for example. At the moment this is not user configurable. Searching and Scanning ---------------------- From 04d25544428c4a890c7cc88ebe4f80908a101464 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:36:18 +0000 Subject: [PATCH 628/979] Cosmetic changes to documentation --- doc/source/volshell.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 5a4b21ade..3c4f4ce5d 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -144,12 +144,12 @@ We can provide arguments via the `dpo` method call: 356 4 smss.exe 0x8c0bccf8d040 3 - N/A False 2021-03-13 17:25:33.000000 N/A Disabled ... -Here's we've provided the kernel name that was requested by the volshell plugin itself (the generic volshell does not +Here we've provided the kernel name that was requested by the volshell plugin itself (the generic volshell does not load a kernel module, and instead only has a TranslationLayerRequirement). A different module could be created and provided instead. The context used by the `dpo` method is always `context`. -Instead of print the results directly to screen, they can be gathered into a TreeGrid objects for direct access by +Instead of printing the results directly to screen, they can be gathered into a TreeGrid objects for direct access by using the `generate_treegrid` or `gt` command. :: From e165c78ba70de9961749f1d347fac2bcf6bca13d Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:37:39 +0000 Subject: [PATCH 629/979] Cosmetic changes to documentation --- doc/source/vol2to3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/vol2to3.rst b/doc/source/vol2to3.rst index 2520562a6..9b5a739f8 100644 --- a/doc/source/vol2to3.rst +++ b/doc/source/vol2to3.rst @@ -36,11 +36,11 @@ This was because live memory analysis was barely ever used, and this feature cou re-read many times over for no benefit (particularly since each re-read could result in many additional image reads from following page table translations). -Finally, in order to provide Volatility specific information without impact on the ability for structures to have members +Further, in order to provide Volatility specific information without impact on the ability for structures to have members with arbitrary names, all the metadata about the object (such as its layer or offset) have been moved to a read-only :py:meth:`~volatility3.framework.interfaces.objects.ObjectInterface.vol` dictionary. -Further the distinction between a :py:class:`~volatility3.framework.interfaces.objects.Template` (the thing that +Finally, the distinction between a :py:class:`~volatility3.framework.interfaces.objects.Template` (the thing that constructs an object) and the :py:class:`Object ` itself has been made more explicit. In Volatility 2, some information (such as size) could only be determined from a constructed object, leading to instantiating a template on an empty buffer, just to determine the size. In Volatility 3, templates contain From 45f9064623cedf0ac0ceb9b95a5f2729904ba12b Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 7 Dec 2024 17:11:49 +0000 Subject: [PATCH 630/979] Cosmetic changes to documentation --- doc/source/symbol-tables.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index 7f0b4153d..59c1febcc 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -35,7 +35,7 @@ Windows symbol tables can be manually constructed from an appropriate PDB file. is built into Volatility 3, called :file:`pdbconv.py`. It can be run from the top-level Volatility path, using the following command: -:command:`PYTHONPATH="."; python volatility3/framework/symbols/windows/pdbconv.py` +:command:`PYTHONPATH="." python volatility3/framework/symbols/windows/pdbconv.py` The :envvar:`PYTHONPATH` environment variable is not required if the Volatility library is installed in the system's library path or a virtual environment. From b86e8397188900740e3fce848b6ac3abddf1389c Mon Sep 17 00:00:00 2001 From: eve Date: Mon, 9 Dec 2024 18:30:19 +0000 Subject: [PATCH 631/979] Interfaces: change allow list for filenames to ensure they work safely on windows. Fixes issue #1387 --- volatility3/framework/interfaces/plugins.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 697e4cdc3..74902636e 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -59,14 +59,14 @@ class FileHandlerInterface(io.RawIOBase): @staticmethod def sanitize_filename(filename: str) -> str: - """Sanititizes the filename to ensure only a specific whitelist of characters is allowed through""" - allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]{}!$%^:#~?<>,|" + """Sanititizes the filename to ensure only a specific allow list of characters is allowed through""" + allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]{}!$%^#~," result = "" for char in filename: if char in allowed: result += char else: - result += "?" + result += "_" # change unwanted chars to an underscore return result def __enter__(self): From 582feccf938a75571d455552c3a3e51c026d3a66 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 9 Dec 2024 12:56:07 -0600 Subject: [PATCH 632/979] Address feedback --- .../framework/plugins/windows/mftscan.py | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 014516b7c..feea78ece 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -5,7 +5,7 @@ import contextlib import datetime import logging -from typing import Generator, Iterable, Dict, Tuple +from typing import Generator, Iterable, Dict, Tuple, Callable from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -42,7 +42,15 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): context: interfaces.context.ContextInterface, config_path: str, primary_layer_name: str, - attr_callback, + attr_callback: Callable[ + [ + Dict[int, Tuple[str, int, int]], + interfaces.objects.ObjectInterface, + interfaces.objects.ObjectInterface, + str, + ], + Generator, + ], ) -> interfaces.objects.ObjectInterface: try: primary = context.layers[primary_layer_name] @@ -121,7 +129,12 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) @staticmethod - def parse_mft_records(record_map, mft_record, attr, symbol_table): + def parse_mft_records( + record_map: Dict[int, Tuple[str, int, int]], + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table_name: str, + ): # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record try: @@ -131,7 +144,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Standard Information Attribute if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": - si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" + si_object = ( + symbol_table_name + constants.BANG + "STANDARD_INFORMATION_ENTRY" + ) attr_data = attr.Attr_Data.cast(si_object) yield 0, ( format_hints.Hex(attr_data.vol.offset), @@ -150,7 +165,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # File Name Attribute elif attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + fn_object = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() @@ -201,9 +216,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): else: # past the first $DATA record, attempt to get the ADS name # NotAvailableValue = > 1st Data, but name was not parsable - ads_name = attr.get_resident_filename() - if not ads_name: - ads_name = renderers.NotAvailableValue() + ads_name = attr.get_resident_filename() or renderers.NotAvailableValue() content = attr.get_resident_filecontent() if content: @@ -227,7 +240,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, - symbol_table, + symbol_table_name: str, return_first_record: bool, ) -> Generator[Iterable, None, None]: """ @@ -240,7 +253,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # file name, DATA count, offset record_map[mft_record.vol.offset] = [renderers.NotAvailableValue(), 0, None] if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + fn_object = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" attr_data = attr.Attr_Data.cast(fn_object) rec_name = attr_data.get_full_name() record_map[mft_record.vol.offset][0] = rec_name @@ -337,10 +350,10 @@ class ADS(interfaces.plugins.PluginInterface): record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, - symbol_table, + symbol_table_name: str, ): return MFTScan.parse_data_records( - record_map, mft_record, attr, symbol_table, False + record_map, mft_record, attr, symbol_table_name, False ) def _generator(self): @@ -406,10 +419,10 @@ class ResidentData(interfaces.plugins.PluginInterface): record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, - symbol_table, + symbol_table_name: str, ): return MFTScan.parse_data_records( - record_map, mft_record, attr, symbol_table, True + record_map, mft_record, attr, symbol_table_name, True ) def _generator(self): From e66a3e929b3c1253255f762db5677efd6c0510ec Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Thu, 12 Sep 2024 18:51:47 -0500 Subject: [PATCH 633/979] Add detection of direct and indirect system calls --- .../plugins/windows/direct_system_calls.py | 450 ++++++++++++++++++ .../plugins/windows/indirect_system_calls.py | 124 +++++ 2 files changed, 574 insertions(+) create mode 100644 volatility3/framework/plugins/windows/direct_system_calls.py create mode 100644 volatility3/framework/plugins/windows/indirect_system_calls.py diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py new file mode 100644 index 000000000..409f24750 --- /dev/null +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -0,0 +1,450 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import capstone +from collections import namedtuple +from typing import List, Tuple, Optional, Generator, Callable + +from volatility3.framework.objects import utility +from volatility3.framework import interfaces, renderers, symbols, exceptions +from volatility3.framework.configuration import requirements +from volatility3.plugins import yarascan +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + + +# Full details on the techniques used in these plugins to detect EDR-evading malware +# can be found in our 20 page whitepaper submitted to DEFCON along with the presentation +# https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + +syscall_finder_type = namedtuple( + "syscall_finder_type", + [ + "get_syscall_target_address", + "wants_syscall_inst", + "rule_str", + "invalid_ops", + "termination_ops", + ], +) + +syscall_finder_type.__doc__ = """ +This type to used to specify how malicious system call invocations should be found. + +`get_syscall_target_address` is optionally used to extract the address containing the malicious 'syscall' instruction +`wants_syscall_inst` whether or not this method expects the 'syscall' instrunction directly within the malicious code block +`rule` the opcode string to search for the malicious syscall instructions +`invalid_ops` instructions that only appear in invalid code blocks. Stops processing of the code block when encountered. +`termination_ops` instructions that are expected to be present in the code block and that stop processing +""" + + +class DirectSystemCalls(interfaces.plugins.PluginInterface): + """Detects the Direct System Call technique used to bypass EDRs""" + + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + valid_syscall_handlers = ("ntdll.dll", "win32u.dll") + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.syscall_finder = syscall_finder_type( + # for direct system calls, we find the `syscall` instruction directly, so we already know the address + None, + # yes, we want the syscall instruction present as it is what this technique looks for + True, + # regex to find "\x0f\x05" (syscall) followed later by "\xc3" (ret) + # we allow spacing in between to break naive anti-analysis forms (e.g., TarTarus Gate) + # Standard techniques, such as HellsGate, look like: + # mov r10, rcx + # mov eax, + # syscall + # ret + "/\\x0f\\x05[^\\xc3]{,24}\\xc3/", + # any of these will not be in a workable, malicious direct system call block + ["jmp", "call", "leave", "int3"], + # the expected form is to end with a "ret" back to the calling code + ["ret"], + ) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # create a list of requirements for vadyarascan + vadyarascan_requirements = [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) + ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) + ), + ] + + # get base yarascan requirements for command line options + yarascan_requirements = yarascan.YaraScan.get_yarascan_option_requirements() + + # return the combined requirements + return yarascan_requirements + vadyarascan_requirements + + @staticmethod + def _is_syscall_block( + disasm_func: Callable, + syscall_finder: syscall_finder_type, + data: bytes, + address: int, + ) -> Optional[Tuple[str, capstone._cs_insn]]: + """ + Determines if the bytes starting at `data` represent a valid syscall instrunction invocation block + + To maliciously invoke the system call instruction, malware must do each of the following: + + 1) update RAX to the system call number + 2) update R10 to the first parameter + 3) hit the 'termination' instrunction set in `syscall_finder_type` + + We also track whether the 'syscall' instruction was encountered while parsing + + This function is reusable for every technique we found and studied during the DEFCON research timeframe + + Args: + disasm_func: capstone disassembly function gathered from `get_disasm_function` + syscall_finder: the method and constraints on the malicious system call blocks that the calling plugin knows how to find + data: the bytes from memory to search for malicious syscall invocations + address: the address from where `data` came from in the particular process + Returns: + Optional[Tuple[str, capstone._cs_insn]]: For valid blocks, the disassembled bytes in string from and the last (termination) instruction + """ + found_movr10 = False + found_movreax = False + found_syscall = False + found_end = False + end_inst = None + + disasm_bytes = "" + + for inst in disasm_func(data, address): + disasm_bytes += f"{inst.address:#x}: {inst.mnemonic} {inst.op_str}; " + + # an instruction of all 0x00 opcodes + if inst.opcode.count(0) == len(inst.opcode): + break + + op = inst.mnemonic + + # invalid op, bail + if op in syscall_finder.invalid_ops: + break + + # found the end instruction wanted by the caller + elif op in syscall_finder.termination_ops: + found_end = True + end_inst = inst + break + + # track this no matter what to make code more re-usable + elif op == "syscall": + found_syscall = True + + # if we hit a 'syscall' but RAX or R10 haven't been touched + # then we are in an invalid path, so bail + if not syscall_finder.wants_syscall_inst or ( + not (found_movr10 and found_movreax) + ): + break + + else: + # attempt to see if any other instruction type wrote to registers + try: + _, regs_written = inst.regs_access() + except capstone.CsError: + continue + + if regs_written: + for r in regs_written: + # track writes to eax/rax or R10 + reg = inst.reg_name(r) + if reg in ["eax", "rax"]: + found_movreax = True + + elif reg == "r10": + found_movr10 = True + + # if any of these are missing, the block is invalid regardless of + # the technique we are trying to detect now or in the future + if not (found_movr10 and found_movreax and found_end): + return None + + # if the finder requires a 'syscall' instruction then bail now if we didn't find one + if syscall_finder.wants_syscall_inst and not found_syscall: + return None + + return disasm_bytes, end_inst + + @staticmethod + def get_disasm_function(architecture: str) -> Callable: + """ + Returns the disassembly handler for the given architecture + .detail is used to get full instruction information + + Args: + architecture: the name of the architecture for the process being disassembled + Returns: + The disasm function from capstone for the given architecture + """ + disasm_types = { + "intel": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32), + "intel64": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64), + } + + disasm_type = disasm_types[architecture] + disasm_type.detail = True + return disasm_type.disasm + + @classmethod + def _is_valid_syscall( + cls, + syscall_finder: syscall_finder_type, + proc_layer: interfaces.layers.DataLayerInterface, + architecture: str, + vads: List[Tuple[int, int, str]], + address: int, + ) -> Optional[Tuple[int, str]]: + """ + Args: + syscall_finder: + proc_layer: the memory layer of the process being scanned + architecture: the name of the architecture for the process being disassembled + vads: the ranges of this process under 10MB + address: the starting address to check for malicious syscall code blocks + + Returns: + Optional[Tuple[int, str]]: For valid code blocks, the starting address of the block and the disassembly string + """ + # the number bytes behind the yara rule hit to scan + behind = 32 + + address = address - behind + + try: + data = proc_layer.read(address, behind * 2) + except exceptions.InvalidAddressException: + return None + + disasm_func = cls.get_disasm_function(architecture) + + # since Intel does not have fixed-size instructions, we have to scan + # each byte offset and re-disassemble the remaining block + for offset in range(behind): + # if this looks like a system call back (r10, rax, ret/jmp) + syscall_info = cls._is_syscall_block( + disasm_func, syscall_finder, data[offset:], address + offset + ) + if syscall_info: + disasm_bytes, end_inst = syscall_info + + # if we can recover (and require) a target address for this malware technique + if syscall_finder.get_syscall_target_address: + target_address = syscall_finder.get_syscall_target_address( + proc_layer, end_inst + ) + + # could not determine the address -> invalid basic block + if not target_address: + continue + + # we only care about calls to system call DLLs + path = cls._get_range_path(vads, target_address) + if not isinstance(path, str) or not path.lower().endswith( + cls.valid_syscall_handlers + ): + continue + + # return the address and disassembly string if all checks pass + return address + offset, disasm_bytes + + return None + + @staticmethod + def _get_vad_maps( + task: interfaces.objects.ObjectInterface, + ) -> List[Tuple[int, int, str]]: + """Creates a map of start/end addresses within a virtual address + descriptor tree. + + Args: + task: The EPROCESS object of which to traverse the vad tree + + Returns: + An iterable of tuples containing start and end addresses for each descriptor + """ + vads: List[Tuple[int, int, str]] = [] + + # scan regions under 10MB + scan_max = 10 * 1000 * 1000 + + vad_root = task.get_vad_root() + + for vad in vad_root.traverse(): + if vad.get_size() < scan_max: + vads.append((vad.get_start(), vad.get_size(), vad.get_file_name())) + + return vads + + @staticmethod + def _get_range_path(ranges: List[Tuple[int, int, str]], address: int) -> Optional[str]: + """ + Returns the path for the range holding `address`, if found + + Args: + ranges: VADs collected from `_get_vad_maps` + address: the address to find + Returns: + The path holding the address, if any + """ + for start, size, path in ranges: + if start <= address < start + size: + return path + + return None + + @classmethod + def _get_tasks_to_scan( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table_name: str, + ) -> Generator[ + Tuple[interfaces.objects.ObjectInterface, str, str, str], None, None + ]: + """ + Gathers active processes with the extra information needed + to detect malicious syscall instructions + + Returns: + Generator of the process object, name, memory layer, and architecture + """ + + # gather active processes + filter_func = pslist.PsList.create_active_process_filter() + + is_32bit_arch = not symbols.symbol_table_is_64bit(context, symbol_table_name) + + for proc in pslist.PsList.list_processes( + context=context, + layer_name=layer_name, + symbol_table=symbol_table_name, + filter_func=filter_func, + ): + proc_name = utility.array_to_string(proc.ImageFileName) + + # skip Defender + if proc_name in ["MsMpEng.exe"]: + continue + + try: + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + if is_32bit_arch or proc.get_is_wow64(): + architecture = "intel" + else: + architecture = "intel64" + + yield proc, proc_name, proc_layer_name, architecture + + @classmethod + def _get_rule_hits( + cls, + context: interfaces.objects.ObjectInterface, + proc_layer: interfaces.layers.DataLayerInterface, + vads: List[Tuple[int, int, str]], + pattern: str, + ) -> Generator[Tuple[int, Optional[str]], None, None]: + """ + Runs the given opcode rule through Yara and returns the address and file path of hits + + Args: + context: + proc_layer: the layer to scan + vads: the ranges inside of the process being scanned + pattern: the opcodes rule from the plugin to detect a particular EDR-bypass technique + + Returns: + Generator of the address and file path of hits + """ + sections = [(vad[0], vad[1]) for vad in vads] + + rule = yarascan.YaraScanner.get_rule(pattern) + + for hit in proc_layer.scan( + context=context, + scanner=yarascan.YaraScanner(rules=rule), + sections=sections, + ): + address = hit[0] + + path = cls._get_range_path(vads, address) + + # ignore hits in the system call DLLs + if isinstance(path, str) and path.lower().endswith( + cls.valid_syscall_handlers + ): + continue + + yield address, path + + def _generator(self) -> Generator[Tuple[int, Tuple[str, int, Optional[str], int, str]], None, None]: + kernel = self.context.modules[self.config["kernel"]] + + for proc, proc_name, proc_layer_name, architecture in self._get_tasks_to_scan( + self.context, kernel.layer_name, kernel.symbol_table_name + ): + proc_layer = self.context.layers[proc_layer_name] + + vads = self._get_vad_maps(proc) + + # for each valid process, look for malicious syscall invocations + for address, vad_path in self._get_rule_hits( + self.context, proc_layer, vads, self.syscall_finder.rule_str + ): + syscall_info = self._is_valid_syscall( + self.syscall_finder, proc_layer, architecture, vads, address + ) + if not syscall_info: + continue + + address, disasm_bytes = syscall_info + + yield 0, ( + proc_name, + proc.UniqueProcessId, + vad_path, + format_hints.Hex(address), + disasm_bytes, + ) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("Range", str), + ("Address", format_hints.Hex), + ("Disasm", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py new file mode 100644 index 000000000..417c1b46e --- /dev/null +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -0,0 +1,124 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import struct +import logging +from typing import List, Optional + +import capstone + +from volatility3.framework import interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.plugins import yarascan +from volatility3.plugins.windows import pslist, direct_system_calls + +vollog = logging.getLogger(__name__) + + +class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.syscall_finder = direct_system_calls.syscall_finder_type( + # gets the target address of a indirect jmp + self._indirect_syscall_block_target, + # we are looking for indirect system calls, so we don't want 'syscall' instructions in our code block + False, + # jmp [address]; ret + "/\\xff\\x25[^\\xc3]{,24}\\xc3/", + # any of these mean we aren't in a malicious indirect call + ["call", "leave", "int3", "ret"], + # stop at jmp, this should reference the system call instruction + ["jmp"], + ) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # create a list of requirements for vadyarascan + vadyarascan_requirements = [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) + ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="direct_system_calls", + plugin=direct_system_calls.DirectSystemCalls, + version=(1, 0, 0), + ), + ] + + # get base yarascan requirements for command line options + yarascan_requirements = yarascan.YaraScan.get_yarascan_option_requirements() + + # return the combined requirements + return yarascan_requirements + vadyarascan_requirements + + @staticmethod + def _indirect_syscall_block_target( + proc_layer: interfaces.layers.DataLayerInterface, inst: capstone._cs_insn + ) -> Optional[int]: + """ + This function determines the address of a jmp in the following form: + + jmp [address] + + To determine this, we must: + 1) Pull the 4 byte relative offset of 'address' inside the instruction + 2) Compute the full address of this relative offset + 3) Read from the address as it is being dereferenced + 4) Ensure the target address points to a 'syscall' instruction + + Args: + proc_layer: the layer of the potential syscall block + inst: the terminating instruction of the syscall block check + Returns: + The target address of the jump if it can be computed + """ + + try: + jmp_address_str = proc_layer.read(inst.address, 6) + except exceptions.InvalidAddressException: + return None + + # Should be an jmp... + if jmp_address_str[0:2] != b"\xff\x25": + return None + + # get the address of the 'jmp [address]' instrunction + relative_offset = struct.unpack(" Date: Thu, 12 Sep 2024 18:58:16 -0500 Subject: [PATCH 634/979] Fix formatting problem between black versions --- .../framework/plugins/windows/direct_system_calls.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 409f24750..1edc01671 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -304,7 +304,9 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return vads @staticmethod - def _get_range_path(ranges: List[Tuple[int, int, str]], address: int) -> Optional[str]: + def _get_range_path( + ranges: List[Tuple[int, int, str]], address: int + ) -> Optional[str]: """ Returns the path for the range holding `address`, if found @@ -407,7 +409,9 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): yield address, path - def _generator(self) -> Generator[Tuple[int, Tuple[str, int, Optional[str], int, str]], None, None]: + def _generator( + self, + ) -> Generator[Tuple[int, Tuple[str, int, Optional[str], int, str]], None, None]: kernel = self.context.modules[self.config["kernel"]] for proc, proc_name, proc_layer_name, architecture in self._get_tasks_to_scan( From fce2125a8e4d2742720cc90f9d80e1e2fed9f4c8 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 15:03:51 -0500 Subject: [PATCH 635/979] Make VAD API public as intended --- .../plugins/windows/direct_system_calls.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 1edc01671..1999563fd 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -49,6 +49,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): _required_framework_version = (2, 4, 0) _version = (1, 0, 0) + # DLLs that are expected to host system call invocations valid_syscall_handlers = ("ntdll.dll", "win32u.dll") def __init__(self, *args, **kwargs): @@ -266,7 +267,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): continue # we only care about calls to system call DLLs - path = cls._get_range_path(vads, target_address) + path = cls.get_range_path(vads, target_address) if not isinstance(path, str) or not path.lower().endswith( cls.valid_syscall_handlers ): @@ -278,7 +279,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return None @staticmethod - def _get_vad_maps( + def get_vad_maps( task: interfaces.objects.ObjectInterface, ) -> List[Tuple[int, int, str]]: """Creates a map of start/end addresses within a virtual address @@ -304,14 +305,14 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return vads @staticmethod - def _get_range_path( + def get_range_path( ranges: List[Tuple[int, int, str]], address: int ) -> Optional[str]: """ Returns the path for the range holding `address`, if found Args: - ranges: VADs collected from `_get_vad_maps` + ranges: VADs collected from `get_vad_maps` address: the address to find Returns: The path holding the address, if any @@ -399,7 +400,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): ): address = hit[0] - path = cls._get_range_path(vads, address) + path = cls.get_range_path(vads, address) # ignore hits in the system call DLLs if isinstance(path, str) and path.lower().endswith( @@ -419,7 +420,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): ): proc_layer = self.context.layers[proc_layer_name] - vads = self._get_vad_maps(proc) + vads = self.get_vad_maps(proc) # for each valid process, look for malicious syscall invocations for address, vad_path in self._get_rule_hits( From 262c7f1aa7e7113508d067afc0e45219643ab943 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 14 Sep 2024 15:22:50 -0500 Subject: [PATCH 636/979] Make VAD API public as intended --- volatility3/framework/plugins/windows/direct_system_calls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 1999563fd..151776e75 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -324,7 +324,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return None @classmethod - def _get_tasks_to_scan( + def get_tasks_to_scan( cls, context: interfaces.context.ContextInterface, layer_name: str, @@ -415,7 +415,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): ) -> Generator[Tuple[int, Tuple[str, int, Optional[str], int, str]], None, None]: kernel = self.context.modules[self.config["kernel"]] - for proc, proc_name, proc_layer_name, architecture in self._get_tasks_to_scan( + for proc, proc_name, proc_layer_name, architecture in self.get_tasks_to_scan( self.context, kernel.layer_name, kernel.symbol_table_name ): proc_layer = self.context.layers[proc_layer_name] From e7fca5a83f6607a72183d560d4646121d85291e2 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 16 Sep 2024 09:38:15 -0500 Subject: [PATCH 637/979] Update year --- volatility3/framework/plugins/windows/direct_system_calls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 151776e75..eac18e8d4 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -1,4 +1,4 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # From af65d7e32daf778488928960e176d8774a2ed1b1 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 18 Oct 2024 11:33:38 -0500 Subject: [PATCH 638/979] Address feedback --- .../plugins/windows/direct_system_calls.py | 16 ++++++++++++++-- .../plugins/windows/indirect_system_calls.py | 10 +++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index eac18e8d4..ec8cd811b 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -3,7 +3,7 @@ # import logging -import capstone + from collections import namedtuple from typing import List, Tuple, Optional, Generator, Callable @@ -16,6 +16,12 @@ from volatility3.plugins.windows import pslist vollog = logging.getLogger(__name__) +try: + import capstone + + has_capstone = True +except ImportError: + has_capstone = False # Full details on the techniques used in these plugins to detect EDR-evading malware # can be found in our 20 page whitepaper submitted to DEFCON along with the presentation @@ -114,7 +120,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): 1) update RAX to the system call number 2) update R10 to the first parameter - 3) hit the 'termination' instrunction set in `syscall_finder_type` + 3) hit the 'termination' instruction set in `syscall_finder_type` We also track whether the 'syscall' instruction was encountered while parsing @@ -413,6 +419,12 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): def _generator( self, ) -> Generator[Tuple[int, Tuple[str, int, Optional[str], int, str]], None, None]: + if not has_capstone: + vollog.warning( + "capstone is not installed. This plugin requires capstone to operate." + ) + return + kernel = self.context.modules[self.config["kernel"]] for proc, proc_name, proc_layer_name, architecture in self.get_tasks_to_scan( diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py index 417c1b46e..b6494645e 100644 --- a/volatility3/framework/plugins/windows/indirect_system_calls.py +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -6,8 +6,6 @@ import struct import logging from typing import List, Optional -import capstone - from volatility3.framework import interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.plugins import yarascan @@ -15,6 +13,12 @@ from volatility3.plugins.windows import pslist, direct_system_calls vollog = logging.getLogger(__name__) +# The generator of DirectSystemCalls will bail with a warning if capstone is not installed +try: + import capstone +except ImportError: + pass + class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): _required_framework_version = (2, 4, 0) @@ -98,7 +102,7 @@ class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): if jmp_address_str[0:2] != b"\xff\x25": return None - # get the address of the 'jmp [address]' instrunction + # get the address of the 'jmp [address]' instruction relative_offset = struct.unpack(" Date: Mon, 9 Dec 2024 13:19:06 -0600 Subject: [PATCH 639/979] Add capstone to test system requirements. Allow lazy type checks --- pyproject.toml | 1 + volatility3/framework/plugins/windows/direct_system_calls.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fc1ab96cb..9b4b8d485 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dev = [ test = [ "volatility3[dev]", "pytest>=8.3.3,<9", + "capstone>=5.0.3,<6", "yara-x>=0.10.0,<1", ] diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index ec8cd811b..51035b2bc 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -112,7 +112,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): syscall_finder: syscall_finder_type, data: bytes, address: int, - ) -> Optional[Tuple[str, capstone._cs_insn]]: + ) -> Optional[Tuple[str, "capstone._cs_insn"]]: """ Determines if the bytes starting at `data` represent a valid syscall instrunction invocation block From a07ee5a0d53b553b854a8f2ff697ddf7922255a2 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 9 Dec 2024 17:49:22 -0600 Subject: [PATCH 640/979] fix(Windows: Handles): Unreliable SAR value on 24H2 Handles are not being decoded in 24H2+ samples. This is because the `Handles._decode_pointer` method grabs the SAR shift value from the disassemble function, but in these samples this value (`0x11`) is incorrect. Adding a fallback to the default SAR value of `0x10` if the obtained pointer is not valid in the kernel address space resolves the issue. --- volatility3/framework/plugins/windows/handles.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 3e5a2fd82..e5cbbf4ca 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -21,11 +21,14 @@ except ImportError: has_capstone = False +DEFAULT_SAR_VALUE = 0x10 # to be used only when decoding fails + + class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -118,6 +121,10 @@ class Handles(interfaces.plugins.PluginInterface): ) offset = self._decode_pointer(handle_table_entry.LowValue, magic) + if not self.context.layers[virtual].is_valid(offset): + offset = self._decode_pointer( + handle_table_entry.LowValue, DEFAULT_SAR_VALUE + ) else: if handle_table_entry.InfoTable == 0: return None @@ -142,7 +149,6 @@ class Handles(interfaces.plugins.PluginInterface): pointers in the _HANDLE_TABLE_ENTRY which allows us to find the associated _OBJECT_HEADER. """ - DEFAULT_SAR_VALUE = 0x10 # to be used only when decoding fails if self._sar_value is None: if not has_capstone: From 1cde9ae06ee855e084fe5631eb37f2ffa979ef5c Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 09:46:28 +0000 Subject: [PATCH 641/979] Slightly modify volshell.rst --- doc/source/volshell.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 3c4f4ce5d..c95456dda 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -36,7 +36,7 @@ operating system mode for volshell, and the current layer available for use. (primary) >>> -Volshell itself in essentially a plugin, but an interactive one. As such, most values are accessed through `self` +Volshell itself is essentially a plugin, but an interactive one. As such, most values are accessed through `self` although there is also a `context` object whenever a context must be provided. The prompt for the tool will indicate the name of the current layer (which can be accessed as `self.current_layer` @@ -92,7 +92,7 @@ It can also be provided with an object and will interpret the data for each in t 0x2e8 : UniqueProcessId symbol_table_name1!pointer 4 ... -These values can be accessed directory as attributes +These values can be accessed directly as attributes :: @@ -180,7 +180,7 @@ used: layer = cc(mynewlayer.MyNewLayer, on_top_of = 'primary', other_parameter = 'important') with open('output.dmp', 'wb') as fp: - for i in range(0, 1073741824, 0x1000): + for i in range(0, 0x4000000, 0x1000): data = layer.read(i, 0x1000, pad = True) fp.write(data) From fdd49d0921a8ca22ea388573dec33ba93a1ef485 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 10:11:26 +0000 Subject: [PATCH 642/979] Slightly modify documentation --- doc/source/glossary.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 66dabfafe..c4a93f908 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -145,9 +145,9 @@ Struct, Structure Symbol This is used in many different contexts, as a short term for many things. Within Volatility, a symbol is a - construct that usually encompasses a specific type :ref:`type` at a specific :ref:`offset`, + construct that usually encompasses a specific :ref:`type` at a specific :ref:`offset`, representing a particular instance of that type within the memory of a compiled and running program. An example - would be the location in memory of a list of active tcp endpoints maintained by the networking stack + would be the location in memory of a list of active TCP endpoints maintained by the networking stack within an operating system. T From 4bccf116292e6269f0ecc306b8dba0973af55697 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:31:16 +0000 Subject: [PATCH 643/979] Remove redundant part of if statement Also reorder imports. --- volatility3/cli/volshell/generic.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 82c470e1a..534546dcd 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -11,11 +11,6 @@ import sys from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union from urllib import parse, request -from volatility3.cli import text_renderer, volshell -from volatility3.framework import exceptions, interfaces, objects, plugins, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.layers import intel, physical, resources, scanners - try: import capstone @@ -23,6 +18,11 @@ try: except ImportError: has_capstone = False +from volatility3.cli import text_renderer, volshell +from volatility3.framework import exceptions, interfaces, objects, plugins, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import intel, physical, resources, scanners + class Volshell(interfaces.plugins.PluginInterface): """Shell environment to directly interact with a memory image.""" @@ -553,12 +553,11 @@ class Volshell(interfaces.plugins.PluginInterface): if argname in kwargs: del kwargs[argname] - for keyword in kwargs: - val = kwargs[keyword] + for keyword, val in kwargs.items(): if not isinstance( val, interfaces.configuration.BasicTypes ) and not isinstance(val, list): - if not isinstance(val, list) or all( + if all( isinstance(x, interfaces.configuration.BasicTypes) for x in val ): raise TypeError( From faa6cab797da8719305f49e1e824448a159509eb Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:16:42 +0000 Subject: [PATCH 644/979] Remove redundant part of if statement Also reorder imports. --- volatility3/cli/volshell/generic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 534546dcd..b1a61fcff 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -555,8 +555,8 @@ class Volshell(interfaces.plugins.PluginInterface): for keyword, val in kwargs.items(): if not isinstance( - val, interfaces.configuration.BasicTypes - ) and not isinstance(val, list): + val, (interfaces.configuration.BasicTypes, list) + ): if all( isinstance(x, interfaces.configuration.BasicTypes) for x in val ): From 0b2f4fdeb772ccb097aaf310ff3169ae37279f13 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:26:39 +0000 Subject: [PATCH 645/979] Remove redundant part of if statement Also reorder imports. --- volatility3/cli/volshell/generic.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index b1a61fcff..08132608b 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -554,12 +554,8 @@ class Volshell(interfaces.plugins.PluginInterface): del kwargs[argname] for keyword, val in kwargs.items(): - if not isinstance( - val, (interfaces.configuration.BasicTypes, list) - ): - if all( - isinstance(x, interfaces.configuration.BasicTypes) for x in val - ): + if not isinstance(val, (interfaces.configuration.BasicTypes, list)): + if all(isinstance(x, interfaces.configuration.BasicTypes) for x in val): raise TypeError( "Configurable values must be simple types (int, bool, str, bytes)" ) From 39cb75accae827436d8d923592f1d62b34cf9ede Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:02:48 +0000 Subject: [PATCH 646/979] Slightly modify documentation Include regex_scan, new functionality of volshell. --- doc/source/volshell.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index c95456dda..73b000763 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -187,8 +187,22 @@ used: As this demonstrates, all of the python is accessible, as are the volshell built in functions (such as `cc` which creates a constructable, like a layer or a symbol table). +User Convenience +---------------- + +There are functions available that make often-done tasks easiers, and generally provide a shell-like experience. These can be listed using `help()` which, as already mentioned, is mentioned when volshell starts. + Loading files -------------- +^^^^^^^^^^^^^ Files can be loaded as physical layers using the `load_file` or `lf` command, which takes a filename or a URI. This will be added to `context.layers` and can be accessed by the name returned by `lf`. + +Regex +^^^^^ + +It is easy to scan for some bytes or a pattern using `regex_scan` or `rx`. + +An optional size can be given for the displayed results as with the other fuctions (db, dw, dd, dq, etc). + +You can of course specify a different layer name as well. From fc33fd912787d0420cd9c4c5effbfba0ea89a933 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:14:59 +0000 Subject: [PATCH 647/979] Slightly modify documentation Include regex_scan, new functionality of volshell. --- doc/source/volshell.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 73b000763..b46647dc3 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -190,7 +190,7 @@ creates a constructable, like a layer or a symbol table). User Convenience ---------------- -There are functions available that make often-done tasks easiers, and generally provide a shell-like experience. These can be listed using `help()` which, as already mentioned, is mentioned when volshell starts. +There are functions available that make often-done tasks easier, and generally provide a shell-like experience. These can be listed using `help()` which, as already mentioned, is advertised when volshell starts. Loading files ^^^^^^^^^^^^^ From b6717d80d9ac5c820cf84803b53c9aee1e115464 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 10 Dec 2024 23:55:44 +0000 Subject: [PATCH 648/979] Windows: Fix up minor typo and CodeQL warning --- volatility3/framework/plugins/windows/direct_system_calls.py | 2 +- volatility3/framework/plugins/windows/indirect_system_calls.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 51035b2bc..b0c162f46 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -39,7 +39,7 @@ syscall_finder_type = namedtuple( ) syscall_finder_type.__doc__ = """ -This type to used to specify how malicious system call invocations should be found. +This type is used to specify how malicious system call invocations should be found. `get_syscall_target_address` is optionally used to extract the address containing the malicious 'syscall' instruction `wants_syscall_inst` whether or not this method expects the 'syscall' instrunction directly within the malicious code block diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py index b6494645e..1a5eb317f 100644 --- a/volatility3/framework/plugins/windows/indirect_system_calls.py +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -13,10 +13,10 @@ from volatility3.plugins.windows import pslist, direct_system_calls vollog = logging.getLogger(__name__) -# The generator of DirectSystemCalls will bail with a warning if capstone is not installed try: import capstone except ImportError: + # The generator of DirectSystemCalls will bail with a warning if capstone is not installed pass From 3d260f3829e5f7bd4611db726358169498bb6ae8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:42:20 +0000 Subject: [PATCH 649/979] Slightly modify documentation Include regex_scan, new functionality of volshell. --- doc/source/volshell.rst | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index b46647dc3..47ea2e905 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -203,6 +203,45 @@ Regex It is easy to scan for some bytes or a pattern using `regex_scan` or `rx`. +:: + + (layer_name) >>> rx(rb"(Linux version|Darwin Kernel Version) [0-9]+\.[0-9]+\.[0-9]+") + 0x880001400070 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0x880001400080 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0x880001400090 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0x8800014000a0 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0x8800014000b0 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0x8800014000c0 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0x8800014000d0 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0x8800014000e0 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0x880001769027 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0x880001769037 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0x880001769047 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0x880001769057 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0x880001769067 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0x880001769077 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0x880001769087 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0x880001769097 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0xffff81400070 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0xffff81400080 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0xffff81400090 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0xffff814000a0 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0xffff814000b0 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0xffff814000c0 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0xffff814000d0 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0xffff814000e0 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0xffff81769027 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0xffff81769037 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0xffff81769047 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0xffff81769057 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0xffff81769067 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0xffff81769077 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0xffff81769087 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0xffff81769097 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + An optional size can be given for the displayed results as with the other fuctions (db, dw, dd, dq, etc). -You can of course specify a different layer name as well. +You can, of course, specify a different layer name as well. From 9d0cd4b4c985a568083e20cb4ff13164e6d60963 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 12 Dec 2024 11:17:15 +1100 Subject: [PATCH 650/979] Linux: PageCache: Update inode plugin to conform to framework dumping convention --- .../framework/plugins/linux/pagecache.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 005fc9acc..6d2607ada 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -389,7 +389,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -412,9 +412,10 @@ class InodePages(plugins.PluginInterface): description="Inode address", optional=True, ), - requirements.StringRequirement( + requirements.BooleanRequirement( name="dump", - description="Output file path", + description="Extract inode content", + default=False, optional=True, ), ] @@ -436,7 +437,7 @@ class InodePages(plugins.PluginInterface): """ if not inode.is_reg: vollog.error("The inode is not a regular file") - return + return None # By using truncate/seek, provided the filesystem supports it, a sparse file will be # created, saving both disk space and I/O time. @@ -471,7 +472,7 @@ class InodePages(plugins.PluginInterface): if self.config["inode"] and self.config["find"]: vollog.error("Cannot use --inode and --find simultaneously") - return + return None if self.config["find"]: inodes_iter = Files.get_inodes( @@ -487,15 +488,15 @@ class InodePages(plugins.PluginInterface): inode = vmlinux.object("inode", self.config["inode"], absolute=True) else: vollog.error("You must use either --inode or --find") - return + return None if not inode.is_valid(): vollog.error("Invalid inode at 0x%x", inode.vol.offset) - return + return None if not inode.is_reg: vollog.error("The inode is not a regular file") - return + return None inode_size = inode.i_size for page_obj in inode.get_pages(): @@ -520,8 +521,13 @@ class InodePages(plugins.PluginInterface): if self.config["dump"]: filename = self.config["dump"] - vollog.info("[*] Writing inode at 0x%x to '%s'", inode.vol.offset, filename) - self.write_inode_content_to_file(inode, filename, self.open, vmlinux_layer) + open_method = self.open + inode_address = inode.vol.offset + filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") + vollog.info("[*] Writing inode at 0x%x to '%s'", inode_address, filename) + self.write_inode_content_to_file( + inode, filename, open_method, vmlinux_layer + ) def run(self): headers = [ From 58a9c3d6dae0759e0dbf590d53e14d7553f30a28 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 07:20:30 +0000 Subject: [PATCH 651/979] Slightly modify documentation Include regex_scan, new functionality of volshell. Add Intermediate Symbol File (ISF) to glossary. --- doc/source/glossary.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index c4a93f908..a9460b1a2 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -23,7 +23,7 @@ Alignment .. _Array: Array - This represents a list of items, which can be access by an index, which is zero-based (meaning the first + This represents a list of items, which can be accessed by an index, which is zero-based (meaning the first element has index 0). Items in arrays are almost always the same size (it is not a generic list, as in python) even if they are :ref:`pointers` to different sized objects. @@ -43,7 +43,14 @@ Dereference .. _Domain: Domain - This the grouping for input values for a mapping or mathematical function. + The set of input values for a mapping or mathematical function. + +I +- +.. _Intermediate Symbol File (ISF): + +Intermediate Symbol File (ISF) + They contain kernel structures and specific offsets formatted as JSON. For macOS and Linux analysis, the kernel needs to be added as an ISF file to the volatility 3 symbols directory. For Windows, the required ISF file can often be generated from PDB files automatically downloaded from Microsoft servers, and therefore does not require manual intervention. M - @@ -55,7 +62,7 @@ Map, mapping attempts to use mathematical functional notation where possible. Within volatility a mapping is most often used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). For further information, please see - `Function (mathematics) in wikipedia https://en.wikipedia.org/wiki/Function_(mathematics)` + `https://en.wikipedia.org/wiki/Function_(mathematics)`. .. _Member: @@ -69,7 +76,7 @@ O .. _Object: Object - This has a specific meaning within computer programming (as in Object Oriented Programming), but within the world + This has a specific meaning within computer programming (as in object-oriented programming), but within the world of Volatility it is used to refer to a type that has been associated with a chunk of data, or a specific instance of a type. See also :ref:`Type`. From 6ffef285f4c1ba39816d76726f00086029abdedc Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:36:51 +0000 Subject: [PATCH 652/979] Tweak the getting started linux tutorial --- doc/source/getting-started-linux-tutorial.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst index d4b40d053..a1aad235d 100644 --- a/doc/source/getting-started-linux-tutorial.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -27,7 +27,7 @@ To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol Listing plugins --------------- -The following is a sample of the linux plugins available for volatility3, it is not complete and more more plugins may +The following is a sample of the linux plugins available for volatility3, it is not complete and more plugins may be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. For plugin requests, please create an issue with a description of the requested plugin. @@ -40,7 +40,7 @@ For plugin requests, please create an issue with a description of the requested linux.check_creds.Check_creds linux.check_idt.Check_idt -.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of linux plugins. +.. note:: Here the the command is piped to grep and head to provide the start of the list of linux plugins. Using plugins @@ -80,9 +80,9 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for the needed ISF file from the ISF server. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. +If an ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. -.. tip:: Use the banner text which is most repeated to search from ISF Server. +.. tip:: Use the banner text which is most repeated to search on the ISF Server. linux.pslist ~~~~~~~~~~~~ @@ -157,7 +157,7 @@ linux.pstree ***** 1548 1266 gsd-keyboard ***** 1550 1266 gsd-media-keys -``linux.pstree`` helps us to display the parent child relationships between processes. +``linux.pstree`` helps us to display the parent-child relationships between processes. linux.bash ~~~~~~~~~~ From e31e13f471006f7dcff11f8b7f601b45a9cdf471 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:44:01 +0000 Subject: [PATCH 653/979] Tweak the getting started mac tutorial --- doc/source/getting-started-mac-tutorial.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst index 42e58c0d5..61af7089b 100644 --- a/doc/source/getting-started-mac-tutorial.rst +++ b/doc/source/getting-started-mac-tutorial.rst @@ -37,7 +37,7 @@ For plugin requests, please create an issue with a description of the requested mac.check_sysctl.Check_sysctl mac.check_trap_table.Check_trap_table -.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of macOS plugins. +.. note:: Here the the command is piped to grep and head to provide the start of the list of macOS plugins. Using plugins @@ -78,7 +78,7 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-mac-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. +If an ISF file cannot be found then, follow the instructions on :ref:`getting-started-mac-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. mac.pslist ~~~~~~~~~~ @@ -125,7 +125,7 @@ mac.pstree 337 1 system_installd * 455 337 update_dyld_shar -``mac.pstree`` helps us to display the parent child relationships between processes. +``mac.pstree`` helps us to display the parent-child relationships between processes. mac.ifconfig ~~~~~~~~~~~~ @@ -150,4 +150,4 @@ mac.ifconfig utun0 False utun0 fe80:5::2a95:bb15:87e3:977c False -we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. +We can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. From d77d696e2b017fa7dc8b2355efa9cdde88470d6e Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:49:25 +0000 Subject: [PATCH 654/979] Tweak the getting started windows tutorial --- doc/source/getting-started-windows-tutorial.rst | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/doc/source/getting-started-windows-tutorial.rst b/doc/source/getting-started-windows-tutorial.rst index c89b065f5..979cf1d96 100644 --- a/doc/source/getting-started-windows-tutorial.rst +++ b/doc/source/getting-started-windows-tutorial.rst @@ -15,19 +15,19 @@ Memory can be acquired using a number of tools, below are some examples but othe Listing Plugins --------------- -The following is a sample of the windows plugins available for volatility3, it is not complete and more more plugins may +The following is a sample of the windows plugins available for volatility3, it is not complete and more plugins may be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. For plugin requests, please create an issue with a description of the requested plugin. .. code-block:: shell-session - $ python3 vol.py --help | grep windows | head -n 5 + $ python3 vol.py --help | grep windows | head -n 4 windows.bigpools.BigPools windows.cmdline.CmdLine windows.crashinfo.Crashinfo windows.dlllist.DllList -.. note:: Here the the command is piped to grep and head in-order to provide the start of a list of the available windows plugins. +.. note:: Here the the command is piped to grep and head to provide the start of a list of the available windows plugins. Using plugins ------------- @@ -95,9 +95,9 @@ windows.pstree ** 616 504 svchost.exe 0xfa8002b86ab0 13 314 0 False 2022-02-07 16:32:16.000000 N/A ** 624 504 svchost.exe 0xfa8002410630 10 350 0 False 2022-02-07 16:30:14.000000 N/A -``windows.pstree`` helps to display the parent child relationships between processes. +``windows.pstree`` helps to display the parent-child relationships between processes. -.. note:: Here the the command is piped to head in-order to provide smaller output, here listing only the first 20. +.. note:: Here the the command is piped to head to provide smaller output, here listing only the first 20. windows.hashdump ~~~~~~~~~~~~~~~~ @@ -116,9 +116,3 @@ windows.hashdump Dennis 1003 aad3b435b51404eeaad3b435b51404ee cf96684bbc7877920adaa9663698bf54 ``windows.hashdump`` helps to list the hashes of the users in the system. - - - - - - From c740a6c77017834329f0361866536509938bc0c4 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:14:35 +0000 Subject: [PATCH 655/979] Modify using as a library documentation Tiny changes. --- doc/source/using-as-a-library.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index 4acf35f98..55b77e90a 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -3,7 +3,7 @@ Using Volatility 3 as a Library This portion of the documentation discusses how to access the Volatility 3 framework from an external application. -The general process of using volatility as a library is to as follows: +The general process of using volatility as a library is as follows: 1. :ref:`create_context` 2. (Optional) :ref:`available_plugins` @@ -21,7 +21,7 @@ Creating a context First we make sure the volatility framework works the way we expect it (and is the version we expect). The versioning used is semantic versioning, meaning any version with the same major number and a higher or equal minor number will satisfy the requirement. An example is below since the CLI doesn't need any of the features -from versions 1.1 or 1.2: +from version 1.1: :: @@ -86,7 +86,7 @@ List requirements are a list of simple types (integers, booleans, floats and str options, multiple requirements needs all their subrequirements fulfilled and the other types require the names of valid translation layers or symbol tables within the context, respectively. Luckily, each of these requirements can tell you whether they've been fulfilled or not later in the process. For now, they can be used to ask the user to -fill in any parameters they made need to. Some requirements are optional, others are not. +fill in any parameters they may need to. Some requirements are optional, others are not. The plugin is essentially a multiple requirement. It should also be noted that automagic classes can have requirements (as can translation layers). @@ -100,7 +100,7 @@ Once you know what requirements the plugin will need, you can populate them with The configuration is essentially a hierarchical tree of values, much like the windows registry. Each plugin is instantiated at a particular branch within the hierarchy and will look for its configuration options under that hierarchy (if it holds any configurable items, it will likely instantiate those at a point -underneaths its own branch). To set the hierarchy, you'll need to know where the configurables will be constructed. +underneath its own branch). To set the hierarchy, you'll need to know where the configurables will be constructed. For this example, we'll assume plugins' base_config_path is set as `plugins`, and that automagics are configured under the `automagic` tree. We'll see later how to ensure this matches up with the plugins and automagic when they're @@ -139,7 +139,7 @@ A suitable list of automagics for a particular plugin (based on operating system This will take the plugin module, extract the operating system (first level of the hierarchy) and then return just the automagics which apply to the operating system. Each automagic can exclude itself from being used for specific -operating systems, so that an automagic designed for linux is not used for windows or mac plugins. +operating systems, so that for example an automagic designed for linux is not used for windows or mac plugins. These automagics can then be run by providing the list, the context, the plugin to be run, the hierarchy name that the plugin will be constructed on ('plugins' by default) and a progress_callback. This is a callable which takes @@ -157,8 +157,8 @@ Any exceptions that occur during the execution of the automagic will be returned Run the plugin -------------- -Firstly, we should check whether the plugin will be able to run (ie, whether the configuration options it needs -have been successfully set). We do this as follow (where plugin_config_path is the base_config_path (which defaults +Firstly, we should check whether the plugin will be able to run (i.e., whether the configuration options it needs +have been successfully set). We do this as follows, where plugin_config_path is the base_config_path (which defaults to `plugins` and then the name of the class itself): :: @@ -166,7 +166,7 @@ to `plugins` and then the name of the class itself): unsatisfied = plugin.unsatisfied(context, plugin_config_path) If unsatisfied is an empty list, then the plugin has been given everything it requires. If not, it will be a -Dictionary of the hierarchy paths and their associated requirements that weren't satisfied. +dictionary of the hierarchy paths and their associated requirements that weren't satisfied. The plugin can then be instantiated with the context (containing the plugin's configuration) and the path that the plugin can find its configuration at. This configuration path only needs to be a unique value to identify where the From b235ed05b7f916de864916db98e7f2385a52bb07 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:50:23 +0000 Subject: [PATCH 656/979] Update the CLI manual documentation --- doc/source/vol-cli.rst | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 7b91e815d..9fb48e67a 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -58,7 +58,7 @@ Options EXTEND. Extensions must be of the form **configuration.item.name=value** -p PLUGIN_DIRS, --plugin-dirs PLUGIN_DIRS - Specified a semi-colon separated list of paths that contain directories + Specified as a semi-colon separated list of paths that contain directories where plugins may be found. These paths are searched before the default paths when loading python files for plugins. This can therefore be used to override built-in plugins. NOTE: All python code within this directory @@ -67,12 +67,12 @@ Options -s SYMBOL_DIRS, --symbol-dirs SYMBOL_DIRS SYMBOL_DIRS is a semi-colon separated list of paths that contain symbol files or symbol zip packs. Symbols must be within a particular directory - structure if they depending on the operating system of the symbols, + structure if they depend on the operating system of the symbols, whilst symbol packs must be in the root of the directory and named after - the after the operating system to which they apply. + the operating system to which they apply. -v, --verbose - A flag which can be used multiple times, each time increasing the level of + A flag which can be used multiple times (up to four), each time increasing the level of detail in the logs produced. -l LOG, --log LOG @@ -87,7 +87,7 @@ Options -q, --quiet When present, this flag mutes the progress feedback for operations. This can be beneficial when piping the output directly to a file or another - tool. This also removes the + tool. -r RENDERER, --renderer RENDERER Specifies the output format in which to display results. The default is @@ -120,9 +120,7 @@ Options Change the default path used to store the cache. --offline - Do not search online for additional JSON files. - Run offline mode (defaults to false) and for - remote windows symbol tables, linux/mac banner repositories. + Run offline mode (defaults to false). Do not search online for additional JSON files, remote windows symbol tables, nor linux/mac banner repositories. --single-location SINGLE_LOCATION This specifies a URL which will be downloaded if necessary, and built @@ -152,7 +150,7 @@ but can be overridden by creating a JSON file (`%APPDATA%/volatility3/vol.json` systems, or `~/.config/volatility3/vol.json` or `volshell.json` for all others). The format of this file is a JSON dictionary, containing the options above and their value. -It should be noted that the ordering is (`<` means is overridden by): +It should be noted that the ordering is (`x < y` means `x` is overridden by `y`): `in-built default value < config file value < command line parameter` From 04517ca79768c16cb8afc323a5625567fe87d225 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 12 Dec 2024 23:24:40 -0600 Subject: [PATCH 657/979] Windows: Handle missing _MM_SESSION_SPACE As of Windows 11 24H2, the `_MM_SESSION_SPACE` type no longer appears in the kernel PDB. Instead, the `_EPROCESS.Session` member refers to a new type, `_PSP_SESSION_SPACE`, which does not have a type definition. However, experimentation has shown that this new structure is functionally identical to the old structure - the `ProcessList` and `SessionId` members still appear to be at their old offsets. In order to account for this when analyzing these newer Windows versions, this catches the `SymbolError` and instantiates an `unsigned long` at the offset (8) where the `SessionId` member would normally be defined within an `_MM_SESSION_SPACE` structure. --- .../framework/plugins/windows/modules.py | 34 ++++++++++++++---- .../symbols/windows/extensions/__init__.py | 35 +++++++++++++------ 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index ba45834d5..5ff252074 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -165,13 +165,33 @@ class Modules(interfaces.plugins.PluginInterface): # create the session space object in the process' own layer. # not all processes have a valid session pointer. - session_space = context.object( - symbol_table + constants.BANG + "_MM_SESSION_SPACE", - layer_name=layer_name, - offset=proc.Session, - ) + try: + session_space = context.object( + symbol_table + constants.BANG + "_MM_SESSION_SPACE", + layer_name=layer_name, + offset=proc.Session, + ) + session_id = session_space.SessionId - if session_space.SessionId in seen_ids: + except exceptions.SymbolError: + # In Windows 11 24H2, the _MM_SESSION_SPACE type was + # replaced with _PSP_SESSION_SPACE, and the kernel PDB + # doesn't contain information about its members (otherwise, + # we would just fall back to the new type). However, it + # appears to be, for our purposes, functionally identical + # to the _MM_SESSION_SPACE. Because _MM_SESSION_SPACE + # stores its session ID at offset 8 as an unsigned long, we + # create an unsigned long at that offset and use that + # instead. + session_id = int( + context.object( + layer_name=layer_name, + object_type=symbol_table + constants.BANG + "unsigned long", + offset=proc.Session + 8, + ) + ) + + if session_id in seen_ids: continue except exceptions.InvalidAddressException: @@ -184,7 +204,7 @@ class Modules(interfaces.plugins.PluginInterface): continue # save the layer if we haven't seen the session yet - seen_ids.append(session_space.SessionId) + seen_ids.append(session_id) yield proc_layer_name @classmethod diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 793e506c3..78f59fc0c 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -813,23 +813,36 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): offset=kvo, native_layer_name=self.vol.native_layer_name, ) - session = ntkrnlmp.object( - object_type="_MM_SESSION_SPACE", offset=self.Session, absolute=True - ) - - if session.has_member("SessionId"): - return session.SessionId + try: + session = ntkrnlmp.object( + object_type="_MM_SESSION_SPACE", + offset=self.Session, + absolute=True, + ) + if session.has_member("SessionId"): + return session.SessionId + except exceptions.SymbolError: + # In Windows 11 24H2, the _MM_SESSION_SPACE type was + # replaced with _PSP_SESSION_SPACE, and the kernel PDB + # doesn't contain information about its members (otherwise, + # we would just fall back to the new type). However, it + # appears to be, for our purposes, functionally identical + # to the _MM_SESSION_SPACE. Because _MM_SESSION_SPACE + # stores its session ID at offset 8 as an unsigned long, we + # create an unsigned long at that offset and use that + # instead. + session_id = ntkrnlmp.object( + object_type="unsigned long", + offset=self.Session + 8, + absolute=True, + ) + return int(session_id) except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, f"Cannot access _EPROCESS.Session.SessionId at {self.vol.offset:#x}", ) - except exceptions.SymbolError: - vollog.log( - constants.LOGLEVEL_VVV, - "Could not lookup _MM_SESSION_SPACE in symbol table", - ) return renderers.UnreadableValue() From e8b318552839e25264ebade5c0cc58d3fae27b12 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 13 Dec 2024 00:09:02 -0600 Subject: [PATCH 658/979] Windows: Handles - New pointer calculation method After researching this structure (`_HANDLE_TABLE_ENTRY`), it appears to be stable as far back as Windows 8. It's also a union, with an `ObjectPointerBits` member at the same offset as `LowValue` but within a specific bit range (bit length 44, bit position 20). Taking this value and shifting it left by four produces the correct pointer. This four-bit shift is due to 16-byte alignment of object header structures, and is what we would expect to see with 44-bit pointers in Windows. See https://www.alex-ionescu.com/behind-windows-x64s-44-bit-memory-addressing-limit/ --- .../framework/plugins/windows/handles.py | 122 +----------------- 1 file changed, 4 insertions(+), 118 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index e5cbbf4ca..977a8c804 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -3,9 +3,9 @@ # import logging -from typing import List, Optional, Dict +from typing import Dict, List, Optional -from volatility3.framework import constants, exceptions, renderers, interfaces, symbols +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints @@ -13,16 +13,6 @@ from volatility3.plugins.windows import pslist, psscan vollog = logging.getLogger(__name__) -try: - import capstone - - has_capstone = True -except ImportError: - has_capstone = False - - -DEFAULT_SAR_VALUE = 0x10 # to be used only when decoding fails - class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" @@ -32,7 +22,6 @@ class Handles(interfaces.plugins.PluginInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._sar_value = None self._type_map = None self._cookie = None self._level_mask = 7 @@ -65,21 +54,6 @@ class Handles(interfaces.plugins.PluginInterface): ), ] - def _decode_pointer(self, value, magic): - """Windows encodes pointers to objects and decodes them on the fly - before using them. - - This function mimics the decoding routine so we can generate the - proper pointer values as well. - """ - - value = value & 0xFFFFFFFFFFFFFFF8 - value = value >> magic - # if (value & (1 << 47)): - # value = value | 0xFFFF000000000000 - - return value - def _get_item(self, handle_table_entry, handle_value): """Given a handle table entry (_HANDLE_TABLE_ENTRY) structure from a process' handle table, determine where the corresponding object's @@ -103,28 +77,11 @@ class Handles(interfaces.plugins.PluginInterface): ) if is_64bit: - if handle_table_entry.LowValue == 0: + if handle_table_entry.ObjectPointerBits == 0: return None - magic = self.find_sar_value() + offset = handle_table_entry.ObjectPointerBits << 4 - # is this the right thing to raise here? - if magic is None: - if has_capstone: - raise AttributeError( - "Unable to find the SAR value for decoding handle table pointers" - ) - else: - raise exceptions.MissingModuleException( - "capstone", - "Requires capstone to find the SAR value for decoding handle table pointers", - ) - - offset = self._decode_pointer(handle_table_entry.LowValue, magic) - if not self.context.layers[virtual].is_valid(offset): - offset = self._decode_pointer( - handle_table_entry.LowValue, DEFAULT_SAR_VALUE - ) else: if handle_table_entry.InfoTable == 0: return None @@ -142,77 +99,6 @@ class Handles(interfaces.plugins.PluginInterface): object_header.HandleValue = handle_value return object_header - def find_sar_value(self): - """Locate ObpCaptureHandleInformationEx if it exists in the sample. - - Once found, parse it for the SAR value that we need to decode - pointers in the _HANDLE_TABLE_ENTRY which allows us to find the - associated _OBJECT_HEADER. - """ - - if self._sar_value is None: - if not has_capstone: - vollog.debug( - "capstone module is missing, unable to create disassembly of ObpCaptureHandleInformationEx" - ) - return None - kernel = self.context.modules[self.config["kernel"]] - - virtual_layer_name = kernel.layer_name - kvo = self.context.layers[virtual_layer_name].config[ - "kernel_virtual_offset" - ] - ntkrnlmp = self.context.module( - kernel.symbol_table_name, layer_name=virtual_layer_name, offset=kvo - ) - - try: - func_addr = ntkrnlmp.get_symbol("ObpCaptureHandleInformationEx").address - except exceptions.SymbolError: - vollog.debug("Unable to locate ObpCaptureHandleInformationEx symbol") - return None - - try: - func_addr_to_read = kvo + func_addr - num_bytes_to_read = 0x200 - vollog.debug( - f"ObpCaptureHandleInformationEx symbol located at {hex(func_addr_to_read)}" - ) - data = self.context.layers.read( - virtual_layer_name, func_addr_to_read, num_bytes_to_read - ) - except exceptions.InvalidAddressException: - vollog.warning( - f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of {hex(DEFAULT_SAR_VALUE)}" - ) - self._sar_value = DEFAULT_SAR_VALUE - return self._sar_value - - md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) - - instruction_count = 0 - for address, size, mnemonic, op_str in md.disasm_lite( - data, kvo + func_addr - ): - # print("{} {} {} {}".format(address, size, mnemonic, op_str)) - instruction_count += 1 - if mnemonic.startswith("sar"): - # if we don't want to parse op strings, we can disasm the - # single sar instruction again, but we use disasm_lite for speed - self._sar_value = int(op_str.split(",")[1].strip(), 16) - vollog.debug( - f"SAR located at {hex(address)} with value of {hex(self._sar_value)}" - ) - break - - if self._sar_value is None: - vollog.warning( - f"Failed to to locate SAR value having parsed {instruction_count} instructions, failing back to a common value of {hex(DEFAULT_SAR_VALUE)}" - ) - self._sar_value = DEFAULT_SAR_VALUE - - return self._sar_value - @classmethod def get_type_map( cls, From 31492f4ab80dcc3c5ca38c2c4754ccfafc46a994 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 14 Dec 2024 15:50:50 +0000 Subject: [PATCH 659/979] Rectify maximum repetition of verbose flag From four to six (-vvvvvv). --- doc/source/vol-cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 9fb48e67a..43ca33f04 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -72,7 +72,7 @@ Options the operating system to which they apply. -v, --verbose - A flag which can be used multiple times (up to four), each time increasing the level of + A flag which can be used multiple times (up to six, -vvvvvv), each time increasing the level of detail in the logs produced. -l LOG, --log LOG From 5086be30b2c153bf168704022cff793190e1a750 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Sun, 15 Dec 2024 14:04:15 +0800 Subject: [PATCH 660/979] Refactor: move version None check to top --- volatility3/framework/configuration/requirements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 86e1aac52..f130f9544 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -529,6 +529,8 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): component: Type[interfaces.configuration.VersionableInterface] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: + if version is None: + raise TypeError("Version cannot be None") if description is None: description = f"Version {'.'.join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( @@ -537,8 +539,6 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): if component is None: raise TypeError("Component cannot be None") self._component: Type[interfaces.configuration.VersionableInterface] = component - if version is None: - raise TypeError("Version cannot be None") self._version = version def unsatisfied( From c8c39837abdf489c8316aa51ebb4f2634321d77c Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 15 Dec 2024 19:26:22 +0000 Subject: [PATCH 661/979] Tiny change text_renderer.py --- volatility3/cli/text_renderer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 31307f67e..408a562d8 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -49,12 +49,12 @@ def hex_bytes_as_text(value: bytes, width: int = 16) -> str: output += "\n" printables = "" - # Handle leftovers when the lenght is not mutiple of width + # Handle leftovers when the length is not mutiple of width if printables: padding = width - len(printables) - output += " " * (padding) + output += " " * padding output += printables - output += " " * (padding) + output += " " * padding return output @@ -132,7 +132,7 @@ def display_disassembly(disasm: interfaces.renderers.Disassembly) -> str: for i in disasm_types[disasm.architecture].disasm( disasm.data, disasm.offset ): - output += f"\n0x{i.address:x}:\t{i.mnemonic}\t{i.op_str}" + output += f"\n{i.address:#x}:\t{i.mnemonic}\t{i.op_str}" return output return QuickTextRenderer._type_renderers[bytes](disasm.data) @@ -342,7 +342,7 @@ class PrettyTextRenderer(CLIRenderer): column_separator = " | " tree_indent_column = "".join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(20) + random.choices(string.ascii_uppercase + string.digits, k=20) ) max_column_widths = dict( [(column.name, len(column.name)) for column in grid.columns] From bb1ff69e426ad688ea116bcdab2f1a9c77a182e6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 16 Dec 2024 19:25:24 +1100 Subject: [PATCH 662/979] linux: dentry: Fix dentry type support for kernels pre-3.19 --- .../framework/symbols/linux/extensions/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 927f767e2..829622154 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1107,9 +1107,16 @@ class dentry(objects.StructType): walk_member = "d_sib" list_head_member = self.d_children elif self.has_member("d_child") and self.has_member("d_subdirs"): - # 2.5.0 <= kernels < 6.8 + # 3.19.0 <= kernels < 6.8 walk_member = "d_child" list_head_member = self.d_subdirs + elif self.has_member("d_u") and self.has_member("d_subdirs"): + # kernels < 3.19 + + # Actually, 'd_u.d_child' but to_list() doesn't support something like that. + # Since, it's an union, everything is at the same offset than 'd_u'. + walk_member = "d_u" + list_head_member = self.d_subdirs else: raise exceptions.VolatilityException("Unsupported dentry type") From 3b0f0915c7fd24512d12603ab989a53f6ac68928 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 16 Dec 2024 19:36:54 +1100 Subject: [PATCH 663/979] linux: page_cache: add testcase for page_cache.files plugin --- test/test_volatility.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index f7cb23e93..b5910e1c8 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -632,6 +632,26 @@ def test_linux_vmayarascan_yara_string(image, volatility, python): assert rc == 0 +def test_linux_page_cache_files(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.pagecache.Files", + image, + volatility, + python, + pluginargs=["--find", "/etc/passwd"], + ) + out = out.lower() + + assert out.count(b"\n") > 4 + + # inode_num inode_addr ... file_path + assert re.search( + rb"146829\s0x88001ab5c270.*?/etc/passwd", + out, + ) + assert rc == 0 + + # MAC From 02b11b44a28634c230b0676bf4feafe37fac6dff Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:59:58 +0000 Subject: [PATCH 664/979] Remove use of int function after math.ceil Return type of math.ceil is already an int. --- volatility3/framework/layers/intel.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index c30ae48a8..846f246dc 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -73,8 +73,8 @@ class Intel(linear.LinearlyMappedLayer): ) # These can vary depending on the type of space - self._index_shift = int( - math.ceil(math.log2(struct.calcsize(self._entry_format))) + self._index_shift = math.ceil( + math.log2(struct.calcsize(self._entry_format)) ) @classproperty @@ -125,7 +125,6 @@ class Intel(linear.LinearlyMappedLayer): high_mask = (1 << (high_bit + 1)) - 1 low_mask = (1 << low_bit) - 1 mask = high_mask ^ low_mask - # print(high_bit, low_bit, bin(mask), bin(value)) return value & mask @staticmethod @@ -147,7 +146,7 @@ class Intel(linear.LinearlyMappedLayer): return self._mask(addr, self._maxvirtaddr, 0) + self._canonical_prefix def decanonicalize(self, addr: int) -> int: - """Removes canonicalization to ensure an adress fits within the correct range if it has been canonicalized + """Removes canonicalization to ensure an address fits within the correct range if it has been canonicalized This will produce an address outside the range if the canonicalization is incorrect """ From b37923c183bec9a6381d5d592b405a9fcf51887f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 16 Dec 2024 17:08:37 +0000 Subject: [PATCH 665/979] Remove use of int function after math.ceil Return type of math.ceil is already an int. --- volatility3/framework/layers/intel.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 846f246dc..7918ebed4 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -73,9 +73,7 @@ class Intel(linear.LinearlyMappedLayer): ) # These can vary depending on the type of space - self._index_shift = math.ceil( - math.log2(struct.calcsize(self._entry_format)) - ) + self._index_shift = math.ceil(math.log2(struct.calcsize(self._entry_format))) @classproperty @functools.lru_cache() From 267c5a60c3b99da48cb0ead9c9d1492b857ea340 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 17 Dec 2024 10:05:38 +1100 Subject: [PATCH 666/979] Linux: PageCache: Remove unused variable --- volatility3/framework/plugins/linux/pagecache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 6d2607ada..46b24b27a 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -520,7 +520,6 @@ class InodePages(plugins.PluginInterface): yield 0, fields if self.config["dump"]: - filename = self.config["dump"] open_method = self.open inode_address = inode.vol.offset filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") From 7299f925dcd8a7ae1a00b47cf4846fc683c8e5ac Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 16 Dec 2024 17:58:53 -0600 Subject: [PATCH 667/979] Windows: Handles - major version bump Bumps the major version in plugin + dependences after removal of a publicly exposed instance method. --- volatility3/framework/plugins/windows/callbacks.py | 2 +- volatility3/framework/plugins/windows/dumpfiles.py | 2 +- volatility3/framework/plugins/windows/handles.py | 2 +- volatility3/framework/plugins/windows/poolscanner.py | 2 +- volatility3/framework/plugins/windows/psxview.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 562846def..414a8814a 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -48,7 +48,7 @@ class Callbacks(interfaces.plugins.PluginInterface): name="driverirp", plugin=driverirp.DriverIrp, version=(1, 0, 0) ), requirements.PluginRequirement( - name="handles", plugin=handles.Handles, version=(1, 0, 0) + name="handles", plugin=handles.Handles, version=(2, 0, 0) ), ] diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 33d2d0d41..bc554c0bf 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -69,7 +69,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): name="pslist", component=pslist.PsList, version=(2, 0, 0) ), requirements.VersionRequirement( - name="handles", component=handles.Handles, version=(1, 0, 0) + name="handles", component=handles.Handles, version=(2, 0, 0) ), ] diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 977a8c804..a3067b09f 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -18,7 +18,7 @@ class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 3) + _version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 1f70cfb8c..8c56d202d 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -139,7 +139,7 @@ class PoolScanner(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="handles", plugin=handles.Handles, version=(1, 0, 0) + name="handles", plugin=handles.Handles, version=(2, 0, 0) ), ] diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index a8d185a2c..6c845bf81 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -62,7 +62,7 @@ class PsXView(plugins.PluginInterface): name="thrdscan", component=thrdscan.ThrdScan, version=(1, 0, 0) ), requirements.VersionRequirement( - name="handles", component=handles.Handles, version=(1, 0, 0) + name="handles", component=handles.Handles, version=(2, 0, 0) ), requirements.BooleanRequirement( name="physical-offsets", From fd9d3ec04c967c5e1a16d7735cf7f064d2b82a47 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 16 Dec 2024 18:17:42 -0600 Subject: [PATCH 668/979] Windows: Typing - Remove type casts, add signature Removes the needless `int` casts, and adds the return type to the `get_session_id` method signature. --- volatility3/framework/plugins/windows/modules.py | 16 +++++++--------- .../symbols/windows/extensions/__init__.py | 4 ++-- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 5ff252074..b9d754328 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -2,14 +2,14 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import List, Iterable, Generator +from typing import Generator, Iterable, List -from volatility3.framework import exceptions, interfaces, constants, renderers +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import pslist, pedump +from volatility3.plugins.windows import pedump, pslist vollog = logging.getLogger(__name__) @@ -183,12 +183,10 @@ class Modules(interfaces.plugins.PluginInterface): # stores its session ID at offset 8 as an unsigned long, we # create an unsigned long at that offset and use that # instead. - session_id = int( - context.object( - layer_name=layer_name, - object_type=symbol_table + constants.BANG + "unsigned long", - offset=proc.Session + 8, - ) + session_id = context.object( + layer_name=layer_name, + object_type=symbol_table + constants.BANG + "unsigned long", + offset=proc.Session + 8, ) if session_id in seen_ids: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 78f59fc0c..12f84ca90 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -797,7 +797,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): return renderers.UnreadableValue() - def get_session_id(self): + def get_session_id(self) -> Union[int, interfaces.renderers.BaseAbsentValue]: try: if self.has_member("Session"): if self.Session == 0: @@ -836,7 +836,7 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): offset=self.Session + 8, absolute=True, ) - return int(session_id) + return session_id except exceptions.InvalidAddressException: vollog.log( From 02f17af8a632fb70d1152c66ff3847b2fb921033 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 17 Dec 2024 12:09:29 +1100 Subject: [PATCH 669/979] linux: fix task parent pid in several plugins. It also adds a method to get the correct one in a unified way from the task object extension --- .../framework/plugins/linux/capabilities.py | 4 ++-- volatility3/framework/plugins/linux/envars.py | 12 ++---------- volatility3/framework/plugins/linux/psaux.py | 10 ++-------- volatility3/framework/plugins/linux/pslist.py | 4 ++-- volatility3/framework/plugins/linux/psscan.py | 7 ++----- volatility3/framework/plugins/linux/pstree.py | 6 +++--- .../framework/symbols/linux/extensions/__init__.py | 14 ++++++++++++++ 7 files changed, 27 insertions(+), 30 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index a8a8fb1fa..a06ee4c1b 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -50,7 +50,7 @@ class Capabilities(plugins.PluginInterface): """Lists process capabilities""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -136,7 +136,7 @@ class Capabilities(plugins.PluginInterface): comm=utility.array_to_string(task.comm), pid=int(task.pid), tgid=int(task.tgid), - ppid=int(task.parent.pid), + ppid=int(task.get_parent_pid()), euid=int(task.cred.euid), ) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 22aba6408..a3eb21cf5 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -17,7 +17,7 @@ class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls): @@ -48,15 +48,7 @@ class Envars(plugins.PluginInterface): # get process name as string name = utility.array_to_string(task.comm) - - # try and get task parent - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." - ) - ppid = 0 + ppid = task.get_parent_pid() # kernel threads never have an mm as they do not have userland mappings try: diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index 5467c3b4c..5a4d75c70 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -15,7 +15,7 @@ class PsAux(plugins.PluginInterface): """Lists processes with their command line arguments""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls): @@ -98,14 +98,8 @@ class PsAux(plugins.PluginInterface): # walk the process list and report the arguments for task in tasks: pid = task.pid - - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - ppid = 0 - + ppid = task.get_parent_pid() name = utility.array_to_string(task.comm) - args = self._get_command_line_args(task, name) yield (0, (pid, ppid, name, args)) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 6460462a7..edfc0688c 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -18,7 +18,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) + _version = (3, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -95,7 +95,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ pid = task.tgid tid = task.pid - ppid = task.parent.tgid if task.parent else 0 + ppid = task.get_parent_pid() name = utility.array_to_string(task.comm) start_time = task.get_create_time() if decorate_comm: diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 40784a647..55e3778ab 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -28,7 +28,7 @@ class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -52,10 +52,7 @@ class PsScan(interfaces.plugins.PluginInterface): """ pid = task.tgid tid = task.pid - ppid = 0 - - if task.parent.is_readable(): - ppid = task.parent.tgid + ppid = task.get_parent_pid() name = utility.array_to_string(task.comm) exit_state = DescExitStateEnum(task.exit_state).name diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index 9dc5ea3cc..7ea9df3d6 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -13,7 +13,7 @@ class PsTree(interfaces.plugins.PluginInterface): ID.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 1, 0) @classmethod def get_requirements(cls): @@ -56,9 +56,9 @@ class PsTree(interfaces.plugins.PluginInterface): seen = set([pid]) level = 0 proc = self._tasks.get(pid) - while proc and proc.parent and proc.parent.pid not in seen: + while proc and proc.get_parent_pid() not in seen: if proc.is_thread_group_leader: - parent_pid = proc.parent.pid + parent_pid = proc.get_parent_pid() else: parent_pid = proc.tgid diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 927f767e2..f27306e67 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -635,6 +635,20 @@ class task_struct(generic.GenericIntelProcess): # root time namespace, not within the task's own time namespace return boottime + task_start_time_timedelta + def get_parent_pid(self) -> int: + """Returns the parent process ID (PPID) + + This method replicates the Linux kernel's `getppid` syscall behavior. + Avoid using `task.parent`; instead, use this function for accurate results. + """ + + if self.real_parent and self.real_parent.is_readable(): + ppid = self.real_parent.pid + else: + ppid = 0 + + return ppid + class fs_struct(objects.StructType): def get_root_dentry(self): From 74ff42a12d2665d05e64c7c71beb2bec5f4c9333 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 17 Dec 2024 13:28:57 +1100 Subject: [PATCH 670/979] Fix ProducerMetadata class bug introduced in #1369 --- volatility3/framework/symbols/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 73ad2cf21..7e069e518 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -27,7 +27,7 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): @property def version(self) -> Optional[Tuple[int]]: """Returns the version of the ISF file producer""" - version = self.version_string() + version = self.version_string if not version: return None if all(x in "0123456789." for x in version): From 0255151ef508f5c4a1e75c71dfafa99218e15b05 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Tue, 17 Dec 2024 12:15:46 +0800 Subject: [PATCH 671/979] Fix: Error early if no inodes are found in linux.pagecache.InodePages plugin --- volatility3/framework/plugins/linux/pagecache.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 46b24b27a..7dbf074b3 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -483,6 +483,9 @@ class InodePages(plugins.PluginInterface): if inode_in.path == self.config["find"]: inode = inode_in.inode break # Only the first match + else: + vollog.error("Unable to find inode with path %s", self.config["find"]) + return None elif self.config["inode"]: inode = vmlinux.object("inode", self.config["inode"], absolute=True) From 054f0496c123ec074a134ab9e75416fdef15e1c4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 17:55:52 +0000 Subject: [PATCH 672/979] Windows: Cannot use capstone typing information if capstone didn'tr import --- volatility3/framework/plugins/windows/indirect_system_calls.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py index 1a5eb317f..c4f3f6d28 100644 --- a/volatility3/framework/plugins/windows/indirect_system_calls.py +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -73,8 +73,7 @@ class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): @staticmethod def _indirect_syscall_block_target( - proc_layer: interfaces.layers.DataLayerInterface, inst: capstone._cs_insn - ) -> Optional[int]: + proc_layer: interfaces.layers.DataLayerInterface, inst) -> Optional[int]: """ This function determines the address of a jmp in the following form: From 246d19c0fadbb986e4ec4c019505bed4d32b6359 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:14:29 +0000 Subject: [PATCH 673/979] Windows: Fix black issue --- volatility3/framework/plugins/windows/indirect_system_calls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py index c4f3f6d28..f09851b30 100644 --- a/volatility3/framework/plugins/windows/indirect_system_calls.py +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -73,7 +73,8 @@ class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): @staticmethod def _indirect_syscall_block_target( - proc_layer: interfaces.layers.DataLayerInterface, inst) -> Optional[int]: + proc_layer: interfaces.layers.DataLayerInterface, inst + ) -> Optional[int]: """ This function determines the address of a jmp in the following form: From c45beb3ebe7feaec42567eb8ef3dd665e15db3ae Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:12:19 +0000 Subject: [PATCH 674/979] Automagic: Fixes #1417 --- volatility3/framework/automagic/symbol_cache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index e38771f79..2c9883c7d 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -299,6 +299,13 @@ class SqliteCache(CacheManagerInterface): This also updates remote locations based on a cache timeout. """ + if progress_callback is None: + + def dummy_progress(*args, **kargs) -> None: + return None + + progress_callback = dummy_progress + on_disk_locations = set( [ filename From f0f3bb65581e433cc7b44c2e877a1e95c927e17e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:31:41 +0000 Subject: [PATCH 675/979] Core: Start to fix up the typing in ModuleCollection Fixes #1418 --- volatility3/framework/contexts/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 6961d9328..1a55656b3 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -11,7 +11,7 @@ without them interfering with each other. import functools import hashlib import logging -from typing import Callable, Iterable, List, Optional, Set, Tuple, Union +from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, interfaces, symbols, exceptions from volatility3.framework.objects import templates @@ -386,10 +386,9 @@ class ModuleCollection(interfaces.context.ModuleContainer): """Class to contain a collection of SizedModules and reason about their contents.""" - def __init__( - self, modules: Optional[List[interfaces.context.ModuleInterface]] = None - ) -> None: + def __init__(self, modules: Optional[List[SizedModule]] = None) -> None: self._prefix_count = {} + self._modules: Dict[str, SizedModule] = {} super().__init__(modules) def deduplicate(self) -> "ModuleCollection": @@ -402,9 +401,9 @@ class ModuleCollection(interfaces.context.ModuleContainer): new_modules = [] seen: Set[str] = set() for mod in self._modules: - if mod.hash not in seen or mod.size == 0: + if self._modules[mod].hash not in seen or self._modules[mod].size == 0: new_modules.append(mod) - seen.add(mod.hash) # type: ignore # FIXME: mypy #5107 + seen.add(self._modules[mod].hash) return ModuleCollection(new_modules) def free_module_name(self, prefix: str = "module") -> str: From a0b169cf6b1f0a5d8ced886ff855e7ad7dd791c0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:34:19 +0000 Subject: [PATCH 676/979] This PR does not strictly change any interfaces, just the inner workings of a function. --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 55ef19e4b..2ea034176 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 12 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From 1b8f831fda1fc0d47eaf81144dec81354dca8490 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:37:39 +0000 Subject: [PATCH 677/979] Core: Also fix up the interface to match the concrete classes --- volatility3/framework/interfaces/context.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 8b5e816e8..e85429732 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -295,6 +295,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): def has_enumeration(self, name: str) -> bool: """Determines whether an enumeration is present in the module's symbol table.""" + @property def symbols(self) -> List: """Lists the symbols contained in the symbol table for this module""" From 9ef90c2091a5b838817fc9251251b74f34dccf7e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 17 Dec 2024 13:14:33 -0600 Subject: [PATCH 678/979] Windows: PeDump - use contextmanager Several tools, including pyright and PyCharm, report that `file_handle` may be an unbound local. Regardless of whether or not this is likely to happen in practice, it makes sense to just use a `ContextManager` here anyway, since `FileHandlerInterface` implements it. --- .../framework/plugins/windows/pedump.py | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 858d0615a..5b4bb07d7 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -64,30 +64,27 @@ class PEDump(interfaces.plugins.PluginInterface): """ Returns the filename of the dump file or None """ - try: - file_handle = open_method(file_name) + with open_method(file_name) as file_handle: + try: + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base, + layer_name=layer_name, + ) - dos_header = context.object( - pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset=base, - layer_name=layer_name, - ) + for offset, data in dos_header.reconstruct(): + file_handle.seek(offset) + file_handle.write(data) + except ( + IOError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: + vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") + return None - for offset, data in dos_header.reconstruct(): - file_handle.seek(offset) - file_handle.write(data) - except ( - IOError, - exceptions.VolatilityException, - OverflowError, - ValueError, - ) as excp: - vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") - return None - finally: - file_handle.close() - - return file_handle.preferred_filename + return file_handle.preferred_filename @classmethod def dump_ldr_entry( From 24e1904376aa501e5ec4b240f2cbe6fb23267ebe Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 19:22:35 +0000 Subject: [PATCH 679/979] Modify using as a library documentation Tiny changes. --- doc/source/using-as-a-library.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index 55b77e90a..144cae644 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -21,7 +21,7 @@ Creating a context First we make sure the volatility framework works the way we expect it (and is the version we expect). The versioning used is semantic versioning, meaning any version with the same major number and a higher or equal minor number will satisfy the requirement. An example is below since the CLI doesn't need any of the features -from version 1.1: +from version 1.1 or later: :: @@ -139,7 +139,7 @@ A suitable list of automagics for a particular plugin (based on operating system This will take the plugin module, extract the operating system (first level of the hierarchy) and then return just the automagics which apply to the operating system. Each automagic can exclude itself from being used for specific -operating systems, so that for example an automagic designed for linux is not used for windows or mac plugins. +operating systems, such that an automagic designed for linux is not used for windows or mac plugins. These automagics can then be run by providing the list, the context, the plugin to be run, the hierarchy name that the plugin will be constructed on ('plugins' by default) and a progress_callback. This is a callable which takes @@ -166,7 +166,7 @@ to `plugins` and then the name of the class itself): unsatisfied = plugin.unsatisfied(context, plugin_config_path) If unsatisfied is an empty list, then the plugin has been given everything it requires. If not, it will be a -dictionary of the hierarchy paths and their associated requirements that weren't satisfied. +dict of the hierarchy paths and their associated requirements that weren't satisfied. The plugin can then be instantiated with the context (containing the plugin's configuration) and the path that the plugin can find its configuration at. This configuration path only needs to be a unique value to identify where the From b04ca88d0754bfc2e79fbe70c9280eba344d8375 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 17 Dec 2024 13:28:37 -0600 Subject: [PATCH 680/979] Windows Extensions: Fixes type-hint on list_timers This method is incorrectly type-hinted as returning a `Tuple` when it should be returning an instance of the `KTIMER` extension class. This leaves the version number as is since it only updates the type-hint, but let me know if that's incorrect and we need to bump it. --- volatility3/framework/plugins/windows/timers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index d49c28784..fad25df72 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -14,7 +14,7 @@ from volatility3.framework import ( ) from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints -from volatility3.framework.symbols.windows import versions +from volatility3.framework.symbols.windows import versions, extensions from volatility3.plugins.windows import ssdt, kpcrs vollog = logging.getLogger(__name__) @@ -49,7 +49,7 @@ class Timers(interfaces.plugins.PluginInterface): kernel_module_name: str, layer_name: str, symbol_table: str, - ) -> Iterable[Tuple[str, int, str]]: + ) -> Iterable[extensions.KTIMER]: """Lists all kernel timers. Args: From c134dcd64306e3c5f355f97d4e117d34c3e76f33 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 17 Dec 2024 13:55:34 -0600 Subject: [PATCH 681/979] Windows Timers: Bump patch version --- volatility3/framework/plugins/windows/timers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index fad25df72..8bd7c8eb4 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -24,7 +24,7 @@ class Timers(interfaces.plugins.PluginInterface): """Print kernel timers and associated module DPCs""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From d76bf96f36a60a4c17aa51fa9033b80836b10812 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:25:10 +0000 Subject: [PATCH 682/979] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/framework/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 23ea745de..61ad787a1 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -64,8 +64,8 @@ def require_interface_version(*args) -> None: if args[1] > interface_version()[1]: raise RuntimeError( "Framework interface version {} is an older revision than the required version {}".format( - ".".join([str(x) for x in interface_version()[0:2]]), - ".".join([str(x) for x in args[0:2]]), + ".".join(str(x) for x in interface_version()[0:2]), + ".".join(str(x) for x in args[0:2]), ) ) From 60ca99710728fc923420b8e7c64b8b8dfa60e1a2 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:40:54 +0000 Subject: [PATCH 683/979] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 55ef19e4b..8c4856f0b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -5,7 +5,7 @@ VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( - ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + ".".join(str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]) + VERSION_SUFFIX ) """The canonical version of the volatility3 package""" From c8e2347a0f1577f349cbc212821e7b3122cc60e1 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:47:36 +0000 Subject: [PATCH 684/979] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/framework/plugins/windows/getservicesids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index 9b20ed2d0..12fc44682 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -26,7 +26,7 @@ def createservicesid(svc) -> str: ## The use of struct here is OK. It doesn't make much sense ## to leverage obj.Object inside this loop. dec.append(struct.unpack(" Date: Tue, 17 Dec 2024 20:48:42 +0000 Subject: [PATCH 685/979] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/framework/plugins/linux/check_creds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 0857576d5..4916b67d2 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -55,7 +55,7 @@ class Check_creds(interfaces.plugins.PluginInterface): for cred_addr, pids in creds.items(): if len(pids) > 1: - pid_str = ", ".join([str(pid) for pid in pids]) + pid_str = ", ".join(str(pid) for pid in pids) fields = [ format_hints.Hex(cred_addr), From eeb7cf71317bb5a195172d756d44ec11916b84a9 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:50:38 +0000 Subject: [PATCH 686/979] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/cli/volshell/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 82c470e1a..d02d054ee 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -203,7 +203,7 @@ class Volshell(interfaces.plugins.PluginInterface): connector = " " if chunk_size < 2: connector = "" - ascii_data = connector.join([self._ascii_bytes(x) for x in valid_data]) + ascii_data = connector.join(self._ascii_bytes(x) for x in valid_data) print(hex(offset), " ", hex_data, " ", ascii_data) offset += 16 From ae1079f923a4b5f36aa394805c26805e0b31408e Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:52:29 +0000 Subject: [PATCH 687/979] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/framework/configuration/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index f130f9544..a52d7ff27 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -532,7 +532,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): if version is None: raise TypeError("Version cannot be None") if description is None: - description = f"Version {'.'.join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" + description = f"Version {'.'.join(str(x) for x in version)} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) From 8e9b719a1641a8c72fcbae61c313061fb60864f0 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 13:23:11 +0100 Subject: [PATCH 688/979] use ruff for linting and enforce linting via ci --- .github/workflows/ruff.yaml | 15 +++++++++++++++ pyproject.toml | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 .github/workflows/ruff.yaml diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml new file mode 100644 index 000000000..77e3aa864 --- /dev/null +++ b/.github/workflows/ruff.yaml @@ -0,0 +1,15 @@ +--- +name: Ruff + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/ruff-action@v1 + with: + args: check + src: "." diff --git a/pyproject.toml b/pyproject.toml index 9b4b8d485..7035f7a15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,24 @@ show_traceback = true [tool.mypy.overrides] ignore_missing_imports = true +[tool.ruff] +line-length = 88 +target-version = "py38" + +[tool.ruff.lint] +select = [ + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "G", # flake8-logging-format + "PIE", # flake8-pie + "UP", # pyupgrade +] + +ignore = [ + "E501", # ignore due to conflict with formatter +] + [build-system] requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" From ad7daafc508da70e140ccb4babb3c53b9a2085d7 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 16:45:18 +0100 Subject: [PATCH 689/979] linted with ruff (`ruff check . --fix`) --- development/compare-vol.py | 2 +- development/pdbparse-to-json.py | 5 +- development/schema_validate.py | 4 +- volatility3/cli/__init__.py | 22 +++----- volatility3/cli/text_filter.py | 2 +- volatility3/cli/text_renderer.py | 2 +- volatility3/cli/volargparse.py | 2 +- volatility3/cli/volshell/__init__.py | 6 +-- volatility3/cli/volshell/generic.py | 1 - volatility3/framework/__init__.py | 10 ++-- volatility3/framework/automagic/linux.py | 4 +- volatility3/framework/automagic/mac.py | 2 +- volatility3/framework/automagic/pdbscan.py | 4 +- .../framework/automagic/symbol_cache.py | 9 +--- .../framework/automagic/symbol_finder.py | 2 +- .../framework/configuration/requirements.py | 4 +- volatility3/framework/contexts/__init__.py | 2 +- .../framework/interfaces/configuration.py | 10 ++-- volatility3/framework/interfaces/layers.py | 7 +-- volatility3/framework/interfaces/plugins.py | 2 +- volatility3/framework/interfaces/renderers.py | 12 +++-- volatility3/framework/interfaces/symbols.py | 3 +- volatility3/framework/layers/crash.py | 9 ++-- volatility3/framework/layers/intel.py | 19 +++---- volatility3/framework/layers/leechcore.py | 2 +- volatility3/framework/layers/qemu.py | 2 +- volatility3/framework/layers/registry.py | 8 +-- volatility3/framework/layers/resources.py | 2 +- .../framework/layers/scanners/multiregexp.py | 2 +- volatility3/framework/objects/utility.py | 4 +- volatility3/framework/plugins/layerwriter.py | 2 +- .../framework/plugins/linux/check_idt.py | 2 +- .../framework/plugins/linux/check_syscall.py | 2 +- .../framework/plugins/linux/pagecache.py | 2 +- volatility3/framework/plugins/linux/proc.py | 4 +- volatility3/framework/plugins/linux/psscan.py | 2 +- .../framework/plugins/mac/proc_maps.py | 4 +- volatility3/framework/plugins/timeliner.py | 4 +- .../framework/plugins/windows/cmdline.py | 4 +- .../framework/plugins/windows/consoles.py | 22 ++------ .../framework/plugins/windows/dlllist.py | 15 ++---- .../framework/plugins/windows/dumpfiles.py | 8 +-- .../framework/plugins/windows/envars.py | 6 +-- .../plugins/windows/getservicesids.py | 2 +- .../framework/plugins/windows/getsids.py | 6 +-- .../plugins/windows/hollowprocesses.py | 37 +++++-------- volatility3/framework/plugins/windows/iat.py | 8 +-- .../framework/plugins/windows/malfind.py | 8 +-- .../framework/plugins/windows/memmap.py | 10 +--- .../framework/plugins/windows/modules.py | 4 +- .../framework/plugins/windows/netscan.py | 31 ++--------- .../framework/plugins/windows/netstat.py | 8 +-- .../framework/plugins/windows/pe_symbols.py | 2 +- .../framework/plugins/windows/pedump.py | 54 +++++++++---------- .../framework/plugins/windows/privileges.py | 2 +- .../framework/plugins/windows/psxview.py | 1 - .../plugins/windows/registry/hivelist.py | 26 +++------ .../plugins/windows/registry/userassist.py | 6 +-- .../plugins/windows/skeleton_key_check.py | 8 +-- .../framework/plugins/windows/strings.py | 4 +- .../framework/plugins/windows/svclist.py | 4 +- .../framework/plugins/windows/svcscan.py | 14 ++--- .../framework/plugins/windows/thrdscan.py | 2 +- .../framework/plugins/windows/threads.py | 2 +- .../framework/plugins/windows/timers.py | 2 +- .../plugins/windows/unhooked_system_calls.py | 2 +- .../framework/plugins/windows/vadinfo.py | 4 +- .../framework/plugins/windows/verinfo.py | 4 +- volatility3/framework/renderers/__init__.py | 8 +-- volatility3/framework/symbols/intermed.py | 2 +- .../framework/symbols/linux/__init__.py | 2 +- .../symbols/linux/extensions/__init__.py | 4 +- .../symbols/windows/extensions/__init__.py | 8 +-- .../symbols/windows/extensions/consoles.py | 2 +- .../symbols/windows/extensions/mbr.py | 7 +-- .../symbols/windows/extensions/network.py | 8 ++- .../symbols/windows/extensions/pe.py | 4 +- .../symbols/windows/extensions/pool.py | 6 +-- .../symbols/windows/extensions/registry.py | 4 +- .../framework/symbols/windows/pdbconv.py | 6 +-- .../framework/symbols/windows/pdbutil.py | 6 +-- .../plugins/windows/registry/certificates.py | 2 +- volatility3/plugins/windows/statistics.py | 4 +- volatility3/schemas/__init__.py | 6 +-- 84 files changed, 191 insertions(+), 381 deletions(-) diff --git a/development/compare-vol.py b/development/compare-vol.py index d0d834038..1074c5d9c 100644 --- a/development/compare-vol.py +++ b/development/compare-vol.py @@ -101,7 +101,7 @@ class Volatility2Test(VolatilityTest): print(f" Tested volatility2 imageinfo with image {image.filepath}: {end_time - start_time}") with open(os.path.join(self.output_directory, f'vol2_imageinfo_{image_hash}_stdout'), "wb") as f: f.write(vol2_completed.stdout) - image.vol2_profile = re.search(b"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout)[1] + image.vol2_profile = re.search(rb"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout)[1] class RekallTest(VolatilityTest): diff --git a/development/pdbparse-to-json.py b/development/pdbparse-to-json.py index 819e44e15..6eb265227 100644 --- a/development/pdbparse-to-json.py +++ b/development/pdbparse-to-json.py @@ -145,8 +145,7 @@ class PDBConvertor: """Generates the metadata necessary for this object""" dbg = self._pdb.STREAM_DBI last_bytes = str(binascii.hexlify(self._pdb.STREAM_PDB.GUID.Data4), 'ascii')[-16:] - guidstr = u'{:08x}{:04x}{:04x}{}'.format(self._pdb.STREAM_PDB.GUID.Data1, self._pdb.STREAM_PDB.GUID.Data2, - self._pdb.STREAM_PDB.GUID.Data3, last_bytes) + guidstr = f'{self._pdb.STREAM_PDB.GUID.Data1:08x}{self._pdb.STREAM_PDB.GUID.Data2:04x}{self._pdb.STREAM_PDB.GUID.Data3:04x}{last_bytes}' pdb_data = { "GUID": guidstr.upper(), "age": self._pdb.STREAM_PDB.Age, @@ -195,7 +194,7 @@ class PDBConvertor: try: sects = self._pdb.STREAM_SECT_HDR_ORIG.sections omap = self._pdb.STREAM_OMAP_FROM_SRC - except AttributeError as e: + except AttributeError: # In this case there is no OMAP, so we use the given section # headers and use the identity function for omap.remap sects = self._pdb.STREAM_SECT_HDR.sections diff --git a/development/schema_validate.py b/development/schema_validate.py index 0908e934f..031039e38 100644 --- a/development/schema_validate.py +++ b/development/schema_validate.py @@ -28,7 +28,7 @@ if __name__ == '__main__': schema = None if args.schema: - with open(os.path.abspath(args.schema), 'r') as s: + with open(os.path.abspath(args.schema)) as s: schema = json.load(s) failures = [] @@ -36,7 +36,7 @@ if __name__ == '__main__': try: if os.path.exists(filename): print(f"[?] Validating file: {filename}") - with open(filename, 'r') as t: + with open(filename) as t: test = json.load(t) if args.schema: diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 901f299a8..cf1335443 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -57,7 +57,7 @@ formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) -class PrintedProgress(object): +class PrintedProgress: """A progress handler that prints the progress value and the description onto the command line.""" @@ -126,9 +126,7 @@ class CommandLine: "--help", action="help", default=argparse.SUPPRESS, - help="Show this help message and exit, for specific plugin options use '{} --help'".format( - parser.prog - ), + help=f"Show this help message and exit, for specific plugin options use '{parser.prog} --help'", ) parser.add_argument( "-c", @@ -360,9 +358,7 @@ class CommandLine: subparser = parser.add_subparsers( title="Plugins", dest="plugin", - description="For plugin specific options, run '{} --help'".format( - self.CLI_NAME - ), + description=f"For plugin specific options, run '{self.CLI_NAME} --help'", action=volargparse.HelpfulSubparserAction, metavar="PLUGIN", ) @@ -416,7 +412,7 @@ class CommandLine: # UI fills in the config, here we load it from the config file and do it before we process the CL parameters if args.config: - with open(args.config, "r") as f: + with open(args.config) as f: json_val = json.load(f) ctx.config.splice( plugin_config_path, @@ -722,9 +718,7 @@ class CommandLine: if isinstance(requirement, requirements.ListRequirement): if not isinstance(value, list): raise TypeError( - "Configuration for ListRequirement was not a list: {}".format( - requirement.name - ) + f"Configuration for ListRequirement was not a list: {requirement.name}" ) value = [requirement.element_type(x) for x in value] if not inspect.isclass(configurables_list[configurable]): @@ -797,7 +791,7 @@ class CommandLine: fd, self._name = tempfile.mkstemp( suffix=".vol3", prefix="tmp_", dir=output_dir ) - self._file = io.open(fd, mode="w+b") + self._file = open(fd, mode="w+b") CLIFileHandler.__init__(self, filename) for item in dir(self._file): if not item.startswith("_") and item not in ( @@ -870,9 +864,7 @@ class CommandLine: requirement, interfaces.configuration.RequirementInterface ): raise TypeError( - "Plugin contains requirements that are not RequirementInterfaces: {}".format( - configurable.__name__ - ) + f"Plugin contains requirements that are not RequirementInterfaces: {configurable.__name__}" ) if isinstance(requirement, interfaces.configuration.SimpleTypeRequirement): additional["type"] = requirement.instance_type diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 3d69934e9..955d647f5 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -76,7 +76,7 @@ class ColumnFilter: if self.regex: return re.search(self.pattern, f"{item}") return self.pattern in f"{item}" - except IOError: + except OSError: return False def found(self, row: List[Any]) -> bool: diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 408a562d8..937ba4ef4 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -467,7 +467,7 @@ class JsonRenderer(CLIRenderer): def output_result(self, outfd, result): """Outputs the JSON data to a file in a particular format""" - outfd.write("{}\n".format(json.dumps(result, indent=2, sort_keys=True))) + outfd.write(f"{json.dumps(result, indent=2, sort_keys=True)}\n") def render(self, grid: interfaces.renderers.TreeGrid): outfd = sys.stdout diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index fd61ddce0..dce9cafa6 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -5,7 +5,7 @@ import argparse import gettext import re -from typing import List, Optional, Sequence, Any, Union +from typing import Optional, Sequence, Any, Union # This effectively overrides/monkeypatches the core argparse module to provide more helpful output around choices diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index e9d3fda08..0affe5d59 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -282,9 +282,7 @@ class VolShell(cli.CommandLine): for plugin in volshell_plugin_list: subparser = parser.add_argument_group( title=plugin.capitalize(), - description="Configuration options based on {} options".format( - plugin.capitalize() - ), + description=f"Configuration options based on {plugin.capitalize()} options", ) self.populate_requirements_argparse(subparser, volshell_plugin_list[plugin]) configurables_list[plugin] = volshell_plugin_list[plugin] @@ -331,7 +329,7 @@ class VolShell(cli.CommandLine): # UI fills in the config, here we load it from the config file and do it before we process the CL parameters if args.config: - with open(args.config, "r") as f: + with open(args.config) as f: json_val = json.load(f) ctx.config.splice( plugin_config_path, diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 82c470e1a..65040bb2a 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -585,7 +585,6 @@ class NullFileHandler(io.BytesIO, interfaces.plugins.FileHandlerInterface): def writelines(self, lines: Iterable[bytes]): """Dummy method""" - pass def write(self, b: bytes): """Dummy method""" diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 23ea745de..e0f7c778a 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -56,9 +56,7 @@ def require_interface_version(*args) -> None: if len(args): if args[0] != interface_version()[0]: raise RuntimeError( - "Framework interface version {} is incompatible with required version {}".format( - interface_version()[0], args[0] - ) + f"Framework interface version {interface_version()[0]} is incompatible with required version {args[0]}" ) if len(args) > 1: if args[1] > interface_version()[1]: @@ -70,7 +68,7 @@ def require_interface_version(*args) -> None: ) -class NonInheritable(object): +class NonInheritable: def __init__(self, value: Any, cls: Type) -> None: self.default_value = value self.cls = cls @@ -187,9 +185,7 @@ def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str traceback.TracebackException.from_exception(e).format(chain=True) ) ) - vollog.debug( - "Failed to import module {} based on file: {}".format(module, path) - ) + vollog.debug(f"Failed to import module {module} based on file: {path}") failures.append(module) if not ignore_errors: raise diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index ef54a0aa5..6b58577a3 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -173,9 +173,7 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if aslr_shift & 0xFFF != 0 or kaslr_shift & 0xFFF != 0: continue vollog.debug( - "Linux ASLR shift values determined: physical {:0x} virtual {:0x}".format( - kaslr_shift, aslr_shift - ) + f"Linux ASLR shift values determined: physical {kaslr_shift:0x} virtual {aslr_shift:0x}" ) return kaslr_shift, aslr_shift diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 7c478b521..89dd5a187 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -184,7 +184,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = [int(x) for x in banner[22:].split(b".")[0:2]] + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) tmp_aslr_shift = offset - cls.virtual_to_physical_address( version_json_address diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 0b4f6c73a..729c48063 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -215,9 +215,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): return (virtual_layer_name, kvo, kernel) else: vollog.debug( - "Potential kernel_virtual_offset did not map to expected location: {}".format( - hex(kvo) - ) + f"Potential kernel_virtual_offset did not map to expected location: {hex(kvo)}" ) except exceptions.InvalidAddressException: vollog.debug( diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index e38771f79..9fad506ae 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -106,7 +106,6 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): def add_identifier(self, location: str, operating_system: str, identifier: str): """Adds an identifier to the store""" - pass def find_location( self, identifier: bytes, operating_system: Optional[str] @@ -120,18 +119,15 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: The location of the symbols file that matches the identifier """ - pass def get_local_locations(self) -> Iterable[str]: """Returns a list of all the local locations""" - pass def update(self): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. """ - pass def get_identifier_dictionary( self, operating_system: Optional[str] = None, local_only: bool = False @@ -145,15 +141,12 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): Returns: A dictionary of identifiers mapped to a location """ - pass def get_identifier(self, location: str) -> Optional[bytes]: """Returns an identifier based on a specific location or None""" - pass def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: """Returns all identifiers for a particular operating system""" - pass def get_location_statistics( self, location: str @@ -572,6 +565,6 @@ class RemoteIdentifierFormat: try: subrbf = RemoteIdentifierFormat(location) yield from subrbf.process(identifiers, operating_system) - except IOError: + except OSError: vollog.debug(f"Remote file not found: {location}") return identifiers diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 6d689e194..1d30f3f51 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -4,7 +4,7 @@ import logging import os -from typing import Any, Callable, Iterable, List, Optional, Tuple +from typing import Callable, List, Optional, Tuple from volatility3.framework import constants, interfaces, layers from volatility3.framework.automagic import symbol_cache diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index f130f9544..828c89daa 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -664,9 +664,7 @@ class ModuleRequirement( if value is not None: vollog.log( constants.LOGLEVEL_V, - "TypeError - Module Requirement only accepts string labels: {}".format( - repr(value) - ), + f"TypeError - Module Requirement only accepts string labels: {repr(value)}", ) return {config_path: self} diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 6961d9328..5111b168a 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -356,7 +356,7 @@ class SizedModule(Module): return size or 0 @property # type: ignore # FIXME: mypy #5107 - @functools.lru_cache() + @functools.lru_cache def hash(self) -> str: """Hashes the module for equality checks. diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index da0a4556c..cbbf7e342 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -94,7 +94,7 @@ class HierarchicalDict(collections.abc.Mapping): raise TypeError(f"Separator must be a one character string: {separator}") self._separator = separator self._data: Dict[str, ConfigSimpleType] = {} - self._subdict: Dict[str, "HierarchicalDict"] = {} + self._subdict: Dict[str, HierarchicalDict] = {} if isinstance(initial_dict, str): initial_dict = json.loads(initial_dict) if isinstance(initial_dict, dict): @@ -182,9 +182,7 @@ class HierarchicalDict(collections.abc.Mapping): else: if not isinstance(value, HierarchicalDict): raise TypeError( - "HierarchicalDicts can only store HierarchicalDicts within their structure: {}".format( - type(value) - ) + f"HierarchicalDicts can only store HierarchicalDicts within their structure: {type(value)}" ) self._subdict[key] = value @@ -498,9 +496,7 @@ class SimpleTypeRequirement(RequirementInterface): if not isinstance(value, self.instance_type): vollog.log( constants.LOGLEVEL_V, - "TypeError - {} requirements only accept {} type: {}".format( - self.name, self.instance_type.__name__, repr(value) - ), + f"TypeError - {self.name} requirements only accept {self.instance_type.__name__} type: {repr(value)}", ) return {config_path: self} return {} diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 78687d8d5..56798aca9 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -188,7 +188,6 @@ class DataLayerInterface( the object unreadable (exceptions will be thrown using a DataLayer after destruction) """ - pass @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -361,9 +360,7 @@ class DataLayerInterface( data += self.context.layers[layer_name].read(address, chunk_size) except exceptions.InvalidAddressException: vollog.debug( - "Invalid address in layer {} found scanning {} at address {:x}".format( - layer_name, self.name, address - ) + f"Invalid address in layer {layer_name} found scanning {self.name} at address {address:x}" ) if len(data) > scanner.chunk_size + scanner.overlap: @@ -721,7 +718,7 @@ class LayerContainer(collections.abc.Mapping): raise NotImplementedError("Cycle checking has not yet been implemented") -class DummyProgress(object): +class DummyProgress: """A class to emulate Multiprocessing/threading Value objects.""" def __init__(self) -> None: diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 74902636e..f763815a6 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -46,7 +46,7 @@ class FileHandlerInterface(io.RawIOBase): def preferred_filename(self, filename: str): """Sets the preferred filename""" if self.closed: - raise IOError("FileHandler name cannot be changed once closed") + raise OSError("FileHandler name cannot be changed once closed") if not isinstance(filename, str): raise TypeError("FileHandler preferred filenames must be strings") if os.path.sep in filename: diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index b13de1834..7105274c0 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -26,7 +26,11 @@ from typing import ( Union, ) -Column = NamedTuple("Column", [("name", str), ("type", Any)]) + +class Column(NamedTuple): + name: str + type: Any + RenderOption = Any @@ -98,11 +102,11 @@ class TreeNode(abc.Sequence, metaclass=ABCMeta): """ -class BaseAbsentValue(object): +class BaseAbsentValue: """Class that represents values which are not present for some reason.""" -class Disassembly(object): +class Disassembly: """A class to indicate that the bytes provided should be disassembled (based on the architecture)""" @@ -137,7 +141,7 @@ ColumnsType = List[Tuple[str, BaseTypes]] VisitorSignature = Callable[[TreeNode, _Type], _Type] -class TreeGrid(object, metaclass=ABCMeta): +class TreeGrid(metaclass=ABCMeta): """Class providing the interface for a TreeGrid (which contains TreeNodes) The structure of a TreeGrid is designed to maintain the structure of the tree in a single object. diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index b645f5cd1..ead91fb4d 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -250,7 +250,6 @@ class BaseSymbolTableInterface: def clear_symbol_cache(self) -> None: """Clears the symbol cache of this symbol table.""" - pass class SymbolSpaceInterface(collections.abc.Mapping): @@ -378,7 +377,7 @@ class NativeTableInterface(BaseSymbolTableInterface): return [] -class MetadataInterface(object): +class MetadataInterface: """Interface for accessing metadata stored within a symbol table.""" def __init__(self, json_data: Dict) -> None: diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 3cfc0a25b..a5b25d178 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -1,7 +1,6 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import contextlib import logging import struct from typing import Tuple, Optional @@ -138,7 +137,7 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): ulong_bitmap_array = summary_header.get_buffer_long() # outer_index points to a 32 bits array inside a list of arrays, # each bit indicating a page mapping state - for outer_index in range(0, ulong_bitmap_array.vol.count): + for outer_index in range(ulong_bitmap_array.vol.count): ulong_bitmap = ulong_bitmap_array[outer_index] # All pages in this 32 bits array are mapped (speedup iteration process) if ulong_bitmap == 0xFFFFFFFF: @@ -166,7 +165,7 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): seg_first_bit = None # Some pages in this 32 bits array are mapped and some aren't else: - for inner_bit_position in range(0, 32): + for inner_bit_position in range(32): current_bit = outer_index * 32 + inner_bit_position page_mapped = ulong_bitmap & (1 << inner_bit_position) if page_mapped: @@ -220,9 +219,7 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): for idx, (start_position, mapped_offset, length, _) in enumerate(segments): vollog.log( constants.LOGLEVEL_VVVV, - "Segment {}: Position {:#x} Offset {:#x} Length {:#x}".format( - idx, start_position, mapped_offset, length - ), + f"Segment {idx}: Position {start_position:#x} Offset {mapped_offset:#x} Length {length:#x}", ) self._segments = segments diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 7918ebed4..7c2c72ac1 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -76,13 +76,13 @@ class Intel(linear.LinearlyMappedLayer): self._index_shift = math.ceil(math.log2(struct.calcsize(self._entry_format))) @classproperty - @functools.lru_cache() + @functools.lru_cache def page_shift(cls) -> int: """Page shift for the intel memory layers.""" return cls._page_size_in_bits @classproperty - @functools.lru_cache() + @functools.lru_cache def page_size(cls) -> int: """Page size for the intel memory layers. @@ -91,25 +91,25 @@ class Intel(linear.LinearlyMappedLayer): return 1 << cls._page_size_in_bits @classproperty - @functools.lru_cache() + @functools.lru_cache def page_mask(cls) -> int: """Page mask for the intel memory layers.""" return ~(cls.page_size - 1) @classproperty - @functools.lru_cache() + @functools.lru_cache def bits_per_register(cls) -> int: """Returns the bits_per_register to determine the range of an IntelTranslationLayer.""" return cls._bits_per_register @classproperty - @functools.lru_cache() + @functools.lru_cache def minimum_address(cls) -> int: return 0 @classproperty - @functools.lru_cache() + @functools.lru_cache def maximum_address(cls) -> int: return (1 << cls._maxvirtaddr) - 1 @@ -251,12 +251,7 @@ class Intel(linear.LinearlyMappedLayer): if INTEL_TRANSLATION_DEBUGGING: vollog.log( constants.LOGLEVEL_VVVV, - "Entry {} at index {} gives data {} as {}".format( - hex(entry), - hex(index), - hex(struct.unpack(self._entry_format, entry_data)[0]), - name, - ), + f"Entry {hex(entry)} at index {hex(index)} gives data {hex(struct.unpack(self._entry_format, entry_data)[0])} as {name}", ) # Read out the new entry from memory diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index 542fd6ca2..eeede1673 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -48,7 +48,7 @@ if HAS_LEECHCORE: try: self._handle = leechcorepyc.LeechCore(self._device) except TypeError: - raise IOError(f"Unable to open LeechCore device {self._device}") + raise OSError(f"Unable to open LeechCore device {self._device}") return self._handle def fileno(self): diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index ff483291c..a8127e954 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -236,7 +236,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer): if self._architecture is None: vollog.log( constants.LOGLEVEL_VV, - f"QEVM architecture could not be determined", + "QEVM architecture could not be determined", ) # Once all segments have been read, determine the PCI hole if any diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 609832886..cc364ad50 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -156,9 +156,7 @@ class RegistryHive(linear.LinearlyMappedLayer): else: # It doesn't matter that we use KeyNode, we're just after the first two bytes vollog.debug( - "Unknown Signature {} (0x{:x}) at offset {}".format( - signature, cell.u.KeyNode.Signature, cell_offset - ) + f"Unknown Signature {signature} (0x{cell.u.KeyNode.Signature:x}) at offset {cell_offset}" ) return cell @@ -178,9 +176,7 @@ class RegistryHive(linear.LinearlyMappedLayer): if not root_node.vol.type_name.endswith(constants.BANG + "_CM_KEY_NODE"): raise RegistryFormatException( self.name, - "Encountered {} instead of _CM_KEY_NODE".format( - root_node.vol.type_name - ), + f"Encountered {root_node.vol.type_name} instead of _CM_KEY_NODE", ) node_key = [root_node] if key.endswith("\\"): diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 2dba7caa8..c7a7fee67 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -57,7 +57,7 @@ def cascadeCloseFile(new_fp: IO[bytes], original_fp: IO[bytes]) -> IO[bytes]: return new_fp -class ResourceAccessor(object): +class ResourceAccessor: """Object for opening URLs as files (downloading locally first if necessary)""" diff --git a/volatility3/framework/layers/scanners/multiregexp.py b/volatility3/framework/layers/scanners/multiregexp.py index be3581f05..9831a9d8e 100644 --- a/volatility3/framework/layers/scanners/multiregexp.py +++ b/volatility3/framework/layers/scanners/multiregexp.py @@ -6,7 +6,7 @@ import re from typing import Generator, List, Tuple -class MultiRegexp(object): +class MultiRegexp: """Algorithm for multi-string matching.""" def __init__(self) -> None: diff --git a/volatility3/framework/objects/utility.py b/volatility3/framework/objects/utility.py index 8aa527cdb..b241ed56a 100644 --- a/volatility3/framework/objects/utility.py +++ b/volatility3/framework/objects/utility.py @@ -22,8 +22,8 @@ def bswap_32(value: int) -> int: def bswap_64(value: int) -> int: - low = bswap_32((value >> 32)) - high = bswap_32((value & 0xFFFFFFFF)) + low = bswap_32(value >> 32) + high = bswap_32(value & 0xFFFFFFFF) return ((high << 32) | low) & 0xFFFFFFFFFFFFFFFF diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 24149a390..10e7a7a72 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -119,7 +119,7 @@ class LayerWriter(plugins.PluginInterface): # Update the filename, which may have changed if a file # with the same name already existed. output_name = file_handle.preferred_filename - except IOError as excp: + except OSError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) yield 0, (f"Layer has been written to {output_name}",) diff --git a/volatility3/framework/plugins/linux/check_idt.py b/volatility3/framework/plugins/linux/check_idt.py index cc3a08933..07582e2c1 100644 --- a/volatility3/framework/plugins/linux/check_idt.py +++ b/volatility3/framework/plugins/linux/check_idt.py @@ -53,7 +53,7 @@ class Check_idt(interfaces.plugins.PluginInterface): address_mask = self.context.layers[vmlinux.layer_name].address_mask # hw handlers + system call - check_idxs = list(range(0, 20)) + [128] + check_idxs = list(range(20)) + [128] if is_32bit: if vmlinux.has_type("gate_struct"): diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index b6634d612..3537a9fa1 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -103,7 +103,7 @@ class Check_syscall(plugins.PluginInterface): try: func_addr = vmlinux.get_symbol(syscall_entry_func).address - except exceptions.SymbolError as e: + except exceptions.SymbolError: # if we can't find the disassemble function then bail and rely on a different method return 0 diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 7dbf074b3..382268515 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -462,7 +462,7 @@ class InodePages(plugins.PluginInterface): f.seek(current_fp) f.write(page_bytes) - except IOError as e: + except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) def _generator(self): diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 00832140a..065f239a9 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -125,9 +125,7 @@ class Maps(plugins.PluginInterface): proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - pid, excp.invalid_address, excp.layer_name - ) + f"Process {pid}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None vm_size = vm_end - vm_start diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 40784a647..0cca4704f 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -133,7 +133,7 @@ class PsScan(interfaces.plugins.PluginInterface): ) elif len(kernel_layer.dependencies) == 0: vollog.error( - f"Kernel layer has no dependencies, meaning there is no memory layer for this plugin to scan." + "Kernel layer has no dependencies, meaning there is no memory layer for this plugin to scan." ) raise exceptions.LayerException( kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index fe5179dfa..5c002e472 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -115,9 +115,7 @@ class Maps(interfaces.plugins.PluginInterface): proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - pid, excp.invalid_address, excp.layer_name - ) + f"Process {pid}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None vm_size = vm_end - vm_start diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index c754e43ef..ba729f898 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -143,9 +143,7 @@ class Timeliner(interfaces.plugins.PluginInterface): times = self.timeline.get((plugin_name, item), {}) if times.get(timestamp_type, None) is not None: vollog.debug( - "Multiple timestamps for the same plugin/file combination found: {} {}".format( - plugin_name, item - ) + f"Multiple timestamps for the same plugin/file combination found: {plugin_name} {item}" ) times[timestamp_type] = timestamp self.timeline[(plugin_name, item)] = times diff --git a/volatility3/framework/plugins/windows/cmdline.py b/volatility3/framework/plugins/windows/cmdline.py index 8cfb5576c..9bd9eda0e 100644 --- a/volatility3/framework/plugins/windows/cmdline.py +++ b/volatility3/framework/plugins/windows/cmdline.py @@ -84,9 +84,7 @@ class CmdLine(interfaces.plugins.PluginInterface): result_text = f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)" except exceptions.InvalidAddressException as exp: - result_text = "Process {}: Required memory at {:#x} is not valid (incomplete layer {}?)".format( - proc_id, exp.invalid_address, exp.layer_name - ) + result_text = f"Process {proc_id}: Required memory at {exp.invalid_address:#x} is not valid (incomplete layer {exp.layer_name}?)" yield (0, (proc.UniqueProcessId, process_name, result_text)) diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index ad1c9d4bd..a448989c0 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -95,9 +95,7 @@ class Consoles(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) @classmethod @@ -176,12 +174,7 @@ class Consoles(interfaces.plugins.PluginInterface): ) vollog.debug( - "Determined OS Version: {}.{} {}.{}".format( - kuser.NtMajorVersion, - kuser.NtMinorVersion, - vers.MajorVersion, - vers.MinorVersion, - ) + f"Determined OS Version: {kuser.NtMajorVersion}.{kuser.NtMinorVersion} {vers.MajorVersion}.{vers.MinorVersion}" ) if nt_major_version == 10 and arch == "x64": @@ -260,9 +253,7 @@ class Consoles(interfaces.plugins.PluginInterface): if ver: conhost_mod_version = ver[3] vollog.debug( - "Determined conhost.exe's FileVersion: {}".format( - conhost_mod_version - ) + f"Determined conhost.exe's FileVersion: {conhost_mod_version}" ) else: vollog.debug("Could not determine conhost.exe's FileVersion.") @@ -311,12 +302,7 @@ class Consoles(interfaces.plugins.PluginInterface): else: raise NotImplementedError( - "This version of Windows is not supported: {}.{} {}.{}!".format( - nt_major_version, - nt_minor_version, - vers.MajorVersion, - vers_minor_version, - ) + f"This version of Windows is not supported: {nt_major_version}.{nt_minor_version} {vers.MajorVersion}.{vers_minor_version}!" ) vollog.debug(f"Determined symbol filename: {filename}") diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 5a1b37fcf..57f19f620 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -5,9 +5,9 @@ import contextlib import datetime import logging import re -from typing import List, Optional, Type +from typing import List -from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed @@ -199,16 +199,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _depth, row_data = row if not isinstance(row_data[6], datetime.datetime): continue - description = ( - "DLL Load: Process {} {} Loaded {} ({}) Size {} Offset {}".format( - row_data[0], - row_data[1], - row_data[4], - row_data[5], - row_data[3], - row_data[2], - ) - ) + description = f"DLL Load: Process {row_data[0]} {row_data[1]} Loaded {row_data[4]} ({row_data[5]}) Size {row_data[3]} Offset {row_data[2]}" yield (description, timeliner.TimeLinerType.CREATED, row_data[6]) def run(self): diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index bc554c0bf..64d9be4db 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -192,13 +192,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): for memory_object, layer, extension in dump_parameters: cache_name = EXTENSION_CACHE_MAP[extension] - desired_file_name = "file.{0:#x}.{1:#x}.{2}.{3}.{4}".format( - file_obj.vol.offset, - memory_object.vol.offset, - cache_name, - ntpath.basename(obj_name), - extension, - ) + desired_file_name = f"file.{file_obj.vol.offset:#x}.{memory_object.vol.offset:#x}.{cache_name}.{ntpath.basename(obj_name)}.{extension}" file_handle = cls.dump_file_producer( file_obj, memory_object, open_method, layer, desired_file_name diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index 66db03c9c..cac4ecf40 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -92,7 +92,7 @@ class Envars(interfaces.plugins.PluginInterface): except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing global environment variables keys (some keys might be excluded)", @@ -113,7 +113,7 @@ class Envars(interfaces.plugins.PluginInterface): except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing user environment variables keys (some keys might be excluded)", @@ -134,7 +134,7 @@ class Envars(interfaces.plugins.PluginInterface): except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing volatile environment variables keys (some keys might be excluded)", diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index 9b20ed2d0..37df940a2 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -55,7 +55,7 @@ class GetServiceSIDs(interfaces.plugins.PluginInterface): ) # Get service sids dictionary (we need only the service sids). - with open(sids_json_file_name, "r") as file_handle: + with open(sids_json_file_name) as file_handle: self.servicesids = json.load(file_handle)["service sids"] @classmethod diff --git a/volatility3/framework/plugins/windows/getsids.py b/volatility3/framework/plugins/windows/getsids.py index 3e332f85d..df0c7a835 100644 --- a/volatility3/framework/plugins/windows/getsids.py +++ b/volatility3/framework/plugins/windows/getsids.py @@ -58,7 +58,7 @@ class GetSIDs(interfaces.plugins.PluginInterface): ) # Get all the sids from the json file. - with open(sids_json_file_name, "r") as file_handle: + with open(sids_json_file_name) as file_handle: sids_json_data = json.load(file_handle) self.servicesids = sids_json_data["service sids"] self.well_known_sids = sids_json_data["well known"] @@ -122,7 +122,7 @@ class GetSIDs(interfaces.plugins.PluginInterface): except ( exceptions.InvalidAddressException, layers.registry.RegistryFormatException, - ) as excp: + ): continue try: value_data = node.decode_data() @@ -156,7 +156,7 @@ class GetSIDs(interfaces.plugins.PluginInterface): ValueError, exceptions.InvalidAddressException, layers.registry.RegistryFormatException, - ) as excp: + ): continue except (KeyError, exceptions.InvalidAddressException): continue diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index 69fa94f06..30d4b602c 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -12,20 +12,15 @@ from volatility3.plugins.windows import pslist, vadinfo vollog = logging.getLogger(__name__) -VadData = NamedTuple( - "VadData", - [ - ("protection", str), - ("path", str), - ], -) -DLLData = NamedTuple( - "DLLData", - [ - ("path", str), - ], -) +class VadData(NamedTuple): + protection: str + path: str + + +class DLLData(NamedTuple): + path: str + ### Useful references on process hollowing # https://cysinfo.com/detecting-deceptive-hollowing-techniques/ @@ -146,9 +141,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): """ image_base = self._get_image_base(proc) if image_base is not None and image_base != proc.SectionBaseAddress: - yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format( - image_base, proc.SectionBaseAddress - ) + yield f"The ImageBaseAddress reported from the PEB ({image_base:#x}) does not match the process SectionBaseAddress ({proc.SectionBaseAddress:#x})" def _check_exe_protection( self, proc, vads: Dict[int, VadData], __ @@ -166,13 +159,9 @@ class HollowProcesses(interfaces.plugins.PluginInterface): base = proc.SectionBaseAddress if base not in vads: - yield "There is no VAD starting at the base address of the process executable ({:#x})".format( - base - ) + yield f"There is no VAD starting at the base address of the process executable ({base:#x})" elif vads[base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for VAD hosting the process executable ({:#x}) with path {}".format( - vads[base].protection, base, vads[base].path - ) + yield f"Unexpected protection ({vads[base].protection}) for VAD hosting the process executable ({base:#x}) with path {vads[base].path}" def _check_dlls_protection( self, _, vads: Dict[int, VadData], dlls: Dict[int, DLLData] @@ -184,9 +173,7 @@ class HollowProcesses(interfaces.plugins.PluginInterface): # PAGE_EXECUTE_WRITECOPY is the only valid permission for mapped DLLs and .exe files if vads[dll_base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format( - vads[dll_base].protection, dll_base, dlls[dll_base].path - ) + yield f"Unexpected protection ({vads[dll_base].protection}) for DLL in the PEB's load order list ({dll_base:#x}) with path {dlls[dll_base].path}" def _generator(self, procs): checks = [ diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index d2fdc0ad8..3bf7f57ed 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,7 +1,9 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -import logging, io, pefile +import logging +import io +import pefile from volatility3.framework.symbols import intermed from volatility3.framework import renderers, interfaces, exceptions, constants from volatility3.framework.configuration import requirements @@ -119,9 +121,7 @@ class IAT(interfaces.plugins.PluginInterface): ) except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 1df090b5b..510719352 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -106,9 +106,7 @@ class Malfind(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None @@ -211,9 +209,7 @@ class Malfind(interfaces.plugins.PluginInterface): file_output = file_handle.preferred_filename except (exceptions.InvalidAddressException, OverflowError) as excp: vollog.debug( - "Unable to dump PE with pid {0}.{1:#x}: {2}".format( - proc.UniqueProcessId, vad.get_start(), excp - ) + f"Unable to dump PE with pid {proc.UniqueProcessId}.{vad.get_start():#x}: {excp}" ) yield ( diff --git a/volatility3/framework/plugins/windows/memmap.py b/volatility3/framework/plugins/windows/memmap.py index b5c9a211e..62ab3c510 100644 --- a/volatility3/framework/plugins/windows/memmap.py +++ b/volatility3/framework/plugins/windows/memmap.py @@ -53,9 +53,7 @@ class Memmap(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - pid, excp.invalid_address, excp.layer_name - ) + f"Process {pid}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue @@ -80,11 +78,7 @@ class Memmap(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: file_output = "Error outputting to file" vollog.debug( - "Unable to write {}'s address {} to {}".format( - proc_layer_name, - offset, - file_handle.preferred_filename, - ) + f"Unable to write {proc_layer_name}'s address {offset} to {file_handle.preferred_filename}" ) yield ( diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index ba45834d5..2e8dc1b0e 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -177,9 +177,7 @@ class Modules(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, - "Process {} does not have a valid Session or a layer could not be constructed for it".format( - proc_id - ), + f"Process {proc_id} does not have a valid Session or a layer could not be constructed for it", ) continue diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 66a24da5a..77bd22ab9 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -169,12 +169,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) vollog.debug( - "Determined OS Version: {}.{} {}.{}".format( - kuser.NtMajorVersion, - kuser.NtMinorVersion, - vers.MajorVersion, - vers.MinorVersion, - ) + f"Determined OS Version: {kuser.NtMajorVersion}.{kuser.NtMinorVersion} {vers.MajorVersion}.{vers.MinorVersion}" ) if nt_major_version == 10 and arch == "x64": @@ -272,9 +267,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if ver: tcpip_mod_version = ver[3] vollog.debug( - "Determined tcpip.sys's FileVersion: {}".format( - tcpip_mod_version - ) + f"Determined tcpip.sys's FileVersion: {tcpip_mod_version}" ) else: vollog.debug("Could not determine tcpip.sys's FileVersion.") @@ -316,12 +309,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): else: raise NotImplementedError( - "This version of Windows is not supported: {}.{} {}.{}!".format( - nt_major_version, - nt_minor_version, - vers.MajorVersion, - vers_minor_version, - ) + f"This version of Windows is not supported: {nt_major_version}.{nt_minor_version} {vers.MajorVersion}.{vers_minor_version}!" ) vollog.debug(f"Determined symbol filename: {filename}") @@ -510,17 +498,8 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for i in row_data ] description = ( - "Network connection: Process {} {} Local Address {}:{} " - "Remote Address {}:{} State {} Protocol {} ".format( - row_data[7], - row_data[8], - row_data[2], - row_data[3], - row_data[4], - row_data[5], - row_data[6], - row_data[1], - ) + f"Network connection: Process {row_data[7]} {row_data[8]} Local Address {row_data[2]}:{row_data[3]} " + f"Remote Address {row_data[4]}:{row_data[5]} State {row_data[6]} Protocol {row_data[1]} " ) yield (description, timeliner.TimeLinerType.CREATED, row_data[9]) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 0908767fc..c774e23a3 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -311,9 +311,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): part_table.Partitions.count = part_count vollog.debug( - "Found TCP connection PartitionTable @ 0x{:x} (partition count: {})".format( - part_table_addr, part_count - ) + f"Found TCP connection PartitionTable @ 0x{part_table_addr:x} (partition count: {part_count})" ) entry_offset = context.symbol_space.get_type(obj_name).relative_child_offset( "ListEntry" @@ -624,9 +622,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): proto = "TCPv6" else: vollog.debug( - "TCP Endpoint @ 0x{:2x} has unknown address family 0x{:x}".format( - netw_obj.vol.offset, netw_obj.get_address_family() - ) + f"TCP Endpoint @ 0x{netw_obj.vol.offset:2x} has unknown address family 0x{netw_obj.get_address_family():x}" ) proto = "TCPv?" diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 955098d6b..002577241 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -645,7 +645,7 @@ class PESymbols(interfaces.plugins.PluginInterface): and wanted_addresses_identifier not in wanted_symbols ): vollog.warning( - f"Invalid `wanted_symbols` sent to `find_symbols`. addresses and names keys both misssing." + "Invalid `wanted_symbols` sent to `find_symbols`. addresses and names keys both misssing." ) return diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 5b4bb07d7..d2a8a7370 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -64,27 +64,30 @@ class PEDump(interfaces.plugins.PluginInterface): """ Returns the filename of the dump file or None """ - with open_method(file_name) as file_handle: - try: - dos_header = context.object( - pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset=base, - layer_name=layer_name, - ) + try: + file_handle = open_method(file_name) - for offset, data in dos_header.reconstruct(): - file_handle.seek(offset) - file_handle.write(data) - except ( - IOError, - exceptions.VolatilityException, - OverflowError, - ValueError, - ) as excp: - vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") - return None + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base, + layer_name=layer_name, + ) - return file_handle.preferred_filename + for offset, data in dos_header.reconstruct(): + file_handle.seek(offset) + file_handle.write(data) + except ( + OSError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: + vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") + return None + finally: + file_handle.close() + + return file_handle.preferred_filename @classmethod def dump_ldr_entry( @@ -116,12 +119,7 @@ class PEDump(interfaces.plugins.PluginInterface): if layer_name is None: layer_name = ldr_entry.vol.layer_name - file_name = "{}{}.{:#x}.{:#x}.dmp".format( - prefix, - ntpath.basename(name), - ldr_entry.vol.offset, - ldr_entry.DllBase, - ) + file_name = f"{prefix}{ntpath.basename(name)}.{ldr_entry.vol.offset:#x}.{ldr_entry.DllBase:#x}.dmp" return cls.dump_pe( context, @@ -143,11 +141,7 @@ class PEDump(interfaces.plugins.PluginInterface): pid: int, base: int, ) -> Optional[str]: - file_name = "PE.{:#x}.{:d}.{:#x}.dmp".format( - proc_offset, - pid, - base, - ) + file_name = f"PE.{proc_offset:#x}.{pid:d}.{base:#x}.dmp" return PEDump.dump_pe( context, pe_table_name, layer_name, open_method, file_name, base diff --git a/volatility3/framework/plugins/windows/privileges.py b/volatility3/framework/plugins/windows/privileges.py index 0370dfc92..7b4d00205 100644 --- a/volatility3/framework/plugins/windows/privileges.py +++ b/volatility3/framework/plugins/windows/privileges.py @@ -39,7 +39,7 @@ class Privs(interfaces.plugins.PluginInterface): ) # Get service sids dictionary (we need only the service sids). - with open(sids_json_file_name, "r") as file_handle: + with open(sids_json_file_name) as file_handle: temp_json = json.load(file_handle)["privileges"] self.privilege_info = { int(priv_num): temp_json[priv_num] for priv_num in temp_json diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 6c845bf81..053ec20d5 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -14,7 +14,6 @@ from volatility3.plugins.windows import ( info, pslist, psscan, - sessions, thrdscan, ) diff --git a/volatility3/framework/plugins/windows/registry/hivelist.py b/volatility3/framework/plugins/windows/registry/hivelist.py index ddc9c1855..91a99a9fb 100644 --- a/volatility3/framework/plugins/windows/registry/hivelist.py +++ b/volatility3/framework/plugins/windows/registry/hivelist.py @@ -232,10 +232,8 @@ class HiveList(interfaces.plugins.PluginInterface): for hive in hg: if hive.vol.offset in seen: vollog.debug( - "Hivelist found an already seen offset {} while " - "traversing forwards, this should not occur".format( - hex(hive.vol.offset) - ) + f"Hivelist found an already seen offset {hex(hive.vol.offset)} while " + "traversing forwards, this should not occur" ) break seen.add(hive.vol.offset) @@ -249,18 +247,14 @@ class HiveList(interfaces.plugins.PluginInterface): forward_invalid = hg.invalid if forward_invalid: vollog.debug( - "Hivelist failed traversing the list forwards at {}, traversing backwards".format( - hex(forward_invalid) - ) + f"Hivelist failed traversing the list forwards at {hex(forward_invalid)}, traversing backwards" ) hg = HiveGenerator(cmhive, forward=False) for hive in hg: if hive.vol.offset in seen: vollog.debug( - "Hivelist found an already seen offset {} while " - "traversing backwards, list walking met in the middle".format( - hex(hive.vol.offset) - ) + f"Hivelist found an already seen offset {hex(hive.vol.offset)} while " + "traversing backwards, list walking met in the middle" ) break seen.add(hive.vol.offset) @@ -281,10 +275,8 @@ class HiveList(interfaces.plugins.PluginInterface): # by walking the list, so revert to scanning, and walk the list forwards and backwards from each # found hive vollog.debug( - "Hivelist failed traversing backwards at {}, a different " - "location from forwards, revert to scanning".format( - hex(backward_invalid) - ) + f"Hivelist failed traversing backwards at {hex(backward_invalid)}, a different " + "location from forwards, revert to scanning" ) for hive in hivescan.HiveScan.scan_hives( context, layer_name, symbol_table @@ -320,9 +312,7 @@ class HiveList(interfaces.plugins.PluginInterface): yield linked_hive except exceptions.InvalidAddressException: vollog.debug( - "InvalidAddressException when traversing hive {} found from scan, skipping".format( - hex(hive.vol.offset) - ) + f"InvalidAddressException when traversing hive {hex(hive.vol.offset)} found from scan, skipping" ) def run(self) -> renderers.TreeGrid: diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index bd832b20c..932ee9d6f 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -39,7 +39,7 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac os.path.join(os.path.dirname(__file__), "userassist.json"), "rb" ) as fp: self._folder_guids = json.load(fp) - except IOError: + except OSError: vollog.error("Usersassist data file not found") @classmethod @@ -308,9 +308,7 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac ) except exceptions.InvalidAddressException as excp: vollog.debug( - "Invalid address identified in lower layer {}: {}".format( - excp.layer_name, excp.invalid_address - ) + f"Invalid address identified in lower layer {excp.layer_name}: {excp.invalid_address}" ) except KeyError: vollog.debug( diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index d321c2cc0..f5d7e1b3a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -172,9 +172,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: vollog.debug( - "Unable to construct cSystems array at given offset: {:x}".format( - array_start - ) + f"Unable to construct cSystems array at given offset: {array_start:x}" ) array = None @@ -291,9 +289,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None, None diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 0eaa65884..b8dea0cdd 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -170,9 +170,7 @@ class Strings(interfaces.plugins.PluginInterface): proc_layer_name = process.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index a59581063..7c26a09bd 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -85,9 +85,7 @@ class SvcList(svcscan.SvcScan): layer_name = proc.add_process_layer() except exceptions.InvalidAddressException: vollog.warning( - "Unable to access memory of services.exe running with PID: {}".format( - proc.UniqueProcessId - ) + f"Unable to access memory of services.exe running with PID: {proc.UniqueProcessId}" ) continue diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index ca390561f..bd477ba27 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -26,13 +26,9 @@ from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) -ServiceBinaryInfo = NamedTuple( - "ServiceBinaryInfo", - [ - ("dll", Union[str, interfaces.renderers.BaseAbsentValue]), - ("binary", Union[str, interfaces.renderers.BaseAbsentValue]), - ], -) +class ServiceBinaryInfo(NamedTuple): + dll: Union[str, interfaces.renderers.BaseAbsentValue] + binary: Union[str, interfaces.renderers.BaseAbsentValue] class SvcScan(interfaces.plugins.PluginInterface): @@ -306,9 +302,7 @@ class SvcScan(interfaces.plugins.PluginInterface): proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index b812a15ff..c0963e754 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -82,7 +82,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ethread.get_exit_time() ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object except exceptions.InvalidAddressException: - vollog.debug("Thread invalid address {:#x}".format(ethread.vol.offset)) + vollog.debug(f"Thread invalid address {ethread.vol.offset:#x}") return None return ( diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index 98a3169a5..a34818fc1 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -3,7 +3,7 @@ # import logging -from typing import Callable, Iterable, List, Generator +from typing import Iterable, List, Generator from volatility3.framework import interfaces, constants from volatility3.framework.configuration import requirements diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index d49c28784..54bca1a1e 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -141,7 +141,7 @@ class Timers(interfaces.plugins.PluginInterface): if dpc.DeferredRoutine == 0: continue deferred_routine = dpc.DeferredRoutine - except Exception as e: + except Exception: continue module_symbols = list( diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index 1a1e59940..5b21225c8 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -191,7 +191,7 @@ class unhooked_system_calls(interfaces.plugins.PluginInterface): # gather processes on small_idx since these are the malware infected ones for pid, pname in cb[small_idx]: - ps.append("{:d}:{}".format(pid, pname)) + ps.append(f"{pid:d}:{pname}") proc_names = ", ".join(ps) diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 2c6ed4daf..0c4a8aaca 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -169,9 +169,7 @@ class VadInfo(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 4a06ed0c9..4930789d2 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -212,9 +212,7 @@ class VerInfo(interfaces.plugins.PluginInterface): proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 43bb59a21..02805acc2 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -88,9 +88,7 @@ class TreeNode(interfaces.renderers.TreeNode): val = values[index] if not isinstance(val, (column.type, interfaces.renderers.BaseAbsentValue)): raise TypeError( - "Values item with index {} is the wrong type for column {} (got {} but expected {})".format( - index, column.name, type(val), column.type - ) + f"Values item with index {index} is the wrong type for column {column.name} (got {type(val)} but expected {column.type})" ) # TODO: Consider how to deal with timezone naive/aware datetimes (and alert plugin uses to be precise) # if isinstance(val, datetime.datetime): @@ -189,9 +187,7 @@ class TreeGrid(interfaces.renderers.TreeGrid): is_simple_type = issubclass(column_type, self.base_types) if not is_simple_type: raise TypeError( - "Column {}'s type is not a simple type: {}".format( - name, column_type.__class__.__name__ - ) + f"Column {name}'s type is not a simple type: {column_type.__class__.__name__}" ) converted_columns.append(interfaces.renderers.Column(name, column_type)) self.RowStructure = RowStructureConstructor( diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 5f558bf12..8a28d732f 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -171,7 +171,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): (indicating that only additive changes have been made) than the consumer (in this case, the file reader). """ - major, minor, patch = [int(x) for x in version.split(".")] + major, minor, patch = (int(x) for x in version.split(".")) supported_versions = [x for x in versions if x[0] == major and x[1] >= minor] if not supported_versions: raise ValueError( diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3289775b6..537b729ad 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -798,7 +798,7 @@ class RadixTree(IDStorage): return True -class PageCache(object): +class PageCache: """Linux Page Cache abstraction""" def __init__( diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 829622154..e9fd09eaa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1469,7 +1469,7 @@ class mount(objects.StructType): def next_peer(self): table_name = self.vol.type_name.split(constants.BANG)[0] - mount_struct = "{0}{1}mount".format(table_name, constants.BANG) + mount_struct = f"{table_name}{constants.BANG}mount" offset = self._context.symbol_space.get_type( mount_struct ).relative_child_offset("mnt_share") @@ -2487,7 +2487,7 @@ class address_space(objects.StructType): class page(objects.StructType): @property - @functools.lru_cache() + @functools.lru_cache def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 793e506c3..07ab5f5a8 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -1081,7 +1081,7 @@ class KTIMER(objects.StructType): return self.Header.Type in self.VALID_TYPES def get_due_time(self): - return "{0:#010x}:{1:#010x}".format(self.DueTime.HighPart, self.DueTime.LowPart) + return f"{self.DueTime.HighPart:#010x}:{self.DueTime.LowPart:#010x}" def get_dpc(self): """Return Dpc, and if Windows 7 or later, decode it""" @@ -1388,7 +1388,7 @@ class SHARED_CACHE_MAP(objects.StructType): ) # Iterate through the entries - for counter in range(0, self.VACB_ARRAY): + for counter in range(self.VACB_ARRAY): # Check if the VACB entry is in use if not vacb_array[counter]: continue @@ -1472,7 +1472,7 @@ class SHARED_CACHE_MAP(objects.StructType): if not section_size > self.VACB_SIZE_OF_FIRST_LEVEL: array_head = vacb_obj - for counter in range(0, full_blocks): + for counter in range(full_blocks): vacb_entry = self._context.object( symbol_table_name + constants.BANG + "pointer", layer_name=self.vol.layer_name, @@ -1531,7 +1531,7 @@ class SHARED_CACHE_MAP(objects.StructType): # Walk the array and if any entry points to the shared cache map object then we extract it. # Otherwise, if it is non-zero, then traverse to the next level. - for counter in range(0, self.VACB_ARRAY): + for counter in range(self.VACB_ARRAY): if not vacb_array[counter]: continue diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 2312149c7..cf6f43a9b 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -73,7 +73,7 @@ class ROW(objects.StructType): ) for i in range(0, len(char_row), 3) ) - except Exception as e: + except Exception: line = "" if truncate: diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index afdc73a17..078c4beb0 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -8,12 +8,7 @@ from volatility3.framework import objects class PARTITION_TABLE(objects.StructType): def get_disk_signature(self) -> str: """Get Disk Signature (GUID).""" - return "{0:02x}-{1:02x}-{2:02x}-{3:02x}".format( - self.DiskSignature[0], - self.DiskSignature[1], - self.DiskSignature[2], - self.DiskSignature[3], - ) + return f"{self.DiskSignature[0]:02x}-{self.DiskSignature[1]:02x}-{self.DiskSignature[2]:02x}-{self.DiskSignature[3]:02x}" class PARTITION_ENTRY(objects.StructType): diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 9b7573c2e..00c24f176 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -22,7 +22,7 @@ def inet_ntop(address_family: int, packed_ip: Union[List[int], Array]) -> str: raise RuntimeError( "This version of python does not have socket.inet_ntop, please upgrade" ) - raise socket.error("[Errno 97] Address family not supported by protocol") + raise OSError("[Errno 97] Address family not supported by protocol") # Python's socket.AF_INET6 is 0x1e but Microsoft defines it @@ -167,11 +167,9 @@ class _TCP_LISTENER(objects.StructType): def is_valid(self): try: - if not self.get_address_family() in (AF_INET, AF_INET6): + if self.get_address_family() not in (AF_INET, AF_INET6): vollog.debug( - "netw obj 0x{:x} invalid due to invalid address_family {}".format( - self.vol.offset, self.get_address_family() - ) + f"netw obj 0x{self.vol.offset:x} invalid due to invalid address_family {self.get_address_family()}" ) return False diff --git a/volatility3/framework/symbols/windows/extensions/pe.py b/volatility3/framework/symbols/windows/extensions/pe.py index 3f34fc3dd..2c7400f25 100644 --- a/volatility3/framework/symbols/windows/extensions/pe.py +++ b/volatility3/framework/symbols/windows/extensions/pe.py @@ -101,9 +101,9 @@ class IMAGE_DOS_HEADER(objects.StructType): ) except OverflowError: vollog.warning( - "Volatility was unable to fix the image base for the PE file at base address {:#x}. " + f"Volatility was unable to fix the image base for the PE file at base address {self.vol.offset:#x}. " "This will cause issues with many static analysis tools if you do not inform the " - "tool of the in-memory load address.".format(self.vol.offset) + "tool of the in-memory load address." ) new_pe = raw_data diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index b761ddad8..5a7847986 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -217,7 +217,7 @@ class POOL_HEADER(objects.StructType): yield mem_object @classmethod - @functools.lru_cache() + @functools.lru_cache def _calculate_optional_header_lengths( cls, context: interfaces.context.ContextInterface, symbol_table_name: str ) -> Tuple[List[str], List[int]]: @@ -430,9 +430,7 @@ class OBJECT_HEADER(objects.StructType): if header_offset == 0: raise ValueError( - "Could not find _OBJECT_HEADER_NAME_INFO for object at {} of layer {}".format( - self.vol.offset, self.vol.layer_name - ) + f"Could not find _OBJECT_HEADER_NAME_INFO for object at {self.vol.offset} of layer {self.vol.layer_name}" ) header = ntkrnlmp.object( diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index bebfaea89..9e2f8df3b 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -196,9 +196,7 @@ class CM_KEY_NODE(objects.StructType): yield cast("CM_KEY_NODE", node) else: vollog.debug( - "Unexpected node type encountered when traversing subkeys: {}, signature: {}".format( - node.vol.type_name, signature - ) + f"Unexpected node type encountered when traversing subkeys: {node.vol.type_name}, signature: {signature}" ) if listjump: diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index 82ec31ccb..4feb396e7 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -263,9 +263,7 @@ class PdbReader: ) if header.index_max < header.index_min: raise ValueError( - "Maximum {} index is smaller than minimum TPI index, found: {} < {} ".format( - stream_name, header.index_max, header.index_min - ) + f"Maximum {stream_name} index is smaller than minimum TPI index, found: {header.index_max} < {header.index_min} " ) # Reset the state info_references: Dict[str, int] = {} @@ -976,7 +974,7 @@ class PdbRetreiver: if __name__ == "__main__": import argparse - class PrintedProgress(object): + class PrintedProgress: """A progress handler that prints the progress value and the description onto the command line.""" diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 3816312cd..1a8644fa8 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -94,7 +94,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): if not requirements.VersionRequirement.matches_required( (1, 0, 0), symbol_cache.SqliteCache.version ): - vollog.debug(f"Required version of SQLiteCache not found") + vollog.debug("Required version of SQLiteCache not found") return None identifiers_path = os.path.join( @@ -291,9 +291,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): break except PermissionError: vollog.warning( - "Cannot write necessary symbol file, please check permissions on {}".format( - potential_output_filename - ) + f"Cannot write necessary symbol file, please check permissions on {potential_output_filename}" ) continue finally: diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 5ef840f32..8587b3719 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -60,7 +60,7 @@ class Certificates(interfaces.plugins.PluginInterface): open_method: Type[interfaces.plugins.FileHandlerInterface], ) -> Optional[interfaces.plugins.FileHandlerInterface]: try: - dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) + dump_name = f"{hive_offset}-{reg_section}-{key_hash}.crt" file_handle = open_method(dump_name) file_handle.write(certificate_data) return file_handle diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 7f56b75f8..e7557dc0c 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -64,9 +64,7 @@ class Statistics(plugins.PluginInterface): other_invalid += 1 page_size = expected_page_size vollog.debug( - "A non-page lookup invalid address exception occurred at: {} in layer {}".format( - hex(excp.invalid_address), excp.layer_name - ) + f"A non-page lookup invalid address exception occurred at: {hex(excp.invalid_address)} in layer {excp.layer_name}" ) page_addr += page_size diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 3ca00e5dc..90cfaba48 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -20,7 +20,7 @@ def load_cached_validations() -> Set[str]: to revalidate them.""" validhashes: Set = set() if os.path.exists(cached_validation_filepath): - with open(cached_validation_filepath, "r") as f: + with open(cached_validation_filepath) as f: validhashes.update(json.load(f)) return validhashes @@ -46,7 +46,7 @@ def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: if not os.path.exists(schema_path): vollog.debug(f"Schema for format not found: {schema_path}") return False - with open(schema_path, "r") as s: + with open(schema_path) as s: schema = json.load(s) return valid(input, schema, use_cache) @@ -66,7 +66,7 @@ def create_json_hash( if not os.path.exists(schema_path): vollog.debug(f"Schema for format not found: {schema_path}") return None - with open(schema_path, "r") as s: + with open(schema_path) as s: schema = json.load(s) return hashlib.sha1( bytes(json.dumps((input, schema), sort_keys=True), "utf-8") From a63ea662f46bcbcf156df1d4bd69257c4ac0a0b3 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 17:05:34 +0100 Subject: [PATCH 690/979] apply unsafe fixes (`ruff check --unsafe-fixes --fix`) --- test/plugins/windows/test_scheduled_tasks.py | 3 +- volatility3/framework/__init__.py | 9 +++-- volatility3/framework/plugins/linux/lsmod.py | 3 +- volatility3/framework/plugins/linux/proc.py | 4 ++- .../framework/plugins/linux/sockstat.py | 2 +- .../framework/plugins/mac/check_sysctl.py | 5 ++- volatility3/framework/plugins/mac/kevents.py | 3 +- volatility3/framework/plugins/mac/mount.py | 3 +- .../framework/plugins/mac/proc_maps.py | 4 ++- volatility3/framework/plugins/mac/pslist.py | 4 ++- volatility3/framework/plugins/timeliner.py | 3 +- .../framework/plugins/windows/handles.py | 6 ++-- .../framework/plugins/windows/modules.py | 3 +- .../framework/plugins/windows/netstat.py | 5 ++- .../framework/plugins/windows/pslist.py | 35 +++++++++++++------ .../framework/plugins/windows/psscan.py | 35 ++++++++++++------- .../framework/plugins/windows/shimcachemem.py | 7 ++-- .../framework/plugins/windows/svclist.py | 5 ++- .../framework/plugins/windows/threads.py | 3 +- .../plugins/windows/unloadedmodules.py | 3 +- .../framework/plugins/windows/virtmap.py | 3 +- .../framework/renderers/format_hints.py | 30 +++++++++------- .../framework/symbols/linux/__init__.py | 6 ++-- .../symbols/linux/extensions/__init__.py | 12 +++---- volatility3/framework/symbols/mac/__init__.py | 15 ++++---- .../symbols/windows/extensions/__init__.py | 15 ++++---- .../symbols/windows/extensions/consoles.py | 23 +++++------- .../framework/symbols/windows/pdbconv.py | 5 ++- 28 files changed, 129 insertions(+), 125 deletions(-) diff --git a/test/plugins/windows/test_scheduled_tasks.py b/test/plugins/windows/test_scheduled_tasks.py index 8f771b323..fdb19fbae 100644 --- a/test/plugins/windows/test_scheduled_tasks.py +++ b/test/plugins/windows/test_scheduled_tasks.py @@ -84,8 +84,7 @@ class TestActionsDecoding(unittest.TestCase): self.assertEqual(actions[0].action_type, scheduled_tasks.ActionType.Exe) except Exception: self.fail( - "ActionDecoder.decode should not raise exception:\n%s" - % traceback.format_exc() + f"ActionDecoder.decode should not raise exception:\n{traceback.format_exc()}" ) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index e0f7c778a..244b353a2 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -97,8 +97,7 @@ def class_subclasses(cls: Type[T]) -> Generator[Type[T], None, None]: # The typing system is not clever enough to realize that clazz has a hidden attr after the hasattr check if not hasattr(clazz, "hidden") or not clazz.hidden: # type: ignore yield clazz - for return_value in class_subclasses(clazz): - yield return_value + yield from class_subclasses(clazz) def import_files(base_module, ignore_errors: bool = False) -> List[str]: @@ -159,9 +158,9 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]: def _filter_files(filename: str): """Ensures that a filename traversed is an importable python file""" - return ( - filename.endswith(".py") or filename.endswith(".pyc") - ) and not filename.startswith("__") + return (filename.endswith((".py", ".pyc"))) and not filename.startswith( + "__" + ) def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str]: diff --git a/volatility3/framework/plugins/linux/lsmod.py b/volatility3/framework/plugins/linux/lsmod.py index a65b0d00b..49e990e93 100644 --- a/volatility3/framework/plugins/linux/lsmod.py +++ b/volatility3/framework/plugins/linux/lsmod.py @@ -54,8 +54,7 @@ class Lsmod(plugins.PluginInterface): table_name = modules.vol.type_name.split(constants.BANG)[0] - for module in modules.to_list(table_name + constants.BANG + "module", "list"): - yield module + yield from modules.to_list(table_name + constants.BANG + "module", "list") def _generator(self): try: diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 065f239a9..893eea71e 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -163,7 +163,9 @@ class Maps(plugins.PluginInterface): address_list = self.config.get("address", None) if not address_list: # do not filter as no address_list was supplied - vma_filter_func = lambda _: True + def vma_filter_func(_): + return True + else: # filter for any vm_start that matches the supplied address config def vma_filter_function(x: interfaces.objects.ObjectInterface) -> bool: diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index e5cf48d16..aee0b1e2e 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -372,7 +372,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): bt_sock = sock.cast("bt_sock") def bt_addr(addr): - return ":".join(reversed(["%02x" % x for x in addr.b])) + return ":".join(reversed([f"{x:02x}" for x in addr.b])) src_addr = src_port = dst_addr = dst_port = None bt_protocol = bt_sock.get_protocol() diff --git a/volatility3/framework/plugins/mac/check_sysctl.py b/volatility3/framework/plugins/mac/check_sysctl.py index 4f64eaed8..e8218962d 100644 --- a/volatility3/framework/plugins/mac/check_sysctl.py +++ b/volatility3/framework/plugins/mac/check_sysctl.py @@ -93,10 +93,9 @@ class Check_sysctl(plugins.PluginInterface): val = self._parse_global_variable_sysctls(kernel, name) elif ctltype == "CTLTYPE_NODE": if sysctl.oid_handler == 0: - for info in self._process_sysctl_list( + yield from self._process_sysctl_list( kernel, sysctl.oid_arg1, recursive=1 - ): - yield info + ) val = "Node" diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 2a8692b77..41fde31ca 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -119,8 +119,7 @@ class Kevents(interfaces.plugins.PluginInterface): return None for klist in klist_array: - for kn in mac.MacUtilities.walk_slist(klist, "kn_link"): - yield kn + yield from mac.MacUtilities.walk_slist(klist, "kn_link") @classmethod def _get_task_kevents(cls, kernel, task): diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index ff654e1a7..1a1e33571 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -49,8 +49,7 @@ class Mount(plugins.PluginInterface): list_head = kernel.object_from_symbol(symbol_name="mountlist") - for mount in mac.MacUtilities.walk_tailq(list_head, "mnt_list"): - yield mount + yield from mac.MacUtilities.walk_tailq(list_head, "mnt_list") def _generator(self): for mount in self.list_mounts(self.context, self.config["kernel"]): diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 5c002e472..bd905615d 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -152,7 +152,9 @@ class Maps(interfaces.plugins.PluginInterface): address_list = self.config.get("address", None) if not address_list: # do not filter as no address_list was supplied - vma_filter_func = lambda _: True + def vma_filter_func(_): + return True + else: # filter for any vm_start that matches the supplied address config def vma_filter_function(task: interfaces.objects.ObjectInterface) -> bool: diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 9b570f3f9..74d044ba9 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -83,7 +83,9 @@ class PsList(interfaces.plugins.PluginInterface): @classmethod def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[int], bool]: - filter_func = lambda _: False + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 pid_list = pid_list or [] filter_list = [x for x in pid_list if x is not None] diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index ba729f898..4e483922b 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -204,8 +204,7 @@ class Timeliner(interfaces.plugins.PluginInterface): ) vollog.log(logging.DEBUG, traceback.format_exc()) - for data_item in sorted(data, key=self._sort_function): - yield data_item + yield from sorted(data, key=self._sort_function) # Write out a body file if necessary if self.config.get("create-bodyfile", True): diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index a3067b09f..62eceb973 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -227,8 +227,7 @@ class Handles(interfaces.plugins.PluginInterface): for entry in table: if level > 0: - for x in self._make_handle_array(entry, level - 1, depth): - yield x + yield from self._make_handle_array(entry, level - 1, depth) depth += 1 else: handle_multiplier = 4 @@ -264,8 +263,7 @@ class Handles(interfaces.plugins.PluginInterface): ) return None - for handle_table_entry in self._make_handle_array(TableCode, table_levels): - yield handle_table_entry + yield from self._make_handle_array(TableCode, table_levels) def _generator(self, procs): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 2e8dc1b0e..00424938f 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -248,8 +248,7 @@ class Modules(interfaces.plugins.PluginInterface): object_type=type_name, offset=list_entry.vol.offset - reloff, absolute=True ) - for mod in module.InLoadOrderLinks: - yield mod + yield from module.InLoadOrderLinks def run(self): return renderers.TreeGrid( diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index c774e23a3..a1521a8c6 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -488,14 +488,13 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """ # first, TCP endpoints by parsing the partition table - for endpoint in cls.parse_partitions( + yield from cls.parse_partitions( context, layer_name, net_symbol_table, tcpip_symbol_table, tcpip_module_offset, - ): - yield endpoint + ) # then, towards the UDP and TCP port pools # first, find their addresses diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 478cc8b1b..f262aeae6 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -126,15 +126,24 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Returns: Filter function for passing to the `list_processes` method """ - filter_func = lambda _: False + + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 pid_list = pid_list or [] filter_list = [x for x in pid_list if x is not None] if filter_list: if exclude: - filter_func = lambda x: x.UniqueProcessId in filter_list + + def filter_func(x): + return x.UniqueProcessId in filter_list + else: - filter_func = lambda x: x.UniqueProcessId not in filter_list + + def filter_func(x): + return x.UniqueProcessId not in filter_list + return filter_func @classmethod @@ -173,20 +182,24 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Returns: Filter function for passing to the `list_processes` method """ - filter_func = lambda _: False + + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 name_list = name_list or [] filter_list = [x for x in name_list if x is not None] if filter_list: if exclude: - filter_func = ( - lambda x: utility.array_to_string(x.ImageFileName) in filter_list - ) + + def filter_func(x): + return utility.array_to_string(x.ImageFileName) in filter_list + else: - filter_func = ( - lambda x: utility.array_to_string(x.ImageFileName) - not in filter_list - ) + + def filter_func(x): + return utility.array_to_string(x.ImageFileName) not in filter_list + return filter_func @classmethod diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 5ce470cd8..86eb47300 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -102,29 +102,38 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): Returns: Filter function to be passed to the list of processes. """ - filter_func = lambda _: False + + def filter_func(_): + return False if offset: if physical: if exclude: - filter_func = ( - lambda proc: cls.physical_offset_from_virtual( - context, layer_name, proc + + def filter_func(proc): + return ( + cls.physical_offset_from_virtual(context, layer_name, proc) + == offset ) - == offset - ) + else: - filter_func = ( - lambda proc: cls.physical_offset_from_virtual( - context, layer_name, proc + + def filter_func(proc): + return ( + cls.physical_offset_from_virtual(context, layer_name, proc) + != offset ) - != offset - ) + else: if exclude: - filter_func = lambda proc: proc.vol.offset == offset + + def filter_func(proc): + return proc.vol.offset == offset + else: - filter_func = lambda proc: proc.vol.offset != offset + + def filter_func(proc): + return proc.vol.offset != offset return filter_func diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index 6afaf4356..3cf6d60d8 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -285,10 +285,9 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf if not shim_head: return - for shim_entry in shim_head.ListEntry.to_list( + yield from shim_head.ListEntry.to_list( shimcache_symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", "ListEntry" - ): - yield shim_entry + ) @classmethod def try_get_shim_head_at_offset( @@ -333,7 +332,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf eresource_rel_off = ersrc_size + ((offset - ersrc_size) % ersrc_alignment) eresource_offset = offset - eresource_rel_off - vollog.debug("Constructing ERESOURCE at %s" % hex(eresource_offset)) + vollog.debug(f"Constructing ERESOURCE at {hex(eresource_offset)}") eresource = context.object( kernel_symbol_table + constants.BANG + "_ERESOURCE", layer_name, diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index 7c26a09bd..ea73247ce 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -103,11 +103,10 @@ class SvcList(svcscan.SvcScan): scanner=scanners.BytesScanner(needle=b"Sc27"), sections=exe_range, ): - for record in cls.enumerate_vista_or_later_header( + yield from cls.enumerate_vista_or_later_header( context, service_table_name, service_binary_dll_map, layer_name, offset, - ): - yield record + ) diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index a34818fc1..84daa8595 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -82,5 +82,4 @@ class Threads(thrdscan.ThrdScan): symbol_table=symbol_table_name, filter_func=filter_func, ): - for thread in cls.list_threads(module, proc): - yield thread + yield from cls.list_threads(module, proc) diff --git a/volatility3/framework/plugins/windows/unloadedmodules.py b/volatility3/framework/plugins/windows/unloadedmodules.py index 01e575818..077fe33cb 100644 --- a/volatility3/framework/plugins/windows/unloadedmodules.py +++ b/volatility3/framework/plugins/windows/unloadedmodules.py @@ -116,8 +116,7 @@ class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInt ) unloadedmodules_array.UnloadedDrivers.count = unloaded_count - for mod in unloadedmodules_array.UnloadedDrivers: - yield mod + yield from unloadedmodules_array.UnloadedDrivers def _generator(self): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/virtmap.py b/volatility3/framework/plugins/windows/virtmap.py index 3f3f270e2..e02cca89e 100644 --- a/volatility3/framework/plugins/windows/virtmap.py +++ b/volatility3/framework/plugins/windows/virtmap.py @@ -138,8 +138,7 @@ class VirtMap(interfaces.plugins.PluginInterface): mapping = cls.determine_map(module) for entry in mapping: if "Unused" not in entry: - for value in mapping[entry]: - yield value + yield from mapping[entry] def run(self): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/renderers/format_hints.py b/volatility3/framework/renderers/format_hints.py index 194e38099..d57c7e9f1 100644 --- a/volatility3/framework/renderers/format_hints.py +++ b/volatility3/framework/renderers/format_hints.py @@ -70,15 +70,21 @@ class MultiTypeData(bytes): ) -BinOrAbsent = lambda x: ( - Bin(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -HexOrAbsent = lambda x: ( - Hex(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -HexBytesOrAbsent = lambda x: ( - HexBytes(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -MultiTypeDataOrAbsent = lambda x: ( - MultiTypeData(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) +def BinOrAbsent(x): + return Bin(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def HexOrAbsent(x): + return Hex(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def HexBytesOrAbsent(x): + return HexBytes(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def MultiTypeDataOrAbsent(x): + return ( + MultiTypeData(x) + if not isinstance(x, interfaces.renderers.BaseAbsentValue) + else x + ) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 537b729ad..0230a9c48 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -629,8 +629,7 @@ class IDStorage(ABC): if self.is_valid_node(nodep): yield nodep else: - for child_node in self._iter_node(nodep, height - 1): - yield child_node + yield from self._iter_node(nodep, height - 1) def get_entries(self, root: interfaces.objects.ObjectInterface) -> Iterator[int]: """Walks the tree data structure @@ -659,8 +658,7 @@ class IDStorage(ABC): if self.is_valid_node(nodep): yield nodep else: - for child_node in self._iter_node(nodep, height): - yield child_node + yield from self._iter_node(nodep, height) class XArray(IDStorage): diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e9fd09eaa..61b7b8f73 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -200,8 +200,7 @@ class module(generic.GenericIntelProcess): count=num_sects, ) - for attr in arr: - yield attr + yield from arr def get_elf_table_name(self): elf_table_name = intermed.IntermediateSymbolTable.create( @@ -237,8 +236,7 @@ class module(generic.GenericIntelProcess): count=self.num_symtab + 1, ) if self.section_strtab: - for sym in syms: - yield sym + yield from syms def get_symbols_names_and_addresses(self) -> Iterable[Tuple[str, int]]: """Get names and addresses for each symbol of the module @@ -2665,8 +2663,7 @@ class IDR(objects.StructType): id_storage = linux.IDStorage.choose_id_storage( self._context, kernel_module_name="kernel" ) - for page_addr in id_storage.get_entries(root=self.idr_rt): - yield page_addr + yield from id_storage.get_entries(root=self.idr_rt) def get_entries(self) -> Iterable[int]: """Walks the IDR and yield a pointer associated with each element. @@ -2684,8 +2681,7 @@ class IDR(objects.StructType): # Kernels < 4.11 get_entries_func = self._old_kernel_get_entries - for page_addr in get_entries_func(): - yield page_addr + yield from get_entries_func() class rb_root(objects.StructType): diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index c695ca77a..ee6dd10a3 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -232,10 +232,9 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "tqh_first", "tqe_next", next_member, max_elements - ): - yield element + ) @classmethod def walk_list_head( @@ -244,10 +243,9 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "lh_first", "le_next", next_member, max_elements - ): - yield element + ) @classmethod def walk_slist( @@ -256,7 +254,6 @@ class MacUtilities(interfaces.configuration.VersionableInterface): next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "slh_first", "sle_next", next_member, max_elements - ): - yield element + ) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 07ab5f5a8..d40df6bd6 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -749,11 +749,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() - for entry in peb.Ldr.InLoadOrderModuleList.to_list( + yield from peb.Ldr.InLoadOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InLoadOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None @@ -762,11 +761,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() - for entry in peb.Ldr.InInitializationOrderModuleList.to_list( + yield from peb.Ldr.InInitializationOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InInitializationOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None @@ -775,11 +773,10 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): try: peb = self.get_peb() - for entry in peb.Ldr.InMemoryOrderModuleList.to_list( + yield from peb.Ldr.InMemoryOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InMemoryOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index cf6f43a9b..9666fd79c 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -107,11 +107,10 @@ class EXE_ALIAS_LIST(objects.StructType): def get_aliases(self) -> Generator[interfaces.objects.ObjectInterface, None, None]: """Generator for the individual aliases for a particular executable.""" - for alias in self.AliasList.to_list( + yield from self.AliasList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_ALIAS", "ListEntry", - ): - yield alias + ) class SCREEN_INFORMATION(objects.StructType): @@ -245,11 +244,10 @@ class CONSOLE_INFORMATION(objects.StructType): def get_histories( self, ) -> Generator[interfaces.objects.ObjectInterface, None, None]: - for cmd_hist in self.HistoryList.to_list( + yield from self.HistoryList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_COMMAND_HISTORY", "ListEntry", - ): - yield cmd_hist + ) def get_exe_aliases( self, @@ -258,20 +256,18 @@ class CONSOLE_INFORMATION(objects.StructType): # Windows 10 22000 and Server 20348 made this a Pointer if isinstance(exe_alias_list, objects.Pointer): exe_alias_list = exe_alias_list.dereference() - for exe_alias_list_item in exe_alias_list.to_list( + yield from exe_alias_list.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_EXE_ALIAS_LIST", "ListEntry", - ): - yield exe_alias_list_item + ) def get_processes( self, ) -> Generator[interfaces.objects.ObjectInterface, None, None]: - for proc in self.ConsoleProcessList.to_list( + yield from self.ConsoleProcessList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_CONSOLE_PROCESS_LIST", "ListEntry", - ): - yield proc + ) def get_title(self) -> Union[str, None]: try: @@ -393,8 +389,7 @@ class COMMAND_HISTORY(objects.StructType): rest are coalesced. """ - for i, cmd in self.scan_command_bucket(self.CommandBucket.End): - yield i, cmd + yield from self.scan_command_bucket(self.CommandBucket.End) win10_x64_class_types = { diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index 4feb396e7..ea2884bb2 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -128,7 +128,10 @@ class PdbReader: self._layer_name, self._context = self.load_pdb_layer(context, location) self._dbiheader: Optional[interfaces.objects.ObjectInterface] = None if not progress_callback: - progress_callback = lambda x, y: None + + def progress_callback(x, y): + return None + self._progress_callback = progress_callback self.types: List[ Tuple[ From e708a62eefa6bc48823050850959e689f67abd9e Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 17:28:43 +0100 Subject: [PATCH 691/979] make ruff happy --- development/mac-kdk/parse_pbzx2.py | 7 ++-- development/pdbparse-to-json.py | 6 ++-- development/schema_validate.py | 4 +-- doc/source/conf.py | 12 +++---- volatility3/cli/__init__.py | 2 +- volatility3/framework/__init__.py | 16 ++++----- .../framework/configuration/__init__.py | 2 ++ volatility3/framework/constants/__init__.py | 35 ++++++++++++++++++- volatility3/framework/interfaces/__init__.py | 11 ++++++ volatility3/framework/layers/resources.py | 2 +- .../framework/layers/scanners/__init__.py | 3 ++ volatility3/framework/objects/__init__.py | 12 +++---- volatility3/framework/plugins/isfinfo.py | 9 ++--- volatility3/framework/plugins/linux/kmsg.py | 4 +-- .../framework/plugins/mac/check_sysctl.py | 2 +- .../framework/plugins/mac/kauth_scopes.py | 2 +- .../framework/plugins/windows/netscan.py | 2 +- .../framework/plugins/windows/psxview.py | 2 +- .../framework/plugins/windows/shimcachemem.py | 2 +- volatility3/framework/renderers/__init__.py | 4 +-- .../symbols/mac/extensions/__init__.py | 2 +- 21 files changed, 93 insertions(+), 48 deletions(-) diff --git a/development/mac-kdk/parse_pbzx2.py b/development/mac-kdk/parse_pbzx2.py index 173a4d648..1ca212211 100644 --- a/development/mac-kdk/parse_pbzx2.py +++ b/development/mac-kdk/parse_pbzx2.py @@ -7,6 +7,7 @@ # Cleaned up C version (as the basis for my code) here, thanks to Pepijn Bruienne / @bruienne # https://gist.github.com/bruienne/029494bbcfb358098b41 +import os import struct import sys @@ -22,7 +23,7 @@ def seekread(f, offset = None, length = 0, relative = True): def parse_pbzx(pbzx_path): section = 0 - xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) + xar_out_path = f'{pbzx_path}.part{section:02d}.cpio.xz' with open(pbzx_path, 'rb') as f: # pbzx = f.read() # f.close() @@ -50,12 +51,12 @@ def parse_pbzx(pbzx_path): # ... and split it out ... f_content = seekread(f, length = f_length) section += 1 - decomp_out = '%s.part%02d.cpio' % (pbzx_path, section) + decomp_out = f'{pbzx_path}.part{section:02d}.cpio' with open(decomp_out, 'wb') as g: g.write(f_content) # Now to start the next section, which should hopefully be .xz (we'll just assume it is ...) section += 1 - xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) + xar_out_path = f'{pbzx_path}.part{section:02d}.cpio.xz' else: f_length -= 6 # This part needs buffering diff --git a/development/pdbparse-to-json.py b/development/pdbparse-to-json.py index 6eb265227..6ffa4ea49 100644 --- a/development/pdbparse-to-json.py +++ b/development/pdbparse-to-json.py @@ -32,12 +32,12 @@ class PDBRetreiver: result = None for suffix in [file_name[:-1] + '_', file_name]: try: - logger.debug(f"Attempting to retrieve {url + suffix}") + logger.debug("Attempting to retrieve %s", url + suffix) result, _ = request.urlretrieve(url + suffix) except request.HTTPError as excp: - logger.debug(f"Failed with {excp}") + logger.debug("Failed with %s", excp) if result: - logger.debug(f"Successfully written to {result}") + logger.debug("Successfully written to %s", result) break return result diff --git a/development/schema_validate.py b/development/schema_validate.py index 031039e38..f44b3267e 100644 --- a/development/schema_validate.py +++ b/development/schema_validate.py @@ -6,7 +6,7 @@ import sys # TODO: Rather nasty hack, when volatility's actually installed this would be unnecessary sys.path += ".." -import logging +import logging # noqa: E402 console = logging.StreamHandler() console.setLevel(logging.DEBUG) @@ -17,7 +17,7 @@ logger = logging.getLogger("") logger.addHandler(console) logger.setLevel(logging.DEBUG) -from volatility3 import schemas +from volatility3 import schemas # noqa: E402 if __name__ == '__main__': parser = argparse.ArgumentParser("Validates ") diff --git a/doc/source/conf.py b/doc/source/conf.py index cabfdc327..7a9a72891 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -19,6 +19,8 @@ import sys import sphinx.ext.apidoc +from importlib.util import find_spec + def setup(app): volatility_directory = os.path.abspath( @@ -124,7 +126,7 @@ def setup(app): # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath("../..")) -from volatility3.framework import constants +from volatility3.framework import constants # noqa: E402 # -- General configuration ------------------------------------------------ @@ -147,13 +149,9 @@ extensions = [ autosectionlabel_prefix_document = True -try: - import sphinx_autodoc_typehints - +if find_spec("sphinx_autodoc_typehints") is not None: extensions.append("sphinx_autodoc_typehints") -except ImportError: - # If the autodoc typehints extension isn't available, carry on regardless - pass +# If the autodoc typehints extension isn't available, carry on regardless # Add any paths that contain templates here, relative to this directory. # templates_path = ['tools/templates'] diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index cf1335443..da046de57 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -879,7 +879,7 @@ class CommandLine: volatility3.framework.configuration.requirements.ListRequirement, ): # Allow a list of integers, specified with the convenient 0x hexadecimal format - if requirement.element_type == int: + if requirement.element_type is int: additional["type"] = lambda x: int(x, 0) else: additional["type"] = requirement.element_type diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 244b353a2..bf7d3ab74 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -22,14 +22,14 @@ if ( ) ) -import importlib -import inspect -import logging -import os -import traceback -from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar +import importlib # noqa: E402 +import inspect # noqa: E402 +import logging # noqa: E402 +import os # noqa: E402 +import traceback # noqa: E402 +from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar # noqa: E402 -from volatility3.framework import constants, interfaces +from volatility3.framework import constants, interfaces # noqa: E402 # ## @@ -74,7 +74,7 @@ class NonInheritable: self.cls = cls def __get__(self, obj: Any, get_type: Type = None) -> Any: - if type == self.cls: + if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) return self.default_value diff --git a/volatility3/framework/configuration/__init__.py b/volatility3/framework/configuration/__init__.py index 7a84ee455..6ca8b4ee3 100644 --- a/volatility3/framework/configuration/__init__.py +++ b/volatility3/framework/configuration/__init__.py @@ -3,3 +3,5 @@ # from volatility3.framework.configuration import requirements + +__all__ = ["requirements"] diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 8bdf84730..427d666e0 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -14,7 +14,7 @@ import warnings from typing import Callable, Optional import volatility3.framework.constants.linux -import volatility3.framework.constants.windows +import volatility3.framework.constants.windows # noqa: F401 from volatility3.framework.constants._version import ( PACKAGE_VERSION, VERSION_MAJOR, @@ -141,3 +141,36 @@ def __getattr__(name): return globals()[f"{deprecated_tag}{name}"] return getattr(__import__(__name__), name) + + +__all__ = [ + "PACKAGE_VERSION", + "VERSION_MAJOR", + "VERSION_MINOR", + "VERSION_PATCH", + "VERSION_SUFFIX", + "PLUGINS_PATH", + "SYMBOL_BASEPATHS", + "ISF_EXTENSIONS", + "BANG", + "AUTOMAGIC_CONFIG_PATH", + "LOGLEVEL_INFO", + "LOGLEVEL_DEBUG", + "LOGLEVEL_V", + "LOGLEVEL_VV", + "LOGLEVEL_VVV", + "LOGLEVEL_VVVV", + "CACHE_PATH", + "SQLITE_CACHE_PERIOD", + "IDENTIFIERS_FILENAME", + "CACHE_SQLITE_SCHEMA_VERSION", + "BUG_URL", + "ProgressCallback", + "OS_CATEGORIES", + "Parallelism", + "PARALLELISM", + "ISF_MINIMUM_SUPPORTED", + "ISF_MINIMUM_DEPRECATED", + "OFFLINE", + "REMOTE_ISF_URL", +] diff --git a/volatility3/framework/interfaces/__init__.py b/volatility3/framework/interfaces/__init__.py index 51d81d63a..19d11e2f4 100644 --- a/volatility3/framework/interfaces/__init__.py +++ b/volatility3/framework/interfaces/__init__.py @@ -22,3 +22,14 @@ from volatility3.framework.interfaces import ( symbols, automagic, ) + +__all__ = [ + "renderers", + "configuration", + "context", + "layers", + "objects", + "plugins", + "symbols", + "automagic", +] diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index c7a7fee67..6b17db5ff 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -29,7 +29,7 @@ except ImportError: try: # Import so that the handler is found by the framework.class_subclasses callc - import smb.SMBHandler # lgtm [py/unused-import] + import smb.SMBHandler # lgtm [py/unused-import] # noqa: F401 except ImportError: # If we fail to import this, it means that SMB handling won't be available pass diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index dd8dc46be..a36236a11 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -136,3 +136,6 @@ class MultiStringScanner(layers.ScannerInterface): ) for match in re.finditer(self._regex, haystack): yield match.start(0), match.group() + + +__all__ = ["multiregexp", "BytesScanner", "RegExScanner", "MultiStringScanner"] diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index b65277067..5846da070 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -35,13 +35,13 @@ def convert_data_to_value( data_format: DataFormatInfo, ) -> TUnion[int, float, bytes, str, bool]: """Converts a series of bytes to a particular type of value.""" - if struct_type == int: + if struct_type is int: return int.from_bytes( data, byteorder=data_format.byteorder, signed=data_format.signed ) - if struct_type == bool: + if struct_type is bool: struct_format = "?" - elif struct_type == float: + elif struct_type is float: float_vals = "zzezfzzzd" if ( data_format.length > len(float_vals) @@ -70,7 +70,7 @@ def convert_value_to_data( f"Written value is not of the correct type for {struct_type.__name__}" ) - if struct_type == int and isinstance(value, int): + if struct_type is int and isinstance(value, int): # Doubling up on the isinstance is for mypy return int.to_bytes( value, @@ -78,9 +78,9 @@ def convert_value_to_data( byteorder=data_format.byteorder, signed=data_format.signed, ) - if struct_type == bool: + if struct_type is bool: struct_format = "?" - elif struct_type == float: + elif struct_type is float: float_vals = "zzezfzzzd" if ( data_format.length > len(float_vals) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 4f07bd5a8..78e78fb9e 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -7,6 +7,7 @@ import os import pathlib import zipfile from typing import Generator, List +from importlib.util import find_spec from volatility3 import schemas, symbols from volatility3.framework import constants, interfaces, renderers @@ -96,16 +97,12 @@ class IsfInfo(plugins.PluginInterface): if filter_item in isf_file: filtered_list.append(isf_file) - try: - import jsonschema - - if not self.config["validate"]: - raise ImportError # Act as if we couldn't import if validation is turned off + if find_spec("jsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False" - except ImportError: + else: def check_valid(data): return "Unknown" diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index e26d69543..d66e3b9ca 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return "%lu.%06lu" % (nsec / 1000000000, (nsec % 1000000000) / 1000) + return f"{nsec / 1000000000:lu}.{(nsec % 1000000000) / 1000:06lu}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = "%s(%u)" % (caller_name, caller_id & ~0x80000000) + caller = f"{caller_name}({caller_id & ~0x80000000:u})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: diff --git a/volatility3/framework/plugins/mac/check_sysctl.py b/volatility3/framework/plugins/mac/check_sysctl.py index e8218962d..ed3e34aea 100644 --- a/volatility3/framework/plugins/mac/check_sysctl.py +++ b/volatility3/framework/plugins/mac/check_sysctl.py @@ -60,7 +60,7 @@ class Check_sysctl(plugins.PluginInterface): return var_str def _process_sysctl_list(self, kernel, sysctl_list, recursive=0): - if type(sysctl_list) == volatility3.framework.objects.Pointer: + if type(sysctl_list) is volatility3.framework.objects.Pointer: sysctl_list = sysctl_list.dereference().cast("sysctl_oid_list") sysctl = sysctl_list.slh_first diff --git a/volatility3/framework/plugins/mac/kauth_scopes.py b/volatility3/framework/plugins/mac/kauth_scopes.py index afb320a07..c2c473eac 100644 --- a/volatility3/framework/plugins/mac/kauth_scopes.py +++ b/volatility3/framework/plugins/mac/kauth_scopes.py @@ -80,7 +80,7 @@ class Kauth_scopes(interfaces.plugins.PluginInterface): ( identifier, format_hints.Hex(scope.ks_idata), - len([l for l in scope.get_listeners()]), + len([listener for listener in scope.get_listeners()]), format_hints.Hex(callback), module_name, symbol_name, diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 77bd22ab9..dd8e4b133 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -161,7 +161,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): raise NotImplementedError( "Kernel Debug Structure version format not supported!" ) - except: + except: # noqa: E722 # unsure what to raise here. Also, it might be useful to add some kind of fallback, # either to a user-provided version or to another method to determine tcpip.sys's version raise exceptions.VolatilityException( diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index 053ec20d5..e3ec216dd 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -218,7 +218,7 @@ class PsXView(plugins.PluginInterface): name = self._proc_name_to_string(proc) exit_time = proc.get_exit_time() - if type(exit_time) != datetime.datetime: + if type(exit_time) is not datetime.datetime: exit_time = "" else: exit_time = str(exit_time) diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index 3cf6d60d8..59f33510d 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -146,7 +146,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf context, layer_name, kernel_symbol_table ): pid = process.UniqueProcessId - vollog.debug("checking process %d" % pid) + vollog.debug("checking process %d", pid) for vad in vadinfo.VadInfo.list_vads( process, lambda x: x.get_tag() == b"Vad " and x.Protection == 4 ): diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 02805acc2..39ce1135d 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -430,10 +430,10 @@ class ColumnSortKey(interfaces.renderers.ColumnSortKey): value = datetime.datetime.min elif self._type in [int, float]: value = -1 - elif self._type == bool: + elif self._type is bool: value = False elif self._type in [str, renderers.Disassembly]: value = "-" - elif self._type == bytes: + elif self._type is bytes: value = b"" return value diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 15fe7aeda..d2573fb95 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -237,7 +237,7 @@ class vm_map_entry(objects.StructType): def get_path(self, context, config_prefix): node = self.get_vnode(context, config_prefix) - if type(node) == str and node == "sub_map": + if type(node) is str and node == "sub_map": ret = node elif node: path = [] From 75a324ddba73462d13362ab3f0f5bbdfeb0a3d33 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 18:04:22 +0100 Subject: [PATCH 692/979] chore(developement/schema_validate): move logging import up --- development/schema_validate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/development/schema_validate.py b/development/schema_validate.py index f44b3267e..0ea82d537 100644 --- a/development/schema_validate.py +++ b/development/schema_validate.py @@ -1,13 +1,12 @@ import argparse import json +import logging import os import sys # TODO: Rather nasty hack, when volatility's actually installed this would be unnecessary sys.path += ".." -import logging # noqa: E402 - console = logging.StreamHandler() console.setLevel(logging.DEBUG) formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') From 2e093c1a53520c12c209d49cb108b654a2cc2290 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 18:08:22 +0100 Subject: [PATCH 693/979] chore(framework/automagic/linux): convert lambda to function --- volatility3/framework/automagic/linux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 6b58577a3..93131a120 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -196,5 +196,8 @@ class LinuxSymbolFinder(symbol_finder.SymbolFinder): banner_config_key = "kernel_banner" operating_system = "linux" symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" - find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] exclusion_list = ["mac", "windows"] + + @classmethod + def find_aslr(cls, *args): + return LinuxIntelStacker.find_aslr(*args)[1] From c33e378c847b3cddcf3730f8b6ea5fc5efa8384e Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 18:18:54 +0100 Subject: [PATCH 694/979] chore(framework/__init__): move up imports --- volatility3/framework/__init__.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index bf7d3ab74..364e6f5ec 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -6,6 +6,14 @@ import glob import sys import zipfile +import importlib +import inspect +import logging +import os +import traceback +from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar + +from volatility3.framework import constants, interfaces required_python_version = (3, 8, 0) if ( @@ -22,15 +30,6 @@ if ( ) ) -import importlib # noqa: E402 -import inspect # noqa: E402 -import logging # noqa: E402 -import os # noqa: E402 -import traceback # noqa: E402 -from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar # noqa: E402 - -from volatility3.framework import constants, interfaces # noqa: E402 - # ## # From 2119b8c7b820e3fde75f9f4871cb7504ef0b5473 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 18:37:46 +0100 Subject: [PATCH 695/979] fix framework/__init__ --- volatility3/framework/__init__.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 364e6f5ec..df8d2d2b8 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,15 +5,6 @@ # Check the python version to ensure it's suitable import glob import sys -import zipfile -import importlib -import inspect -import logging -import os -import traceback -from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar - -from volatility3.framework import constants, interfaces required_python_version = (3, 8, 0) if ( @@ -25,10 +16,18 @@ if ( ) ): raise RuntimeError( - "Volatility framework requires python version {}.{}.{} or greater".format( - *required_python_version - ) + f"Volatility framework requires python version {'.'.join(map(str, required_python_version))} or greater" ) +else: + import zipfile + import importlib + import inspect + import logging + import os + import traceback + from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar + + from volatility3.framework import constants, interfaces # ## From ccd3e8f367df22d60eb5be300a0c8a830b8db59a Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 18:43:27 +0100 Subject: [PATCH 696/979] move python version check into its own module --- volatility3/framework/__init__.py | 31 ++++++------------- volatility3/framework/check_python_version.py | 14 +++++++++ 2 files changed, 23 insertions(+), 22 deletions(-) create mode 100644 volatility3/framework/check_python_version.py diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index df8d2d2b8..bf71c3c99 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,29 +5,16 @@ # Check the python version to ensure it's suitable import glob import sys +import volatility3.framework.check_python_version # noqa: F401 +import zipfile +import importlib +import inspect +import logging +import os +import traceback +from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar -required_python_version = (3, 8, 0) -if ( - sys.version_info.major != required_python_version[0] - or sys.version_info.minor < required_python_version[1] - or ( - sys.version_info.minor == required_python_version[1] - and sys.version_info.micro < required_python_version[2] - ) -): - raise RuntimeError( - f"Volatility framework requires python version {'.'.join(map(str, required_python_version))} or greater" - ) -else: - import zipfile - import importlib - import inspect - import logging - import os - import traceback - from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar - - from volatility3.framework import constants, interfaces +from volatility3.framework import constants, interfaces # ## diff --git a/volatility3/framework/check_python_version.py b/volatility3/framework/check_python_version.py new file mode 100644 index 000000000..f2d284f2a --- /dev/null +++ b/volatility3/framework/check_python_version.py @@ -0,0 +1,14 @@ +import sys + +required_python_version = (3, 8, 0) +if ( + sys.version_info.major != required_python_version[0] + or sys.version_info.minor < required_python_version[1] + or ( + sys.version_info.minor == required_python_version[1] + and sys.version_info.micro < required_python_version[2] + ) +): + raise RuntimeError( + f"Volatility framework requires python version {required_python_version[0]}.{required_python_version[1]}.{required_python_version[2]} or greater" + ) From 53133f4478bab042d96c468d5fd14c95f96b9913 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 21:27:38 +0100 Subject: [PATCH 697/979] fix codeQL error --- volatility3/framework/plugins/windows/modules.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index 00424938f..3c0f5ab7c 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -104,12 +104,11 @@ class Modules(interfaces.plugins.PluginInterface): try: BaseDllName = mod.BaseDllName.get_string() + if self.config["name"] and self.config["name"] not in BaseDllName: + continue except exceptions.InvalidAddressException: BaseDllName = interfaces.renderers.BaseAbsentValue() - if self.config["name"] and self.config["name"] not in BaseDllName: - continue - try: FullDllName = mod.FullDllName.get_string() except exceptions.InvalidAddressException: From 1c0d0b086adba32763668f2b06dd1b659163d350 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Fri, 22 Nov 2024 23:07:42 +0100 Subject: [PATCH 698/979] use redundant import aliases instead of __all__ and noqa's --- volatility3/framework/__init__.py | 2 +- .../framework/configuration/__init__.py | 4 +- volatility3/framework/constants/__init__.py | 47 +++---------------- volatility3/framework/interfaces/__init__.py | 27 ++++------- .../framework/layers/scanners/__init__.py | 5 +- 5 files changed, 18 insertions(+), 67 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index bf71c3c99..b60ae5576 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,7 +5,7 @@ # Check the python version to ensure it's suitable import glob import sys -import volatility3.framework.check_python_version # noqa: F401 +from volatility3.framework import check_python_version as check_python_version import zipfile import importlib import inspect diff --git a/volatility3/framework/configuration/__init__.py b/volatility3/framework/configuration/__init__.py index 6ca8b4ee3..7b914cf16 100644 --- a/volatility3/framework/configuration/__init__.py +++ b/volatility3/framework/configuration/__init__.py @@ -2,6 +2,4 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework.configuration import requirements - -__all__ = ["requirements"] +from volatility3.framework.configuration import requirements as requirements diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 427d666e0..23cc2dde5 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -13,14 +13,14 @@ import sys import warnings from typing import Callable, Optional -import volatility3.framework.constants.linux -import volatility3.framework.constants.windows # noqa: F401 +from volatility3.framework.constants import linux as linux +from volatility3.framework.constants import windows as windows from volatility3.framework.constants._version import ( - PACKAGE_VERSION, - VERSION_MAJOR, - VERSION_MINOR, - VERSION_PATCH, - VERSION_SUFFIX, + PACKAGE_VERSION as PACKAGE_VERSION, + VERSION_MAJOR as VERSION_MAJOR, + VERSION_MINOR as VERSION_MINOR, + VERSION_PATCH as VERSION_PATCH, + VERSION_SUFFIX as VERSION_SUFFIX, ) PLUGINS_PATH = [ @@ -141,36 +141,3 @@ def __getattr__(name): return globals()[f"{deprecated_tag}{name}"] return getattr(__import__(__name__), name) - - -__all__ = [ - "PACKAGE_VERSION", - "VERSION_MAJOR", - "VERSION_MINOR", - "VERSION_PATCH", - "VERSION_SUFFIX", - "PLUGINS_PATH", - "SYMBOL_BASEPATHS", - "ISF_EXTENSIONS", - "BANG", - "AUTOMAGIC_CONFIG_PATH", - "LOGLEVEL_INFO", - "LOGLEVEL_DEBUG", - "LOGLEVEL_V", - "LOGLEVEL_VV", - "LOGLEVEL_VVV", - "LOGLEVEL_VVVV", - "CACHE_PATH", - "SQLITE_CACHE_PERIOD", - "IDENTIFIERS_FILENAME", - "CACHE_SQLITE_SCHEMA_VERSION", - "BUG_URL", - "ProgressCallback", - "OS_CATEGORIES", - "Parallelism", - "PARALLELISM", - "ISF_MINIMUM_SUPPORTED", - "ISF_MINIMUM_DEPRECATED", - "OFFLINE", - "REMOTE_ISF_URL", -] diff --git a/volatility3/framework/interfaces/__init__.py b/volatility3/framework/interfaces/__init__.py index 19d11e2f4..fd6b1e062 100644 --- a/volatility3/framework/interfaces/__init__.py +++ b/volatility3/framework/interfaces/__init__.py @@ -13,23 +13,12 @@ components of volatility to write plugins. # This will also avoid namespace issues, because people can use interfaces.layers to # avoid clashing with the layers package from volatility3.framework.interfaces import ( - renderers, - configuration, - context, - layers, - objects, - plugins, - symbols, - automagic, + renderers as renderers, + configuration as configuration, + context as context, + layers as layers, + objects as objects, + plugins as plugins, + symbols as symbols, + automagic as automagic, ) - -__all__ = [ - "renderers", - "configuration", - "context", - "layers", - "objects", - "plugins", - "symbols", - "automagic", -] diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index a36236a11..f54b44ff4 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -5,7 +5,7 @@ import re from typing import Generator, List, Tuple, Dict, Optional from volatility3.framework.interfaces import layers -from volatility3.framework.layers.scanners import multiregexp +from volatility3.framework.layers.scanners import multiregexp as multiregexp class BytesScanner(layers.ScannerInterface): @@ -136,6 +136,3 @@ class MultiStringScanner(layers.ScannerInterface): ) for match in re.finditer(self._regex, haystack): yield match.start(0), match.group() - - -__all__ = ["multiregexp", "BytesScanner", "RegExScanner", "MultiStringScanner"] From 6283ab6a1a7d871eacb1c7fc9e3d71f9333f4bab Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 11:26:25 +0100 Subject: [PATCH 699/979] adjust SMBHandler import to use redundat alias --- volatility3/framework/layers/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 6b17db5ff..dc510452a 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -29,7 +29,7 @@ except ImportError: try: # Import so that the handler is found by the framework.class_subclasses callc - import smb.SMBHandler # lgtm [py/unused-import] # noqa: F401 + from smb import SMBHandler as SMBHandler except ImportError: # If we fail to import this, it means that SMB handling won't be available pass From d95c32b25a65e87fc1a3a57f46dafa628d0b5568 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 11:27:41 +0100 Subject: [PATCH 700/979] run `ruff check --fix` --- volatility3/framework/automagic/linux.py | 2 -- volatility3/framework/automagic/mac.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 93131a120..c044fdd93 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -3,12 +3,10 @@ # import logging -import os from typing import Optional, Tuple, Type from volatility3.framework import constants, interfaces from volatility3.framework.automagic import symbol_cache, symbol_finder -from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import linux diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 89dd5a187..f3679d160 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -3,13 +3,11 @@ # import logging -import os import struct from typing import Optional from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.automagic import symbol_cache, symbol_finder -from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import mac From 0b4d595bac567b478c343a9a9f18957e0270c410 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 11:35:33 +0100 Subject: [PATCH 701/979] don't use bare except in windows netscan plugin --- volatility3/framework/plugins/windows/netscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index dd8e4b133..9462df43d 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -161,7 +161,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): raise NotImplementedError( "Kernel Debug Structure version format not supported!" ) - except: # noqa: E722 + except Exception: # unsure what to raise here. Also, it might be useful to add some kind of fallback, # either to a user-provided version or to another method to determine tcpip.sys's version raise exceptions.VolatilityException( From 8dc9bd037d67d0dd3805ba3421a19d8b8a76304f Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 11:39:08 +0100 Subject: [PATCH 702/979] add fixme to windows/netscan plugin --- volatility3/framework/plugins/windows/netscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 9462df43d..162031104 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -162,7 +162,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): "Kernel Debug Structure version format not supported!" ) except Exception: - # unsure what to raise here. Also, it might be useful to add some kind of fallback, + # FIXME: unsure what to raise here. Also, it might be useful to add some kind of fallback, # either to a user-provided version or to another method to determine tcpip.sys's version raise exceptions.VolatilityException( "Kernel Debug Structure missing VERSION/KUSER structure, unable to determine Windows version!" From eee015033ac3a7b17742ea149d3f3e01b9a7698e Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 12:30:44 +0100 Subject: [PATCH 703/979] add back lgtm imperative --- volatility3/framework/layers/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index dc510452a..00215f624 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -29,7 +29,7 @@ except ImportError: try: # Import so that the handler is found by the framework.class_subclasses callc - from smb import SMBHandler as SMBHandler + from smb import SMBHandler as SMBHandler # lgtm [py/unused-import] except ImportError: # If we fail to import this, it means that SMB handling won't be available pass From f6c852c478e5bf676cbc3def50900f8af66679b5 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 13:04:13 +0100 Subject: [PATCH 704/979] format with black --- development/banner_server.py | 45 ++- development/compare-vol.py | 340 ++++++++++++------- development/mac-kdk/parse_pbzx2.py | 50 +-- development/pdbparse-to-json.py | 188 ++++++---- development/schema_validate.py | 8 +- development/stock-linux-json.py | 98 +++--- test/plugins/windows/test_scheduled_tasks.py | 2 + volatility3/framework/layers/resources.py | 2 +- 8 files changed, 462 insertions(+), 271 deletions(-) diff --git a/development/banner_server.py b/development/banner_server.py index 3aea41c82..b62477a26 100644 --- a/development/banner_server.py +++ b/development/banner_server.py @@ -28,10 +28,10 @@ class BannerCacheGenerator: def run(self): context = contexts.Context() - json_output = {'version': 1} + json_output = {"version": 1} path = self._path - filename = '*' + filename = "*" for banner_cache in [linux.LinuxBannerCache, mac.MacBannerCache]: sub_path = banner_cache.os @@ -39,37 +39,54 @@ class BannerCacheGenerator: for extension in constants.ISF_EXTENSIONS: # Hopefully these will not be large lists, otherwise this might be slow try: - for found in pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + extension): + for found in ( + pathlib.Path(path) + .joinpath(sub_path) + .resolve() + .rglob(filename + extension) + ): potentials.append(found.as_uri()) except FileNotFoundError: # If there's no linux symbols, don't cry about it pass - new_banners = banner_cache.read_new_banners(context, 'BannerServer', potentials, banner_cache.symbol_name, - banner_cache.os, progress_callback = PrintedProgress()) + new_banners = banner_cache.read_new_banners( + context, + "BannerServer", + potentials, + banner_cache.symbol_name, + banner_cache.os, + progress_callback=PrintedProgress(), + ) result_banners = {} for new_banner in new_banners: # Only accept file schemes - value = [self.convert_url(url) for url in new_banners[new_banner] if - urllib.parse.urlparse(url).scheme == 'file'] + value = [ + self.convert_url(url) + for url in new_banners[new_banner] + if urllib.parse.urlparse(url).scheme == "file" + ] if value and new_banner: # Convert files into URLs - result_banners[str(base64.b64encode(new_banner), 'latin-1')] = value + result_banners[str(base64.b64encode(new_banner), "latin-1")] = value json_output[banner_cache.os] = result_banners - output_path = os.path.join(self._path, 'banners.json') - with open(output_path, 'w') as fp: + output_path = os.path.join(self._path, "banners.json") + with open(output_path, "w") as fp: vollog.warning(f"Banners file written to {output_path}") json.dump(json_output, fp) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--path', default = os.path.dirname(__file__)) - parser.add_argument('--urlprefix', help = 'Web prefix that will eventually serve the ISF files', - default = 'http://localhost/symbols') + parser.add_argument("--path", default=os.path.dirname(__file__)) + parser.add_argument( + "--urlprefix", + help="Web prefix that will eventually serve the ISF files", + default="http://localhost/symbols", + ) args = parser.parse_args() diff --git a/development/compare-vol.py b/development/compare-vol.py index 1074c5d9c..a01d8e93c 100644 --- a/development/compare-vol.py +++ b/development/compare-vol.py @@ -15,17 +15,17 @@ class VolatilityImage: filepath: str = "" vol2_profile: str = "" vol2_imageinfo_time: float = None - vol2_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) - vol3_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) - rekall_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) + vol2_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) + vol3_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) + rekall_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) @dataclass class VolatilityPlugin: name: str = "" - vol2_plugin_parameters: List[str] = field(default_factory = list) - vol3_plugin_parameters: List[str] = field(default_factory = list) - rekall_plugin_parameters: List[str] = field(default_factory = list) + vol2_plugin_parameters: List[str] = field(default_factory=list) + vol3_plugin_parameters: List[str] = field(default_factory=list) + rekall_plugin_parameters: List[str] = field(default_factory=list) class VolatilityTest: @@ -39,32 +39,50 @@ class VolatilityTest: def result_titles(self) -> List[str]: return [self.long_name] - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> None: + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> None: pass - def create_results(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> List[float]: + def create_results( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> List[float]: self.create_prerequisites(plugin, image, image_hash) # Volatility 2 Test - print(f"[*] Testing {self.short_name} {plugin.name} with image {image.filepath}") + print( + f"[*] Testing {self.short_name} {plugin.name} with image {image.filepath}" + ) os.chdir(self.path) cmd = self.plugin_cmd(plugin, image) start_time = time.perf_counter() try: - completed = subprocess.run(cmd, cwd = self.path, capture_output = True, timeout = 420) + completed = subprocess.run( + cmd, cwd=self.path, capture_output=True, timeout=420 + ) except subprocess.TimeoutExpired as excp: completed = excp end_time = time.perf_counter() total_time = end_time - start_time - print(f" Tested {self.short_name} {plugin.name} with image {image.filepath}: {total_time}") + print( + f" Tested {self.short_name} {plugin.name} with image {image.filepath}: {total_time}" + ) with open( - os.path.join(self.output_directory, f'{self.short_name}_{plugin.name}_{image_hash}_stdout'), - "wb") as f: + os.path.join( + self.output_directory, + f"{self.short_name}_{plugin.name}_{image_hash}_stdout", + ), + "wb", + ) as f: f.write(completed.stdout) if completed.stderr: with open( - os.path.join(self.output_directory, f'{self.short_name}_{plugin.name}_{image_hash}_stderr'), - "wb") as f: + os.path.join( + self.output_directory, + f"{self.short_name}_{plugin.name}_{image_hash}_stderr", + ), + "wb", + ) as f: f.write(completed.stderr) return [total_time] @@ -77,31 +95,57 @@ class Volatility2Test(VolatilityTest): long_name = "Volatility 2" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage): - return ["python2", "-u", "vol.py", "-f", image.filepath, "--profile", image.vol2_profile - ] + plugin.vol2_plugin_parameters + image.vol2_plugin_parameters.get(plugin.name, []) + return ( + [ + "python2", + "-u", + "vol.py", + "-f", + image.filepath, + "--profile", + image.vol2_profile, + ] + + plugin.vol2_plugin_parameters + + image.vol2_plugin_parameters.get(plugin.name, []) + ) def result_titles(self): return [self.long_name, "Imageinfo", f"{self.long_name} + Imageinfo"] - def create_results(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash) -> List[float]: + def create_results( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash + ) -> List[float]: result = super().create_results(plugin, image, image_hash) result += [image.vol2_imageinfo_time, result[0] + image.vol2_imageinfo_time] return result - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash): + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash + ): # Volatility 2 image info if not image.vol2_profile: - print(f"[*] Testing {self.short_name} imageinfo with image {image.filepath}") + print( + f"[*] Testing {self.short_name} imageinfo with image {image.filepath}" + ) os.chdir(self.path) cmd = ["python2", "-u", "vol.py", "-f", image.filepath, "imageinfo"] start_time = time.perf_counter() - vol2_completed = subprocess.run(cmd, cwd = self.path, capture_output = True) + vol2_completed = subprocess.run(cmd, cwd=self.path, capture_output=True) end_time = time.perf_counter() image.vol2_imageinfo_time = end_time - start_time - print(f" Tested volatility2 imageinfo with image {image.filepath}: {end_time - start_time}") - with open(os.path.join(self.output_directory, f'vol2_imageinfo_{image_hash}_stdout'), "wb") as f: + print( + f" Tested volatility2 imageinfo with image {image.filepath}: {end_time - start_time}" + ) + with open( + os.path.join( + self.output_directory, f"vol2_imageinfo_{image_hash}_stdout" + ), + "wb", + ) as f: f.write(vol2_completed.stdout) - image.vol2_profile = re.search(rb"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout)[1] + image.vol2_profile = re.search( + rb"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout + )[1] class RekallTest(VolatilityTest): @@ -113,11 +157,16 @@ class RekallTest(VolatilityTest): plugin.rekall_plugin_parameters = plugin.vol2_plugin_parameters if not image.rekall_plugin_parameters: image.rekall_plugin_parameters = image.vol2_plugin_parameters - return ["rekall", "-f", image.filepath] + plugin.rekall_plugin_parameters + image.rekall_plugin_parameters.get( - plugin.name, []) + return ( + ["rekall", "-f", image.filepath] + + plugin.rekall_plugin_parameters + + image.rekall_plugin_parameters.get(plugin.name, []) + ) - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> None: - shutil.rmtree('/home/mike/.rekall_cache/sessions') + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> None: + shutil.rmtree("/home/mike/.rekall_cache/sessions") class Volatility3Test(VolatilityTest): @@ -125,14 +174,18 @@ class Volatility3Test(VolatilityTest): long_name = "Volatility 3" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage) -> List[str]: - return [ - "python", - "-u", - "vol.py", - "-q", - "-f", - image.filepath, - ] + plugin.vol3_plugin_parameters + image.vol3_plugin_parameters.get(plugin.name, []) + return ( + [ + "python", + "-u", + "vol.py", + "-q", + "-f", + image.filepath, + ] + + plugin.vol3_plugin_parameters + + image.vol3_plugin_parameters.get(plugin.name, []) + ) class Volatility3PyPyTest(VolatilityTest): @@ -140,26 +193,32 @@ class Volatility3PyPyTest(VolatilityTest): long_name = "Volatility 3 (PyPy)" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage) -> List[str]: - return [ - "pypy3", - "-u", - "vol.py", - "-q", - "-f", - image.filepath, - ] + plugin.vol3_plugin_parameters + image.vol3_plugin_parameters.get(plugin.name, []) + return ( + [ + "pypy3", + "-u", + "vol.py", + "-q", + "-f", + image.filepath, + ] + + plugin.vol3_plugin_parameters + + image.vol3_plugin_parameters.get(plugin.name, []) + ) class VolatilityTester: - def __init__(self, - images: List[VolatilityImage], - plugins: List[VolatilityPlugin], - frameworks: List[str], - output_dir: str, - vol2_path: str = None, - vol3_path: str = None, - rekall_path = None): + def __init__( + self, + images: List[VolatilityImage], + plugins: List[VolatilityPlugin], + frameworks: List[str], + output_dir: str, + vol2_path: str = None, + vol3_path: str = None, + rekall_path=None, + ): self.images = images self.plugins = plugins if not vol2_path: @@ -172,7 +231,7 @@ class VolatilityTester: Volatility3Test(vol3_path, output_dir), Volatility3PyPyTest(vol3_path, output_dir), Volatility2Test(vol2_path, output_dir), - RekallTest(rekall_path, output_dir) + RekallTest(rekall_path, output_dir), ] self.tests = [x for x in available_tests if x.short_name.lower() in frameworks] self.csv_writer = None @@ -183,7 +242,7 @@ class VolatilityTester: print(f"[?] Frameworks: {[x.long_name for x in self.tests]}") def run_tests(self): - with open("volatility-timings.csv", 'w') as csvfile: + with open("volatility-timings.csv", "w") as csvfile: self.csv_writer = csv.writer(csvfile) titles = ["Image Hash", "Image Path", "Plugin Name"] for test in self.tests: @@ -203,72 +262,121 @@ class VolatilityTester: self.csv_writer.writerow([image_hash, image.filepath, plugin.name] + results) -if __name__ == '__main__': +if __name__ == "__main__": plugins = [ - VolatilityPlugin(name = "pslist", - vol2_plugin_parameters = ["pslist"], - vol3_plugin_parameters = ["windows.pslist"]), - VolatilityPlugin(name = "psscan", - vol2_plugin_parameters = ["psscan"], - vol3_plugin_parameters = ["windows.psscan"], - rekall_plugin_parameters = ["psscan", "--scan_kernel"]), - VolatilityPlugin(name = "driverscan", - vol2_plugin_parameters = ["driverscan"], - vol3_plugin_parameters = ["windows.driverscan"], - rekall_plugin_parameters = ["driverscan", "--scan_kernel"]), - VolatilityPlugin(name = "handles", - vol2_plugin_parameters = ["handles"], - vol3_plugin_parameters = ["windows.handles"]), - VolatilityPlugin(name = "modules", - vol2_plugin_parameters = ["modules"], - vol3_plugin_parameters = ["windows.modules"]), - VolatilityPlugin(name = "hivelist", - vol2_plugin_parameters = ["hivelist"], - vol3_plugin_parameters = ["registry.hivelist"], - rekall_plugin_parameters = ["hives"]), - VolatilityPlugin(name = "vadinfo", - vol2_plugin_parameters = ["vadinfo"], - vol3_plugin_parameters = ["windows.vadinfo"], - rekall_plugin_parameters = ["vad"]), - VolatilityPlugin(name = "modscan", - vol2_plugin_parameters = ["modscan"], - vol3_plugin_parameters = ["windows.modscan"], - rekall_plugin_parameters = ["modscan", "--scan_kernel"]), - VolatilityPlugin(name = "svcscan", - vol2_plugin_parameters = ["svcscan"], - vol3_plugin_parameters = ["windows.svcscan"], - rekall_plugin_parameters = ["svcscan"]), - VolatilityPlugin(name = "ssdt", vol2_plugin_parameters = ["ssdt"], vol3_plugin_parameters = ["windows.ssdt"]), - VolatilityPlugin(name = "printkey", - vol2_plugin_parameters = ["printkey", "-K", "Classes"], - vol3_plugin_parameters = ["registry.printkey", "--key", "Classes"], - rekall_plugin_parameters = ["printkey", "--key", "Classes"]) + VolatilityPlugin( + name="pslist", + vol2_plugin_parameters=["pslist"], + vol3_plugin_parameters=["windows.pslist"], + ), + VolatilityPlugin( + name="psscan", + vol2_plugin_parameters=["psscan"], + vol3_plugin_parameters=["windows.psscan"], + rekall_plugin_parameters=["psscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="driverscan", + vol2_plugin_parameters=["driverscan"], + vol3_plugin_parameters=["windows.driverscan"], + rekall_plugin_parameters=["driverscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="handles", + vol2_plugin_parameters=["handles"], + vol3_plugin_parameters=["windows.handles"], + ), + VolatilityPlugin( + name="modules", + vol2_plugin_parameters=["modules"], + vol3_plugin_parameters=["windows.modules"], + ), + VolatilityPlugin( + name="hivelist", + vol2_plugin_parameters=["hivelist"], + vol3_plugin_parameters=["registry.hivelist"], + rekall_plugin_parameters=["hives"], + ), + VolatilityPlugin( + name="vadinfo", + vol2_plugin_parameters=["vadinfo"], + vol3_plugin_parameters=["windows.vadinfo"], + rekall_plugin_parameters=["vad"], + ), + VolatilityPlugin( + name="modscan", + vol2_plugin_parameters=["modscan"], + vol3_plugin_parameters=["windows.modscan"], + rekall_plugin_parameters=["modscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="svcscan", + vol2_plugin_parameters=["svcscan"], + vol3_plugin_parameters=["windows.svcscan"], + rekall_plugin_parameters=["svcscan"], + ), + VolatilityPlugin( + name="ssdt", + vol2_plugin_parameters=["ssdt"], + vol3_plugin_parameters=["windows.ssdt"], + ), + VolatilityPlugin( + name="printkey", + vol2_plugin_parameters=["printkey", "-K", "Classes"], + vol3_plugin_parameters=["registry.printkey", "--key", "Classes"], + rekall_plugin_parameters=["printkey", "--key", "Classes"], + ), ] parser = argparse.ArgumentParser() - parser.add_argument("--output-dir", type = str, default = os.getcwd(), help = "Directory to store all results") - parser.add_argument("--vol3path", - type = str, - default = os.path.join(os.getcwd(), 'volatility3'), - help = "Path ot the volatility 3 directory") - parser.add_argument("--vol2path", - type = str, - default = os.path.join(os.getcwd(), 'volatility'), - help = "Path to the volatility 2 directory") - parser.add_argument("--rekallpath", - type = str, - default = os.path.join(os.getcwd(), 'rekall'), - help = "Path to the rekall directory") - parser.add_argument("--frameworks", - nargs = "+", - type = str, - choices = [x.short_name.lower() for x in VolatilityTest.__subclasses__()], - default = [x.short_name.lower() for x in VolatilityTest.__subclasses__()], - help = "A comma separated list of frameworks to test") - parser.add_argument('images', metavar = 'IMAGE', type = str, nargs = '+', help = 'The list of images to compare') + parser.add_argument( + "--output-dir", + type=str, + default=os.getcwd(), + help="Directory to store all results", + ) + parser.add_argument( + "--vol3path", + type=str, + default=os.path.join(os.getcwd(), "volatility3"), + help="Path ot the volatility 3 directory", + ) + parser.add_argument( + "--vol2path", + type=str, + default=os.path.join(os.getcwd(), "volatility"), + help="Path to the volatility 2 directory", + ) + parser.add_argument( + "--rekallpath", + type=str, + default=os.path.join(os.getcwd(), "rekall"), + help="Path to the rekall directory", + ) + parser.add_argument( + "--frameworks", + nargs="+", + type=str, + choices=[x.short_name.lower() for x in VolatilityTest.__subclasses__()], + default=[x.short_name.lower() for x in VolatilityTest.__subclasses__()], + help="A comma separated list of frameworks to test", + ) + parser.add_argument( + "images", + metavar="IMAGE", + type=str, + nargs="+", + help="The list of images to compare", + ) args = parser.parse_args() - vt = VolatilityTester([VolatilityImage(filepath = x) for x in args.images], plugins, - [x.lower() for x in args.frameworks], args.output_dir, args.vol2path, args.vol3path, - args.rekallpath) + vt = VolatilityTester( + [VolatilityImage(filepath=x) for x in args.images], + plugins, + [x.lower() for x in args.frameworks], + args.output_dir, + args.vol2path, + args.vol3path, + args.rekallpath, + ) vt.run_tests() diff --git a/development/mac-kdk/parse_pbzx2.py b/development/mac-kdk/parse_pbzx2.py index 1ca212211..b175539b3 100644 --- a/development/mac-kdk/parse_pbzx2.py +++ b/development/mac-kdk/parse_pbzx2.py @@ -12,7 +12,7 @@ import struct import sys -def seekread(f, offset = None, length = 0, relative = True): +def seekread(f, offset=None, length=0, relative=True): if offset is not None: # offset provided, let's seek f.seek(offset, [0, 1, 2][relative]) @@ -23,55 +23,57 @@ def seekread(f, offset = None, length = 0, relative = True): def parse_pbzx(pbzx_path): section = 0 - xar_out_path = f'{pbzx_path}.part{section:02d}.cpio.xz' - with open(pbzx_path, 'rb') as f: + xar_out_path = f"{pbzx_path}.part{section:02d}.cpio.xz" + with open(pbzx_path, "rb") as f: # pbzx = f.read() # f.close() - magic = seekread(f, length = 4) - if magic != 'pbzx': + magic = seekread(f, length=4) + if magic != "pbzx": raise RuntimeError("Error: Not a pbzx file") # Read 8 bytes for initial flags - flags = seekread(f, length = 8) + flags = seekread(f, length=8) # Interpret the flags as a 64-bit big-endian unsigned int - flags = struct.unpack('>Q', flags)[0] + flags = struct.unpack(">Q", flags)[0] while flags & (1 << 24): - with open(xar_out_path, 'wb') as xar_f: + with open(xar_out_path, "wb") as xar_f: xar_f.seek(0, os.SEEK_END) # Read in more flags - flags = seekread(f, length = 8) - flags = struct.unpack('>Q', flags)[0] + flags = seekread(f, length=8) + flags = struct.unpack(">Q", flags)[0] # Read in length - f_length = seekread(f, length = 8) - f_length = struct.unpack('>Q', f_length)[0] - xzmagic = seekread(f, length = 6) - if xzmagic != '\xfd7zXZ\x00': + f_length = seekread(f, length=8) + f_length = struct.unpack(">Q", f_length)[0] + xzmagic = seekread(f, length=6) + if xzmagic != "\xfd7zXZ\x00": # This isn't xz content, this is actually _raw decompressed cpio_ chunk of 16MB in size... # Let's back up ... - seekread(f, offset = -6, length = 0) + seekread(f, offset=-6, length=0) # ... and split it out ... - f_content = seekread(f, length = f_length) + f_content = seekread(f, length=f_length) section += 1 - decomp_out = f'{pbzx_path}.part{section:02d}.cpio' - with open(decomp_out, 'wb') as g: + decomp_out = f"{pbzx_path}.part{section:02d}.cpio" + with open(decomp_out, "wb") as g: g.write(f_content) # Now to start the next section, which should hopefully be .xz (we'll just assume it is ...) section += 1 - xar_out_path = f'{pbzx_path}.part{section:02d}.cpio.xz' + xar_out_path = f"{pbzx_path}.part{section:02d}.cpio.xz" else: f_length -= 6 # This part needs buffering - f_content = seekread(f, length = f_length) - tail = seekread(f, offset = -2, length = 2) + f_content = seekread(f, length=f_length) + tail = seekread(f, offset=-2, length=2) xar_f.write(xzmagic) xar_f.write(f_content) - if tail != 'YZ': + if tail != "YZ": raise RuntimeError("Error: Footer is not xar file footer") def main(): parse_pbzx(sys.argv[1]) - print("Now xz decompress the .xz chunks, then 'cat' them all together in order into a single new.cpio file") + print( + "Now xz decompress the .xz chunks, then 'cat' them all together in order into a single new.cpio file" + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/development/pdbparse-to-json.py b/development/pdbparse-to-json.py index 6ffa4ea49..49b4da009 100644 --- a/development/pdbparse-to-json.py +++ b/development/pdbparse-to-json.py @@ -13,10 +13,10 @@ import pdbparse.undecorate logger = logging.getLogger(__name__) logger.setLevel(1) -if __name__ == '__main__': +if __name__ == "__main__": console = logging.StreamHandler() console.setLevel(1) - formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') + formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) logger.addHandler(console) @@ -25,12 +25,12 @@ class PDBRetreiver: def retreive_pdb(self, guid: str, file_name: str) -> Optional[str]: logger.info("Download PDB file...") - file_name = ".".join(file_name.split(".")[:-1] + ['pdb']) - for sym_url in ['http://msdl.microsoft.com/download/symbols']: + file_name = ".".join(file_name.split(".")[:-1] + ["pdb"]) + for sym_url in ["http://msdl.microsoft.com/download/symbols"]: url = sym_url + f"/{file_name}/{guid}/" result = None - for suffix in [file_name[:-1] + '_', file_name]: + for suffix in [file_name[:-1] + "_", file_name]: try: logger.debug("Attempting to retrieve %s", url + suffix) result, _ = request.urlretrieve(url + suffix) @@ -69,7 +69,7 @@ class PDBConvertor: "float": "float", "double": "float", "long double": "float", - "void": "void" + "void": "void", } base_type_size = { @@ -122,13 +122,18 @@ class PDBConvertor: self._seen_ctypes.add(ctype) return self.ctype[ctype] - def lookup_ctype_pointers(self, ctype_pointer: str) -> Dict[str, Union[str, Dict[str, str]]]: - base_type = ctype_pointer.replace('32P', '').replace('64P', '') + def lookup_ctype_pointers( + self, ctype_pointer: str + ) -> Dict[str, Union[str, Dict[str, str]]]: + base_type = ctype_pointer.replace("32P", "").replace("64P", "") if base_type == ctype_pointer: # We raise a KeyError, because we've been asked about a type that isn't a pointer raise KeyError self._seen_ctypes.add(base_type) - return {"kind": "pointer", "subtype": {"kind": "base", "name": self.ctype[base_type]}} + return { + "kind": "pointer", + "subtype": {"kind": "base", "name": self.ctype[base_type]}, + } def read_pdb(self) -> Dict: """Reads in the PDB file and forms essentially a python dictionary of necessary data""" @@ -137,31 +142,31 @@ class PDBConvertor: "enums": self.read_enums(), "metadata": self.generate_metadata(), "symbols": self.read_symbols(), - "base_types": self.read_basetypes() + "base_types": self.read_basetypes(), } return output def generate_metadata(self) -> Dict[str, Any]: """Generates the metadata necessary for this object""" dbg = self._pdb.STREAM_DBI - last_bytes = str(binascii.hexlify(self._pdb.STREAM_PDB.GUID.Data4), 'ascii')[-16:] - guidstr = f'{self._pdb.STREAM_PDB.GUID.Data1:08x}{self._pdb.STREAM_PDB.GUID.Data2:04x}{self._pdb.STREAM_PDB.GUID.Data3:04x}{last_bytes}' + last_bytes = str(binascii.hexlify(self._pdb.STREAM_PDB.GUID.Data4), "ascii")[ + -16: + ] + guidstr = f"{self._pdb.STREAM_PDB.GUID.Data1:08x}{self._pdb.STREAM_PDB.GUID.Data2:04x}{self._pdb.STREAM_PDB.GUID.Data3:04x}{last_bytes}" pdb_data = { "GUID": guidstr.upper(), "age": self._pdb.STREAM_PDB.Age, "database": "ntkrnlmp.pdb", - "machine_type": int(dbg.machine) + "machine_type": int(dbg.machine), } result = { "format": "6.0.0", "producer": { "datetime": datetime.datetime.now().isoformat(), "name": "pdbconv", - "version": "0.1.0" + "version": "0.1.0", }, - "windows": { - "pdb": pdb_data - } + "windows": {"pdb": pdb_data}, } return result @@ -172,16 +177,21 @@ class PDBConvertor: stream = self._pdb.STREAM_TPI for type_index in stream.types: user_type = stream.types[type_index] - if (user_type.leaf_type == "LF_ENUM" and not user_type.prop.fwdref): + if user_type.leaf_type == "LF_ENUM" and not user_type.prop.fwdref: output.update(self._format_enum(user_type)) return output def _format_enum(self, user_enum): output = { user_enum.name: { - 'base': self.lookup_ctype(user_enum.utype), - 'size': self._determine_size(user_enum.utype), - 'constants': dict([(enum.name, enum.enum_value) for enum in user_enum.fieldlist.substructs]) + "base": self.lookup_ctype(user_enum.utype), + "size": self._determine_size(user_enum.utype), + "constants": dict( + [ + (enum.name, enum.enum_value) + for enum in user_enum.fieldlist.substructs + ] + ), } } return output @@ -201,7 +211,7 @@ class PDBConvertor: omap = None for sym in self._pdb.STREAM_GSYM.globals: - if not hasattr(sym, 'offset'): + if not hasattr(sym, "offset"): continue try: virt_base = sects[sym.segment - 1].VirtualAddress @@ -222,9 +232,9 @@ class PDBConvertor: stream = self._pdb.STREAM_TPI for type_index in stream.types: user_type = stream.types[type_index] - if (user_type.leaf_type == "LF_STRUCTURE" and not user_type.prop.fwdref): + if user_type.leaf_type == "LF_STRUCTURE" and not user_type.prop.fwdref: output.update(self._format_usertype(user_type, "struct")) - elif (user_type.leaf_type == "LF_UNION" and not user_type.prop.fwdref): + elif user_type.leaf_type == "LF_UNION" and not user_type.prop.fwdref: output.update(self._format_usertype(user_type, "union")) return output @@ -232,16 +242,22 @@ class PDBConvertor: """Produces a single usertype""" fields: Dict[str, Dict[str, Any]] = {} [fields.update(self._format_field(s)) for s in usertype.fieldlist.substructs] - return {usertype.name: {'fields': fields, 'kind': kind, 'size': usertype.size}} + return {usertype.name: {"fields": fields, "kind": kind, "size": usertype.size}} def _format_field(self, field) -> Dict[str, Dict[str, Any]]: - return {field.name: {"offset": field.offset, "type": self._format_kind(field.index)}} + return { + field.name: {"offset": field.offset, "type": self._format_kind(field.index)} + } def _determine_size(self, field): output = None if isinstance(field, str): output = self.base_type_size[field] - elif (field.leaf_type == "LF_STRUCTURE" or field.leaf_type == "LF_ARRAY" or field.leaf_type == "LF_UNION"): + elif ( + field.leaf_type == "LF_STRUCTURE" + or field.leaf_type == "LF_ARRAY" + or field.leaf_type == "LF_UNION" + ): output = field.size elif field.leaf_type == "LF_POINTER": output = self.base_type_size[field.ptr_attr.type] @@ -255,6 +271,7 @@ class PDBConvertor: output = self._determine_size(field.index) if output is None: import pdb + pdb.set_trace() raise ValueError(f"Unknown size for field: {field.name}") return output @@ -266,36 +283,37 @@ class PDBConvertor: output = self.lookup_ctype_pointers(kind) except KeyError: try: - output = {'kind': 'base', 'name': self.lookup_ctype(kind)} + output = {"kind": "base", "name": self.lookup_ctype(kind)} except KeyError: - output = {'kind': 'base', 'name': kind} - elif kind.leaf_type == 'LF_MODIFIER': + output = {"kind": "base", "name": kind} + elif kind.leaf_type == "LF_MODIFIER": output = self._format_kind(kind.modified_type) - elif kind.leaf_type == 'LF_STRUCTURE': - output = {'kind': 'struct', 'name': kind.name} - elif kind.leaf_type == 'LF_UNION': - output = {'kind': 'union', 'name': kind.name} - elif kind.leaf_type == 'LF_BITFIELD': + elif kind.leaf_type == "LF_STRUCTURE": + output = {"kind": "struct", "name": kind.name} + elif kind.leaf_type == "LF_UNION": + output = {"kind": "union", "name": kind.name} + elif kind.leaf_type == "LF_BITFIELD": output = { - 'kind': 'bitfield', - 'type': self._format_kind(kind.base_type), - 'bit_length': kind.length, - 'bit_position': kind.position + "kind": "bitfield", + "type": self._format_kind(kind.base_type), + "bit_length": kind.length, + "bit_position": kind.position, } - elif kind.leaf_type == 'LF_POINTER': - output = {'kind': 'pointer', 'subtype': self._format_kind(kind.utype)} - elif kind.leaf_type == 'LF_ARRAY': + elif kind.leaf_type == "LF_POINTER": + output = {"kind": "pointer", "subtype": self._format_kind(kind.utype)} + elif kind.leaf_type == "LF_ARRAY": output = { - 'kind': 'array', - 'count': kind.size // self._determine_size(kind.element_type), - 'subtype': self._format_kind(kind.element_type) + "kind": "array", + "count": kind.size // self._determine_size(kind.element_type), + "subtype": self._format_kind(kind.element_type), } - elif kind.leaf_type == 'LF_ENUM': - output = {'kind': 'enum', 'name': kind.name} - elif kind.leaf_type == 'LF_PROCEDURE': - output = {'kind': "function"} + elif kind.leaf_type == "LF_ENUM": + output = {"kind": "enum", "name": kind.name} + elif kind.leaf_type == "LF_PROCEDURE": + output = {"kind": "function"} else: import pdb + pdb.set_trace() return output @@ -305,40 +323,70 @@ class PDBConvertor: if "64" in self._pdb.STREAM_DBI.machine: ptr_size = 8 - output = {"pointer": {"endian": "little", "kind": "int", "signed": False, "size": ptr_size}} + output = { + "pointer": { + "endian": "little", + "kind": "int", + "signed": False, + "size": ptr_size, + } + } for index in self._seen_ctypes: output[self.ctype[index]] = { "endian": "little", "kind": self.ctype_python_types.get(self.ctype[index], "int"), "signed": False if "_U" in index else True, - "size": self.base_type_size[index] + "size": self.base_type_size[index], } return output -if __name__ == '__main__': - parser = argparse.ArgumentParser(description = "Convertor for PDB files to Volatility 3 Intermediate Symbol Format") - parser.add_argument("-o", "--output", metavar = "OUTPUT", help = "Filename for data output", required = True) - file_group = parser.add_argument_group("file", description = "File-based conversion of PDB to ISF") - file_group.add_argument("-f", "--file", metavar = "FILE", help = "PDB file to translate to ISF") - data_group = parser.add_argument_group("data", description = "Convert based on a GUID and filename pattern") - data_group.add_argument("-p", "--pattern", metavar = "PATTERN", help = "Filename pattern to recover PDB file") - data_group.add_argument("-g", - "--guid", - metavar = "GUID", - help = "GUID + Age string for the required PDB file", - default = None) - data_group.add_argument("-k", - "--keep", - action = "store_true", - default = False, - help = "Keep the downloaded PDB file") +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Convertor for PDB files to Volatility 3 Intermediate Symbol Format" + ) + parser.add_argument( + "-o", + "--output", + metavar="OUTPUT", + help="Filename for data output", + required=True, + ) + file_group = parser.add_argument_group( + "file", description="File-based conversion of PDB to ISF" + ) + file_group.add_argument( + "-f", "--file", metavar="FILE", help="PDB file to translate to ISF" + ) + data_group = parser.add_argument_group( + "data", description="Convert based on a GUID and filename pattern" + ) + data_group.add_argument( + "-p", + "--pattern", + metavar="PATTERN", + help="Filename pattern to recover PDB file", + ) + data_group.add_argument( + "-g", + "--guid", + metavar="GUID", + help="GUID + Age string for the required PDB file", + default=None, + ) + data_group.add_argument( + "-k", + "--keep", + action="store_true", + default=False, + help="Keep the downloaded PDB file", + ) args = parser.parse_args() delfile = False filename = None if args.guid is not None and args.pattern is not None: - filename = PDBRetreiver().retreive_pdb(guid = args.guid, file_name = args.pattern) + filename = PDBRetreiver().retreive_pdb(guid=args.guid, file_name=args.pattern) delfile = True elif args.file: filename = args.file @@ -351,7 +399,7 @@ if __name__ == '__main__': convertor = PDBConvertor(filename) with open(args.output, "w") as f: - json.dump(convertor.read_pdb(), f, indent = 2, sort_keys = True) + json.dump(convertor.read_pdb(), f, indent=2, sort_keys=True) if args.keep: print(f"Temporary PDB file: {filename}") diff --git a/development/schema_validate.py b/development/schema_validate.py index 0ea82d537..cf9565d68 100644 --- a/development/schema_validate.py +++ b/development/schema_validate.py @@ -9,7 +9,7 @@ sys.path += ".." console = logging.StreamHandler() console.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') +formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) logger = logging.getLogger("") @@ -18,10 +18,10 @@ logger.setLevel(logging.DEBUG) from volatility3 import schemas # noqa: E402 -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser("Validates ") - parser.add_argument("-s", "--schema", dest = "schema", default = None) - parser.add_argument("filenames", metavar = "FILE", nargs = '+') + parser.add_argument("-s", "--schema", dest="schema", default=None) + parser.add_argument("filenames", metavar="FILE", nargs="+") args = parser.parse_args() diff --git a/development/stock-linux-json.py b/development/stock-linux-json.py index 877f78e1c..967cc7e18 100644 --- a/development/stock-linux-json.py +++ b/development/stock-linux-json.py @@ -9,7 +9,7 @@ import requests import rpmfile from debian import debfile -DWARF2JSON = './dwarf2json' +DWARF2JSON = "./dwarf2json" class Downloader: @@ -17,7 +17,7 @@ class Downloader: def __init__(self, url_lists: List[List[str]]) -> None: self.url_lists = url_lists - def download_lists(self, keep = False): + def download_lists(self, keep=False): for url_list in self.url_lists: print("Downloading files...") files_for_processing = self.download_list(url_list) @@ -35,43 +35,45 @@ class Downloader: with tempfile.NamedTemporaryFile() as archivedata: archivedata.write(data.content) archivedata.seek(0) - if url.endswith('.rpm'): + if url.endswith(".rpm"): processed_files[url] = self.process_rpm(archivedata) - elif url.endswith('.deb'): + elif url.endswith(".deb"): processed_files[url] = self.process_deb(archivedata) return processed_files def process_rpm(self, archivedata) -> Optional[str]: - rpm = rpmfile.RPMFile(fileobj = archivedata) + rpm = rpmfile.RPMFile(fileobj=archivedata) member = None extracted = None for member in rpm.getmembers(): - if 'vmlinux' in member.name or 'System.map' in member.name: + if "vmlinux" in member.name or "System.map" in member.name: print(f" - Extracting {member.name}") extracted = rpm.extractfile(member) break if not member or not extracted: return None - with tempfile.NamedTemporaryFile(delete = False, - prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: + with tempfile.NamedTemporaryFile( + delete=False, prefix="vmlinux" if "vmlinux" in member.name else "System.map" + ) as output: print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name def process_deb(self, archivedata) -> Optional[str]: - deb = debfile.DebFile(fileobj = archivedata) + deb = debfile.DebFile(fileobj=archivedata) member = None extracted = None for member in deb.data.tgz().getmembers(): - if member.name.endswith('vmlinux') or 'System.map' in member.name: + if member.name.endswith("vmlinux") or "System.map" in member.name: print(f" - Extracting {member.name}") extracted = deb.data.get_file(member.name) break if not member or not extracted: return None - with tempfile.NamedTemporaryFile(delete = False, - prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: + with tempfile.NamedTemporaryFile( + delete=False, prefix="vmlinux" if "vmlinux" in member.name else "System.map" + ) as output: print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name @@ -83,43 +85,55 @@ class Downloader: if named_files[i] is None: print(f"FAILURE: None encountered for {i}") return - args = [DWARF2JSON, 'linux'] - output_filename = 'unknown-kernel.json' + args = [DWARF2JSON, "linux"] + output_filename = "unknown-kernel.json" for named_file in named_files: - prefix = '--system-map' - if 'System' not in named_files[named_file]: - prefix = '--elf' - output_filename = './' + '-'.join((named_file.split('/')[-1]).split('-')[2:])[:-4] + '.json.xz' + prefix = "--system-map" + if "System" not in named_files[named_file]: + prefix = "--elf" + output_filename = ( + "./" + + "-".join((named_file.split("/")[-1]).split("-")[2:])[:-4] + + ".json.xz" + ) args += [prefix, named_files[named_file]] print(f" - Running {args}") - proc = subprocess.run(args, capture_output = True) + proc = subprocess.run(args, capture_output=True) print(f" - Writing to {output_filename}") - with lzma.open(output_filename, 'w') as f: + with lzma.open(output_filename, "w") as f: f.write(proc.stdout) -if __name__ == '__main__': - parser = argparse.ArgumentParser(description = "Takes a list of URLs for Centos and downloads them") - parser.add_argument("-f", - "--file", - dest = 'filename', - metavar = "FILENAME", - help = "Filename to be read", - required = True) - parser.add_argument("-d", - "--dwarf2json", - dest = 'dwarfpath', - metavar = "PATH", - default = DWARF2JSON, - help = "Path to the dwarf2json binary", - required = True) - parser.add_argument("-k", - "--keep", - dest = 'keep', - action = 'store_true', - help = 'Keep extracted temporary files after completion', - default = False) +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Takes a list of URLs for Centos and downloads them" + ) + parser.add_argument( + "-f", + "--file", + dest="filename", + metavar="FILENAME", + help="Filename to be read", + required=True, + ) + parser.add_argument( + "-d", + "--dwarf2json", + dest="dwarfpath", + metavar="PATH", + default=DWARF2JSON, + help="Path to the dwarf2json binary", + required=True, + ) + parser.add_argument( + "-k", + "--keep", + dest="keep", + action="store_true", + help="Keep extracted temporary files after completion", + default=False, + ) args = parser.parse_args() DWARF2JSON = args.dwarfpath @@ -132,4 +146,4 @@ if __name__ == '__main__': urls += [[lines[2 * i].strip(), lines[(2 * i) + 1].strip()]] d = Downloader(urls) - d.download_lists(keep = args.keep) + d.download_lists(keep=args.keep) diff --git a/test/plugins/windows/test_scheduled_tasks.py b/test/plugins/windows/test_scheduled_tasks.py index fdb19fbae..15d7f79a6 100644 --- a/test/plugins/windows/test_scheduled_tasks.py +++ b/test/plugins/windows/test_scheduled_tasks.py @@ -2,9 +2,11 @@ import sys import struct import traceback import unittest + sys.path.insert(0, "../../volatility3") from volatility3.plugins.windows import scheduled_tasks + class TestActionsDecoding(unittest.TestCase): def test_decode_exe_action(self): # fmt: off diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 00215f624..236d256f1 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -29,7 +29,7 @@ except ImportError: try: # Import so that the handler is found by the framework.class_subclasses callc - from smb import SMBHandler as SMBHandler # lgtm [py/unused-import] + from smb import SMBHandler as SMBHandler # lgtm [py/unused-import] except ImportError: # If we fail to import this, it means that SMB handling won't be available pass From 503999ddcafc6eee23d53a0e08c3ebad3139892e Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Sun, 24 Nov 2024 13:04:58 +0100 Subject: [PATCH 705/979] update black workflow (formatter not a linter) --- .github/workflows/black.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index e29ab6f29..d755d9402 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,4 +1,4 @@ -name: Black python linter +name: Black python formatter on: [push, pull_request] From ab19b52d5b85a6fc7275bf8108fa74e966c72694 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Wed, 27 Nov 2024 11:58:14 +0100 Subject: [PATCH 706/979] run `ruff check --fix` --- volatility3/framework/automagic/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index c044fdd93..f22cae012 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -3,7 +3,7 @@ # import logging -from typing import Optional, Tuple, Type +from typing import Optional, Tuple from volatility3.framework import constants, interfaces from volatility3.framework.automagic import symbol_cache, symbol_finder From d9c370c6ae94ba7ff94075279f65608034b91830 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Mon, 9 Dec 2024 15:01:32 +0100 Subject: [PATCH 707/979] format with black --- volatility3/framework/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index b60ae5576..d5e2c50b4 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -143,9 +143,7 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]: def _filter_files(filename: str): """Ensures that a filename traversed is an importable python file""" - return (filename.endswith((".py", ".pyc"))) and not filename.startswith( - "__" - ) + return (filename.endswith((".py", ".pyc"))) and not filename.startswith("__") def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str]: From c6c3b35f52a33d8a4c7ddb57245d73c6270e08f3 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Thu, 12 Dec 2024 09:21:36 +0100 Subject: [PATCH 708/979] run `ruff check . --unsafe-fixes --fix` --- volatility3/framework/plugins/windows/mftscan.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index feea78ece..c4d05e634 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -275,19 +275,17 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): display_data = True if display_data: - for record in cls.parse_data_record( + yield from cls.parse_data_record( mft_record, attr, record_map, return_first_record - ): - yield record + ) def _generator(self): - for record in self.enumerate_mft_records( + yield from self.enumerate_mft_records( self.context, self.config_path, self.config["primary"], self.parse_mft_records, - ): - yield record + ) def generate_timeline(self): for row in self._generator(): From 44e8ac645fe08de903df830b15eca32d006d8f50 Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Tue, 17 Dec 2024 22:59:04 +0100 Subject: [PATCH 709/979] chore: make ruff happy the import is not needed, was previously used for type annotations --- .../framework/plugins/windows/indirect_system_calls.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py index f09851b30..dac0f9c4a 100644 --- a/volatility3/framework/plugins/windows/indirect_system_calls.py +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -13,12 +13,6 @@ from volatility3.plugins.windows import pslist, direct_system_calls vollog = logging.getLogger(__name__) -try: - import capstone -except ImportError: - # The generator of DirectSystemCalls will bail with a warning if capstone is not installed - pass - class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): _required_framework_version = (2, 4, 0) From 0d7c2906eecadcc083e4919176350f113e6d8931 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Tue, 17 Dec 2024 16:01:51 -0600 Subject: [PATCH 710/979] #1324 - update pedump error messages --- volatility3/framework/plugins/windows/pedump.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 5b4bb07d7..d9ab39b4a 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -224,11 +224,11 @@ class PEDump(interfaces.plugins.PluginInterface): ) if self.config["kernel_module"] and self.config["pid"]: - vollog.error("Only --kernel_module or --pid should be set. Not both") + vollog.error("Only 'kernel-module' or 'pid' should be set, not both") return if not self.config["kernel_module"] and not self.config["pid"]: - vollog.error("--kernel_module or --pid must be set") + vollog.error("Either 'kernel-module' or 'pid' argument must be set") return if self.config["kernel_module"]: From e035faa32392bc27d20f4b7b0c01feb0e9da2210 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 09:55:13 +1100 Subject: [PATCH 711/979] bump framework minor version --- volatility3/framework/constants/_version.py | 2 +- volatility3/framework/plugins/linux/capabilities.py | 2 +- volatility3/framework/plugins/linux/envars.py | 2 +- volatility3/framework/plugins/linux/psaux.py | 2 +- volatility3/framework/plugins/linux/pslist.py | 2 +- volatility3/framework/plugins/linux/psscan.py | 2 +- volatility3/framework/plugins/linux/pstree.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 55ef19e4b..93e9c8432 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 12 # Number of changes that only add to the interface +VERSION_MINOR = 13 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index a06ee4c1b..afd91c48e 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -49,7 +49,7 @@ class CapabilitiesData: class Capabilities(plugins.PluginInterface): """Lists process capabilities""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (1, 1, 0) @classmethod diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index a3eb21cf5..aec3eeb14 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -16,7 +16,7 @@ vollog = logging.getLogger(__name__) class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (1, 1, 0) @classmethod diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index 5a4d75c70..60424a990 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -14,7 +14,7 @@ from volatility3.plugins.linux import pslist class PsAux(plugins.PluginInterface): """Lists processes with their command line arguments""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (1, 1, 0) @classmethod diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index edfc0688c..a6d2e6538 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -17,7 +17,7 @@ from volatility3.plugins.linux import elfs class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (3, 1, 0) @classmethod diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 55e3778ab..2b20ce583 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -27,7 +27,7 @@ class DescExitStateEnum(Enum): class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (1, 1, 0) @classmethod diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index 7ea9df3d6..c80cfbec7 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -12,7 +12,7 @@ class PsTree(interfaces.plugins.PluginInterface): """Plugin for listing processes in a tree based on their parent process ID.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) _version = (1, 1, 0) @classmethod From 9c587b037885489442c55525add1ab6bfb69b852 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 10:12:32 +1100 Subject: [PATCH 712/979] linux: testcases: improve existent testcases code and checks --- test/test_volatility.py | 89 ++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 49 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index b5910e1c8..300ab572c 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -334,84 +334,84 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): def test_linux_pslist(image, volatility, python): rc, out, _err = runvol_plugin("linux.pslist.PsList", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) assert out.find(b"watchdog") != -1 assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_check_idt(image, volatility, python): rc, out, _err = runvol_plugin( "linux.check_idt.Check_idt", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.count(b"__kernel__") >= 10 assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_check_syscall(image, volatility, python): rc, out, _err = runvol_plugin( "linux.check_syscall.Check_syscall", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.find(b"sys_close") != -1 assert out.find(b"sys_open") != -1 assert out.count(b"\n") > 100 - assert rc == 0 def test_linux_lsmod(image, volatility, python): rc, out, _err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + out = out.lower() + assert out.count(b"\n") > 10 def test_linux_lsof(image, volatility, python): rc, out, _err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.count(b"socket:") >= 10 assert out.count(b"\n") > 35 - assert rc == 0 def test_linux_proc_maps(image, volatility, python): rc, out, _err = runvol_plugin("linux.proc.Maps", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.count(b"anonymous mapping") >= 10 assert out.count(b"\n") > 100 - assert rc == 0 def test_linux_tty_check(image, volatility, python): rc, out, _err = runvol_plugin( "linux.tty_check.tty_check", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.find(b"__kernel__") != -1 assert out.count(b"\n") >= 5 - assert rc == 0 def test_linux_sockstat(image, volatility, python): rc, out, _err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) + assert rc == 0 assert out.count(b"AF_UNIX") >= 354 assert out.count(b"AF_BLUETOOTH") >= 5 assert out.count(b"AF_INET") >= 32 assert out.count(b"AF_INET6") >= 20 assert out.count(b"AF_PACKET") >= 1 assert out.count(b"AF_NETLINK") >= 43 - assert rc == 0 def test_linux_library_list(image, volatility, python): @@ -423,49 +423,48 @@ def test_linux_library_list(image, volatility, python): pluginargs=["--pids", "2363"], ) + assert rc == 0 assert re.search( rb"NetworkManager\s2363\s0x7f52cdda0000\s/lib/x86_64-linux-gnu/libnss_files.so.2", out, ) assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_pstree(image, volatility, python): rc, out, _err = runvol_plugin("linux.pstree.PsTree", image, volatility, python) - out = out.lower() + assert rc == 0 + out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_pidhashtable(image, volatility, python): rc, out, _err = runvol_plugin( "linux.pidhashtable.PIDHashTable", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_bash(image, volatility, python): rc, out, _err = runvol_plugin("linux.bash.Bash", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_boottime(image, volatility, python): rc, out, _err = runvol_plugin("linux.boottime.Boottime", image, volatility, python) - out = out.lower() - assert out.count(b"utc") >= 1 assert rc == 0 + out = out.lower() + assert out.count(b"utc") >= 1 def test_linux_capabilities(image, volatility, python): @@ -482,36 +481,33 @@ def test_linux_capabilities(image, volatility, python): # However, we can still check that the plugin requirements are met. return None - out = out.lower() - - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_check_creds(image, volatility, python): - rc, _out, _err = runvol_plugin( + rc, out, _err = runvol_plugin( "linux.check_creds.Check_creds", image, volatility, python ) # linux-sample-1.bin has no processes sharing credentials. # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_elfs(image, volatility, python): rc, out, _err = runvol_plugin("linux.elfs.Elfs", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_envars(image, volatility, python): rc, out, _err = runvol_plugin("linux.envars.Envars", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_kthreads(image, volatility, python): @@ -528,44 +524,42 @@ def test_linux_kthreads(image, volatility, python): # However, we can still check that the plugin requirements are met. return None - out = out.lower() - - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_malfind(image, volatility, python): - rc, _out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) + rc, out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) # linux-sample-1.bin has no process memory ranges with potential injected code. # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_mountinfo(image, volatility, python): rc, out, _err = runvol_plugin( "linux.mountinfo.MountInfo", image, volatility, python ) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_psaux(image, volatility, python): rc, out, _err = runvol_plugin("linux.psaux.PsAux", image, volatility, python) - out = out.lower() - assert out.count(b"\n") > 50 assert rc == 0 + assert out.count(b"\n") > 50 def test_linux_ptrace(image, volatility, python): - rc, _out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) + rc, out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) - # linux-sample-1.bin has no processes being ptreaced. + # linux-sample-1.bin has no processes being ptraced. # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 def test_linux_vmaregexscan(image, volatility, python): @@ -576,10 +570,9 @@ def test_linux_vmaregexscan(image, volatility, python): python, pluginargs=["--pid", "1", "--pattern", "\\x7fELF"], ) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_vmayarascan_yara_rule(image, volatility, python): @@ -613,9 +606,8 @@ def test_linux_vmayarascan_yara_rule(image, volatility, python): with contextlib.suppress(FileNotFoundError): os.remove(filename) - out = out.lower() - assert out.count(b"\n") > 4 assert rc == 0 + assert out.count(b"\n") > 4 def test_linux_vmayarascan_yara_string(image, volatility, python): @@ -626,10 +618,9 @@ def test_linux_vmayarascan_yara_string(image, volatility, python): python, pluginargs=["--pid", "1", "--yara-string", "ELF"], ) - out = out.lower() - assert out.count(b"\n") > 10 assert rc == 0 + assert out.count(b"\n") > 10 def test_linux_page_cache_files(image, volatility, python): @@ -640,8 +631,8 @@ def test_linux_page_cache_files(image, volatility, python): python, pluginargs=["--find", "/etc/passwd"], ) - out = out.lower() + assert rc == 0 assert out.count(b"\n") > 4 # inode_num inode_addr ... file_path From 69d291c644910b14c9ff07df83b24f23601f945d Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 10:16:59 +1100 Subject: [PATCH 713/979] linux: testcases: add 10 final test cases to achieve full plugin coverage --- test/test_volatility.py | 123 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index 300ab572c..4b0455596 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -640,7 +640,130 @@ def test_linux_page_cache_files(image, volatility, python): rb"146829\s0x88001ab5c270.*?/etc/passwd", out, ) + + +def test_linux_page_cache_inodepages(image, volatility, python): + + inode_address = hex(0x88001AB5C270) + inode_dump_filename = f"inode_{inode_address}.dmp" + try: + rc, out, _err = runvol_plugin( + "linux.pagecache.InodePages", + image, + volatility, + python, + pluginargs=["--inode", inode_address, "--dump"], + ) + + assert rc == 0 + assert out.count(b"\n") > 4 + + # PageVAddr PagePAddr MappingAddr .. DumpSafe + assert re.search( + rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", + out, + ) + assert os.path.exists(inode_dump_filename) + inode_contents = open(inode_dump_filename, "rb").read() + assert inode_contents.count(b"\n") > 30 + assert inode_contents.count(b"root:x:0:0:root:/root:/bin/bash") > 0 + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(inode_dump_filename) + + +def test_linux_check_afinfo(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.check_afinfo.Check_afinfo", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results. + # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_check_modules(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.check_modules.Check_modules", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_ebpf_progs(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.ebpf.EBPF", + image, + volatility, + python, + globalargs=["-vvv"], + ) + + if rc != 0 and err.count(b"Unsupported kernel") > 0: + # The linux-sample-1.bin kernel implementation isn't supported. + # However, we can still check that the plugin requirements are met. + return None + + assert rc == 0 + assert out.count(b"\n") > 4 + + +def test_linux_iomem(image, volatility, python): + rc, out, _err = runvol_plugin("linux.iomem.IOMem", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_keyboard_notifiers(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.keyboard_notifiers.Keyboard_notifiers", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results for this plugin. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_kmesg(image, volatility, python): + rc, out, _err = runvol_plugin("linux.kmsg.Kmsg", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_netfilter(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.netfilter.Netfilter", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results for this plugin. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_psscan(image, volatility, python): + rc, out, _err = runvol_plugin("linux.psscan.PsScan", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_hidden_modules(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.hidden_modules.Hidden_modules", image, volatility, python + ) + + # linux-sample-1.bin has no hidden modules. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 # MAC From 9c0dcad7b13cca5b3d75b1977ad4ad4a963bb0f9 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 10:42:54 +1100 Subject: [PATCH 714/979] linux: page cache inodepages testcase: explicitly close the file to improve clarity for AI processing --- test/test_volatility.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 4b0455596..5bce07481 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -664,7 +664,8 @@ def test_linux_page_cache_inodepages(image, volatility, python): out, ) assert os.path.exists(inode_dump_filename) - inode_contents = open(inode_dump_filename, "rb").read() + with open(inode_dump_filename, "rb") as fp: + inode_contents = fp.read() assert inode_contents.count(b"\n") > 30 assert inode_contents.count(b"root:x:0:0:root:/root:/bin/bash") > 0 finally: From e8a73dfb2d5ae368c4af758739b5593643b6bd41 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 11:29:28 +1100 Subject: [PATCH 715/979] linux: envars plugin: Add function to retrieve environment variables for a specific task. Code improvements. --- volatility3/framework/plugins/linux/envars.py | 147 ++++++++++-------- 1 file changed, 84 insertions(+), 63 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index aec3eeb14..bb6f52a7a 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -3,8 +3,9 @@ # import logging +from typing import Iterable, Tuple -from volatility3.framework import exceptions, renderers +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -17,7 +18,7 @@ class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -39,76 +40,96 @@ class Envars(plugins.PluginInterface): ), ] + @staticmethod + def get_task_env_variables( + context: interfaces.context.ContextInterface, + task: interfaces.objects.ObjectInterface, + env_area_max_size: int = 8192, + ) -> Iterable[Tuple[str, str]]: + """Yields environment variables for a given task. + + Args: + context: The plugin's operational context. + task: The task object from which to extract environment variables. + + Yields: + Tuples of (key, value) representing each environment variable. + """ + + task_name = utility.array_to_string(task.comm) + task_pid = task.pid + env_start = task.mm.env_start + env_end = task.mm.env_end + env_area_size = env_end - env_start + if not (0 < env_area_size <= env_area_max_size): + vollog.debug( + f"Task {task_pid} {task_name} appears to have environment variables of size " + f"{env_area_size} bytes which fails the sanity checking, will not extract " + "any envars." + ) + return None + + # Get process layer to read envars from + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + return None + proc_layer = context.layers[proc_layer_name] + + # Ensure the entire buffer is readable to prevent relying on exception handling + if not proc_layer.is_valid(env_start, env_area_size): + # Not mapped / swapped out + vollog.debug( + f"Unable to read environment variables for {task_pid} {task_name} starting at " + f" virtual address 0x{env_start:x} for {env_area_size} bytes, will not " + "extract any envars." + ) + return None + + # Read the full task environment variable buffer. + envar_data = proc_layer.read(env_start, env_area_size) + + # Parse envar data, envars are null terminated, keys and values are separated by '=' + envar_data = envar_data.rstrip(b"\x00") + for envar_pair in envar_data.split(b"\x00"): + try: + env_key, env_value = envar_pair.decode().split("=", 1) + except ValueError: + # Some legitimate programs, like 'avahi-daemon', avoid reallocating the args + # and instead exploit the fact that the environment variables area is contiguous + # to the args. This allows them to include a longer process name in the listing, + # causing overwrites and incorrect results. In such cases, it's better to abort + # the current task rather than displaying misleading or incorrect output. + break + + yield env_key, env_value + def _generator(self, tasks): """Generates a listing of processes along with environment variables""" # walk the process list and return the envars for task in tasks: - pid = task.pid - - # get process name as string - name = utility.array_to_string(task.comm) - ppid = task.get_parent_pid() - - # kernel threads never have an mm as they do not have userland mappings - try: - mm = task.mm - except exceptions.InvalidAddressException: - # no mm so cannot get envars - vollog.debug( - f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." - ) - mm = None + if task.is_kernel_thread: continue - # if mm exists attempt to get envars - if mm: - # get process layer to read envars from - proc_layer_name = task.add_process_layer() - if proc_layer_name is None: - vollog.debug( - f"Unable to construct process layer for task {pid} {name}, will not extract any envars." - ) - continue - proc_layer = self.context.layers[proc_layer_name] + task_pid = task.pid + task_name = utility.array_to_string(task.comm) + task_ppid = task.get_parent_pid() - # get the size of the envars with sanity checking - envars_size = task.mm.env_end - task.mm.env_start - if not (0 < envars_size <= 8192): - vollog.debug( - f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." - ) - continue - - # attempt to read all envars data - try: - envar_data = proc_layer.read(task.mm.env_start, envars_size) - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." - ) - continue - - # parse envar data, envars are null terminated, keys and values are separated by '=' - envar_data = envar_data.rstrip(b"\x00") - for envar_pair in envar_data.split(b"\x00"): - try: - key, value = envar_pair.decode().split("=", 1) - except ValueError: - vollog.debug( - f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" - ) - continue - yield (0, (pid, ppid, name, key, value)) + for env_key, env_value in self.get_task_env_variables(self.context, task): + yield (0, (task_pid, task_ppid, task_name, env_key, env_value)) def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - - return renderers.TreeGrid( - [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], - self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ), + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func ) + + headers = [ + ("PID", int), + ("PPID", int), + ("COMM", str), + ("KEY", str), + ("VALUE", str), + ] + + return renderers.TreeGrid(headers, self._generator(tasks)) From e231d826a96f416e988b6f35d5a180e2d9342579 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 11:38:51 +1100 Subject: [PATCH 716/979] linux: envars plugin: Complete get_task_env_variables() docstring --- volatility3/framework/plugins/linux/envars.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index bb6f52a7a..05ce17f8a 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -51,6 +51,8 @@ class Envars(plugins.PluginInterface): Args: context: The plugin's operational context. task: The task object from which to extract environment variables. + env_area_max_size: Maximum allowable size for the environment variables area. + Tasks exceeding this size will be skipped. Default is 8192. Yields: Tuples of (key, value) representing each environment variable. From d934d4421b94c3d7285cc1594830bf84c4846373 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 12:02:14 +1100 Subject: [PATCH 717/979] linux: get_parent_pid: Fix parent ID to correctly mimic getppid() syscall behavior by using TGID instead of PID --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7b025450c..3c26805b7 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -641,7 +641,7 @@ class task_struct(generic.GenericIntelProcess): """ if self.real_parent and self.real_parent.is_readable(): - ppid = self.real_parent.pid + ppid = self.real_parent.tgid else: ppid = 0 From 2de553e1c17eab61cc566308cdabe9f9ba60b4ed Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 15:25:07 +1100 Subject: [PATCH 718/979] linux: cred: add user identifiers to the cred object extension --- .../symbols/linux/extensions/__init__.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7b025450c..2c6d9147d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2079,13 +2079,40 @@ class cred(objects.StructType): return int(value) @property - def euid(self): + def uid(self) -> int: + """Returns the real user ID + + Returns: + The real user ID value + """ + return self._get_cred_int_value("uid") + + @property + def gid(self) -> int: + """Returns the real user ID + + Returns: + The real user ID value + """ + return self._get_cred_int_value("gid") + + @property + def euid(self) -> int: """Returns the effective user ID + Returns: + The effective user ID value + """ + return self._get_cred_int_value("euid") + + @property + def egid(self) -> int: + """Returns the effective group ID + Returns: int: the effective user ID value """ - return self._get_cred_int_value("euid") + return self._get_cred_int_value("egid") class kernel_cap_struct(objects.StructType): From 448ba24eac7bb11aba30d9f3e4d0dedcd9246d43 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 15:53:07 +1100 Subject: [PATCH 719/979] linux: pslist: add user/group , real/effective identifiers to the output: uid,gid, euid and egid. We reimplemented get_task_fields() using a dataclass, reducing the size of the function's interface and preventing unbounded growth. This change simplifies future modifications and enhances maintainability. --- volatility3/framework/plugins/linux/pslist.py | 86 +++++++++++++------ 1 file changed, 61 insertions(+), 25 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index a6d2e6538..2244b91ce 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -2,7 +2,9 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import datetime -from typing import Any, Callable, Iterable, List, Tuple +import dataclasses +import contextlib +from typing import Any, Callable, Iterable, List, Optional from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -14,11 +16,25 @@ from volatility3.plugins import timeliner from volatility3.plugins.linux import elfs +@dataclasses.dataclass +class TaskFields: + offset: int + user_pid: int + user_tid: int + user_ppid: int + name: str + uid: Optional[int] + gid: Optional[int] + euid: Optional[int] + egid: Optional[int] + creation_time: Optional[datetime.datetime] + + class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 13, 0) - _version = (3, 1, 0) + _version = (4, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -82,7 +98,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def get_task_fields( cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False - ) -> Tuple[int, int, int, int, str, datetime.datetime]: + ) -> TaskFields: """Extract the fields needed for the final output Args: @@ -91,21 +107,34 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): and of Kernel threads in square brackets. Defaults to False. Returns: - A tuple with the fields to show in the plugin output. + A TaskFields object with the fields to show in the plugin output. """ - pid = task.tgid - tid = task.pid - ppid = task.get_parent_pid() name = utility.array_to_string(task.comm) - start_time = task.get_create_time() if decorate_comm: if task.is_kernel_thread: name = f"[{name}]" elif task.is_user_thread: name = f"{{{name}}}" - task_fields = (task.vol.offset, pid, tid, ppid, name, start_time) - return task_fields + # This function may be called with a partially initialized/uninitialized task. + # Ensure it always returns a valid TaskFields object, ready for use in a plugin. + valid_cred = task.cred and task.cred.is_readable() + creation_time = None + with contextlib.suppress(Exception): + creation_time = task.get_create_time() + + return TaskFields( + offset=task.vol.offset, + user_pid=task.tgid, + user_tid=task.pid, + user_ppid=task.get_parent_pid(), + name=name, + uid=task.cred.uid if valid_cred else None, + gid=task.cred.gid if valid_cred else None, + euid=task.cred.euid if valid_cred else None, + egid=task.cred.egid if valid_cred else None, + creation_time=creation_time, + ) def _get_file_output(self, task: interfaces.objects.ObjectInterface) -> str: """Extract the elf for the process if requested @@ -179,17 +208,19 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): else: file_output = "Disabled" - offset, pid, tid, ppid, name, creation_time = self.get_task_fields( - task, decorate_comm - ) + task_fields = self.get_task_fields(task, decorate_comm) yield 0, ( - format_hints.Hex(offset), - pid, - tid, - ppid, - name, - creation_time or renderers.NotAvailableValue(), + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, + task_fields.uid, + task_fields.gid, + task_fields.euid, + task_fields.egid, + task_fields.creation_time or renderers.NotAvailableValue(), file_output, ) @@ -238,6 +269,10 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ("TID", int), ("PPID", int), ("COMM", str), + ("UID", int), + ("GID", int), + ("EUID", int), + ("EGID", int), ("CREATION TIME", datetime.datetime), ("File output", str), ] @@ -251,10 +286,11 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): for task in self.list_tasks( self.context, self.config["kernel"], filter_func, include_threads=True ): - offset, user_pid, user_tid, _user_ppid, name, creation_time = ( - self.get_task_fields(task) + task_fields = self.get_task_fields(task) + description = f"Process {task_fields.user_pid}/{task_fields.user_tid} {task_fields.name} ({task_fields.offset})" + + yield ( + description, + timeliner.TimeLinerType.CREATED, + task_fields.creation_time, ) - - description = f"Process {user_pid}/{user_tid} {name} ({offset})" - - yield (description, timeliner.TimeLinerType.CREATED, creation_time) From 8e12cf02e3720c3c58df6a3fb66930278d797b38 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 15:56:55 +1100 Subject: [PATCH 720/979] linux: psscan: reimplemented to make use of pslist.get_task_fields() --- volatility3/framework/plugins/linux/psscan.py | 49 +++++++------------ 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index c93f06088..ba68c4856 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -2,15 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import Iterable, List, Tuple +from typing import Iterable, List import struct from enum import Enum from volatility3.framework import renderers, interfaces, symbols, constants, exceptions from volatility3.framework.configuration import requirements -from volatility3.framework.objects import utility from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints +from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) @@ -28,7 +28,7 @@ class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -38,34 +38,11 @@ class PsScan(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) + ), ] - def _get_task_fields( - self, task: interfaces.objects.ObjectInterface - ) -> Tuple[int, int, int, str, str]: - """Extract the fields needed for the final output - - Args: - task: A task object from where to get the fields. - Returns: - A tuple with the fields to show in the plugin output. - """ - pid = task.tgid - tid = task.pid - ppid = task.get_parent_pid() - name = utility.array_to_string(task.comm) - exit_state = DescExitStateEnum(task.exit_state).name - - task_fields = ( - format_hints.Hex(task.vol.offset), - pid, - tid, - ppid, - name, - exit_state, - ) - return task_fields - def _generator(self): """Generates the tasks found from scanning.""" @@ -75,8 +52,18 @@ class PsScan(interfaces.plugins.PluginInterface): for task in self.scan_tasks( self.context, vmlinux_module_name, vmlinux.layer_name ): - row = self._get_task_fields(task) - yield (0, row) + task_fields = pslist.PsList.get_task_fields(task) + exit_state = DescExitStateEnum(task.exit_state).name + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, + exit_state, + ) + + yield (0, fields) @classmethod def scan_tasks( From 40bdbf62daec2047f7c0e1bc06a5feb85b10ca77 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 16:00:14 +1100 Subject: [PATCH 721/979] linux: pidhashtable: Update to use TaskFields from pslist.get_task_fields(). Fix some type annotations --- .../framework/plugins/linux/pidhashtable.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index 2d210c233..060b3928e 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -3,7 +3,7 @@ # import logging -from typing import List +from typing import List, Iterable from volatility3.framework import renderers, interfaces, constants from volatility3.framework.symbols import linux @@ -19,7 +19,7 @@ class PIDHashTable(plugins.PluginInterface): """Enumerates processes through the PID hash table""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -30,7 +30,7 @@ class PIDHashTable(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) @@ -218,7 +218,7 @@ class PIDHashTable(plugins.PluginInterface): return None - def get_tasks(self) -> interfaces.objects.ObjectInterface: + def get_tasks(self) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerates processes through the PID hash table Yields: @@ -231,14 +231,16 @@ class PIDHashTable(plugins.PluginInterface): yield from sorted(pid_func(), key=lambda t: (t.tgid, t.pid)) - def _generator( - self, decorate_comm: bool = False - ) -> interfaces.objects.ObjectInterface: + def _generator(self, decorate_comm: bool = False): for task in self.get_tasks(): - offset, pid, tid, ppid, name, _creation_time = ( - pslist.PsList.get_task_fields(task, decorate_comm) + task_fields = pslist.PsList.get_task_fields(task, decorate_comm) + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, ) - fields = format_hints.Hex(offset), pid, tid, ppid, name yield 0, fields def run(self): From f3cf182206cd00fd70742c304ce48dbfe80ba246 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 16:00:50 +1100 Subject: [PATCH 722/979] linux: pstree: Update to use TaskFields from pslist.get_task_fields() --- volatility3/framework/plugins/linux/pstree.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index c80cfbec7..74e172139 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -13,7 +13,7 @@ class PsTree(interfaces.plugins.PluginInterface): ID.""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls): @@ -25,7 +25,7 @@ class PsTree(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", @@ -101,13 +101,17 @@ class PsTree(interfaces.plugins.PluginInterface): def yield_processes(pid): task = self._tasks[pid] - offset, pid, tid, ppid, name, _creation_time = ( - pslist.PsList.get_task_fields(task, decorate_comm) + task_fields = pslist.PsList.get_task_fields(task, decorate_comm) + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, ) - fields = format_hints.Hex(offset), pid, tid, ppid, name - yield (self._levels[tid] - 1, fields) + yield (self._levels[task_fields.user_tid] - 1, fields) - for child_pid in sorted(self._children.get(tid, [])): + for child_pid in sorted(self._children.get(task_fields.user_tid, [])): yield from yield_processes(child_pid) for pid, level in self._levels.items(): From e92b55ac862ad1f8df2a7a10bbf2b542eb26b6cb Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 16:03:46 +1100 Subject: [PATCH 723/979] linux: Update version requirements in all plugins dependent on pslist --- volatility3/framework/plugins/linux/bash.py | 4 ++-- volatility3/framework/plugins/linux/boottime.py | 4 ++-- volatility3/framework/plugins/linux/capabilities.py | 4 ++-- volatility3/framework/plugins/linux/check_creds.py | 4 ++-- volatility3/framework/plugins/linux/elfs.py | 4 ++-- volatility3/framework/plugins/linux/envars.py | 4 ++-- volatility3/framework/plugins/linux/kthreads.py | 4 ++-- volatility3/framework/plugins/linux/library_list.py | 4 ++-- volatility3/framework/plugins/linux/lsof.py | 4 ++-- volatility3/framework/plugins/linux/malfind.py | 4 ++-- volatility3/framework/plugins/linux/mountinfo.py | 4 ++-- volatility3/framework/plugins/linux/proc.py | 4 ++-- volatility3/framework/plugins/linux/psaux.py | 4 ++-- volatility3/framework/plugins/linux/ptrace.py | 4 ++-- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- volatility3/framework/plugins/linux/vmaregexscan.py | 4 ++-- volatility3/framework/plugins/linux/vmayarascan.py | 4 ++-- 17 files changed, 34 insertions(+), 34 deletions(-) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index 77a433a3b..056e3cd51 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -22,7 +22,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): """Recovers bash command history from memory.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -33,7 +33,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py index 56de52883..c57bdd65a 100644 --- a/volatility3/framework/plugins/linux/boottime.py +++ b/volatility3/framework/plugins/linux/boottime.py @@ -15,7 +15,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """Shows the time the system was started""" _required_framework_version = (2, 11, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -26,7 +26,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index afd91c48e..1d0c60c11 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -50,7 +50,7 @@ class Capabilities(plugins.PluginInterface): """Lists process capabilities""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -61,7 +61,7 @@ class Capabilities(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pids", diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 4916b67d2..96f77ce4d 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -12,7 +12,7 @@ class Check_creds(interfaces.plugins.PluginInterface): """Checks if any processes are sharing credential structures""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (2, 0, 2) @classmethod def get_requirements(cls): @@ -23,7 +23,7 @@ class Check_creds(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 9f3bd274b..2fd740941 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -25,7 +25,7 @@ class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 2) + _version = (2, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -36,7 +36,7 @@ class Elfs(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index aec3eeb14..9f29ef74e 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -17,7 +17,7 @@ class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls): @@ -29,7 +29,7 @@ class Envars(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index b9ced73f3..40e992069 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -20,7 +20,7 @@ class Kthreads(plugins.PluginInterface): """Enumerates kthread functions""" _required_framework_version = (2, 11, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -34,7 +34,7 @@ class Kthreads(plugins.PluginInterface): name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py index 7ec1f7f7f..e251b5689 100644 --- a/volatility3/framework/plugins/linux/library_list.py +++ b/volatility3/framework/plugins/linux/library_list.py @@ -21,7 +21,7 @@ class LibraryList(interfaces.plugins.PluginInterface): """Enumerate libraries loaded into processes""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls): @@ -32,7 +32,7 @@ class LibraryList(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pids", diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 802954f43..daa8e5a3d 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -110,7 +110,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists open files for each processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (2, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -121,7 +121,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 0b10e60c6..e45688e97 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -18,7 +18,7 @@ class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -29,7 +29,7 @@ class Malfind(interfaces.plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 65775c4aa..b4f80e4f5 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -36,7 +36,7 @@ class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 2, 0) - _version = (1, 2, 2) + _version = (1, 2, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -47,7 +47,7 @@ class MountInfo(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 893eea71e..441c6bc93 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -21,7 +21,7 @@ class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @@ -35,7 +35,7 @@ class Maps(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index 60424a990..a544c9d67 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -15,7 +15,7 @@ class PsAux(plugins.PluginInterface): """Lists processes with their command line arguments""" _required_framework_version = (2, 13, 0) - _version = (1, 1, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls): @@ -27,7 +27,7 @@ class PsAux(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index 271c0e75e..6493f22b9 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -19,7 +19,7 @@ class Ptrace(plugins.PluginInterface): """Enumerates ptrace's tracer and tracee tasks""" _required_framework_version = (2, 10, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -30,7 +30,7 @@ class Ptrace(plugins.PluginInterface): architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index aee0b1e2e..7376bcbee 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -438,7 +438,7 @@ class Sockstat(plugins.PluginInterface): """Lists all network connections for all processes.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 1) + _version = (3, 0, 2) @classmethod def get_requirements(cls): @@ -455,7 +455,7 @@ class Sockstat(plugins.PluginInterface): name="lsof", plugin=lsof.Lsof, version=(2, 0, 0) ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/vmaregexscan.py b/volatility3/framework/plugins/linux/vmaregexscan.py index 4446fc550..8fb96da1e 100644 --- a/volatility3/framework/plugins/linux/vmaregexscan.py +++ b/volatility3/framework/plugins/linux/vmaregexscan.py @@ -21,7 +21,7 @@ class VmaRegExScan(plugins.PluginInterface): """Scans all virtual memory areas for tasks using RegEx.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) MAXSIZE_DEFAULT = 128 @@ -35,7 +35,7 @@ class VmaRegExScan(plugins.PluginInterface): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 650fcf078..4db23e50b 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -18,7 +18,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -31,7 +31,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): optional=True, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(3, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) From ddb4db5eab4d5e6a9e9ea40b81339be89a8be4cc Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 18 Dec 2024 16:22:27 +1100 Subject: [PATCH 724/979] linux: pslist: Handle cases where credential IDs are unavailable due to an invalid credential pointer --- volatility3/framework/plugins/linux/pslist.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 2244b91ce..641a27b92 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -216,10 +216,10 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): task_fields.user_tid, task_fields.user_ppid, task_fields.name, - task_fields.uid, - task_fields.gid, - task_fields.euid, - task_fields.egid, + task_fields.uid or renderers.NotAvailableValue(), + task_fields.gid or renderers.NotAvailableValue(), + task_fields.euid or renderers.NotAvailableValue(), + task_fields.egid or renderers.NotAvailableValue(), task_fields.creation_time or renderers.NotAvailableValue(), file_output, ) From 3da14b3873a3f19e291e21b79735ae4a8c5794a8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 06:40:19 +0000 Subject: [PATCH 725/979] Use generator expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PEP 289 – Generator Expressions. --- volatility3/cli/text_renderer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 937ba4ef4..b1944ae5a 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -176,7 +176,7 @@ class QuickTextRenderer(CLIRenderer): format_hints.HexBytes: optional(hex_bytes_as_text), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), interfaces.renderers.Disassembly: optional(display_disassembly), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), "default": optional(lambda x: f"{x}"), } @@ -256,7 +256,7 @@ class CSVRenderer(CLIRenderer): format_hints.HexBytes: optional(hex_bytes_as_text), format_hints.MultiTypeData: optional(multitypedata_as_text), interfaces.renderers.Disassembly: optional(display_disassembly), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), "default": optional(lambda x: f"{x}"), } @@ -450,7 +450,7 @@ class JsonRenderer(CLIRenderer): format_hints.HexBytes: quoted_optional(hex_bytes_as_text), interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: lambda x: ( x.isoformat() if not isinstance(x, interfaces.renderers.BaseAbsentValue) From 860a8146fbd529da524bac2282fb00a31b3568bc Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 06:44:23 +0000 Subject: [PATCH 726/979] Use floor division --- volatility3/framework/layers/msf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 8d84a774b..03e144e25 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -225,7 +225,7 @@ class PdbMSFStream(linear.LinearlyMappedLayer): returned = 0 page_size = self._pdb_layer.page_size while length > 0: - page = math.floor((offset + returned) / page_size) + page = (offset + returned) // page_size page_position = (offset + returned) % page_size chunk_size = min(page_size - page_position, length) if page >= self._pages_len: From d7f678879d8982b1222e6f5676caed4b0e5a9f70 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Wed, 18 Dec 2024 15:17:26 +0800 Subject: [PATCH 727/979] Minor improvements for `mypy` --- pyproject.toml | 3 ++- volatility3/cli/__init__.py | 6 ++--- volatility3/cli/text_filter.py | 2 +- volatility3/cli/volshell/generic.py | 10 ++++---- volatility3/cli/volshell/linux.py | 6 ++--- volatility3/cli/volshell/mac.py | 6 ++--- volatility3/cli/volshell/windows.py | 6 ++--- volatility3/framework/__init__.py | 7 +++--- volatility3/framework/automagic/stacker.py | 4 +++- .../framework/automagic/symbol_cache.py | 9 +++++++ .../framework/configuration/requirements.py | 24 +++++++++---------- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/automagic.py | 2 +- .../framework/interfaces/configuration.py | 7 +++--- volatility3/framework/interfaces/context.py | 14 +++++++++-- volatility3/framework/interfaces/layers.py | 2 +- volatility3/framework/interfaces/objects.py | 1 + volatility3/framework/interfaces/renderers.py | 4 ++-- volatility3/framework/interfaces/symbols.py | 1 + .../framework/layers/scanners/__init__.py | 2 +- volatility3/framework/objects/__init__.py | 6 ++--- volatility3/framework/plugins/linux/pslist.py | 6 +++-- volatility3/framework/plugins/mac/pslist.py | 6 +++-- volatility3/framework/plugins/timeliner.py | 4 +++- .../framework/plugins/windows/modules.py | 4 ++-- .../framework/plugins/windows/pedump.py | 2 +- .../framework/plugins/windows/poolscanner.py | 2 +- .../framework/plugins/windows/pslist.py | 6 ++--- .../framework/plugins/windows/psscan.py | 2 +- .../plugins/windows/registry/printkey.py | 10 ++++---- .../plugins/windows/scheduled_tasks.py | 1 - volatility3/framework/renderers/__init__.py | 2 +- volatility3/framework/symbols/__init__.py | 8 +++---- .../framework/symbols/generic/__init__.py | 6 ++--- volatility3/framework/symbols/intermed.py | 4 ++-- .../symbols/linux/extensions/__init__.py | 2 +- volatility3/framework/symbols/mac/__init__.py | 4 ++-- .../symbols/mac/extensions/__init__.py | 2 +- volatility3/framework/symbols/metadata.py | 2 +- .../symbols/windows/extensions/__init__.py | 4 +++- .../symbols/windows/extensions/pool.py | 2 +- .../framework/symbols/windows/pdbconv.py | 4 +++- .../framework/symbols/windows/pdbutil.py | 14 +++++------ 43 files changed, 127 insertions(+), 94 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7035f7a15..cc09922e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dev = [ "jsonschema>=4.23.0,<5", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", + "types-jsonschema>=4.23.0,<5", ] test = [ @@ -68,7 +69,7 @@ include = ["volatility3*"] mypy_path = "./stubs" show_traceback = true -[tool.mypy.overrides] +[[tool.mypy.overrides]] ignore_missing_imports = true [tool.ruff] diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index da046de57..6172a17f3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,7 +19,7 @@ import os import sys import tempfile import traceback -from typing import Any, Dict, List, Tuple, Type, Union +from typing import Any, Dict, List, Optional, Tuple, Type, Union from urllib import parse, request try: @@ -64,7 +64,7 @@ class PrintedProgress: def __init__(self): self._max_message_len = 0 - def __call__(self, progress: Union[int, float], description: str = None): + def __call__(self, progress: Union[int, float], description: Optional[str] = None): """A simple function for providing text-based feedback. .. warning:: Only for development use. @@ -81,7 +81,7 @@ class PrintedProgress: class MuteProgress(PrintedProgress): """A dummy progress handler that produces no output when called.""" - def __call__(self, progress: Union[int, float], description: str = None): + def __call__(self, progress: Union[int, float], description: Optional[str] = None): pass diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 955d647f5..6bd6878a5 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -74,7 +74,7 @@ class ColumnFilter: """Identifies whether an item is found in the appropriate column""" try: if self.regex: - return re.search(self.pattern, f"{item}") + return bool(re.search(self.pattern, f"{item}")) return self.pattern in f"{item}" except OSError: return False diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 93a75ca19..12f5499f6 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -240,7 +240,7 @@ class Volshell(interfaces.plugins.PluginInterface): return None return self.context.modules[self.current_kernel_name] - def change_layer(self, layer_name: str = None): + def change_layer(self, layer_name: Optional[str] = None): """Changes the current default layer""" if not layer_name: layer_name = self.current_layer @@ -250,7 +250,7 @@ class Volshell(interfaces.plugins.PluginInterface): self.__current_layer = layer_name sys.ps1 = f"({self.current_layer}) >>> " - def change_symbol_table(self, symbol_table_name: str = None): + def change_symbol_table(self, symbol_table_name: Optional[str] = None): """Changes the current_symbol_table""" if not symbol_table_name: print("No symbol table provided, not changing current symbol table") @@ -262,7 +262,7 @@ class Volshell(interfaces.plugins.PluginInterface): self.__current_symbol_table = symbol_table_name print(f"Current Symbol Table: {self.current_symbol_table}") - def change_kernel(self, kernel_name: str = None): + def change_kernel(self, kernel_name: Optional[str] = None): if not kernel_name: print("No kernel module name provided, not changing current kernel") if kernel_name not in self.context.modules: @@ -347,7 +347,7 @@ class Volshell(interfaces.plugins.PluginInterface): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if not isinstance( @@ -479,7 +479,7 @@ class Volshell(interfaces.plugins.PluginInterface): if treegrid is not None: self.render_treegrid(treegrid) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: print("No symbol table provided") diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index c5e555ec7..41b86f78b 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -61,7 +61,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -69,7 +69,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 2b32ad677..0ed35eb27 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -63,7 +63,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -71,7 +71,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 5c2190c02..303d4d5c3 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -60,7 +60,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -68,7 +68,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index c9a2c92ea..754939460 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -12,7 +12,7 @@ import inspect import logging import os import traceback -from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar +from typing import Any, Dict, Generator, List, Optional, Tuple, Type, TypeVar from volatility3.framework import constants, interfaces @@ -58,7 +58,7 @@ class NonInheritable: self.default_value = value self.cls = cls - def __get__(self, obj: Any, get_type: Type = None) -> Any: + def __get__(self, obj: Any, get_type: Type = Optional[None]) -> Any: if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) @@ -185,8 +185,7 @@ def _zipwalk(path: str): zip_results[os.path.join(path, os.path.dirname(file.filename))] = ( dirlist ) - for value in zip_results: - yield value, zip_results[value] + yield from zip_results.items() def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: diff --git a/volatility3/framework/automagic/stacker.py b/volatility3/framework/automagic/stacker.py index c251d3c46..596864264 100644 --- a/volatility3/framework/automagic/stacker.py +++ b/volatility3/framework/automagic/stacker.py @@ -166,7 +166,9 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): cls, context: interfaces.context.ContextInterface, initial_layer: str, - stack_set: List[Type[interfaces.automagic.StackerLayerInterface]] = None, + stack_set: Optional[ + List[Type[interfaces.automagic.StackerLayerInterface]] + ] = None, progress_callback: constants.ProgressCallback = None, ): """Stacks as many possible layers on top of the initial layer as can be done. diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 9fad506ae..065eb6d43 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -104,9 +104,11 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): for subclazz in framework.class_subclasses(IdentifierProcessor): self._classifiers[subclazz.operating_system] = subclazz + @abstractmethod def add_identifier(self, location: str, operating_system: str, identifier: str): """Adds an identifier to the store""" + @abstractmethod def find_location( self, identifier: bytes, operating_system: Optional[str] ) -> Optional[str]: @@ -120,15 +122,18 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): The location of the symbols file that matches the identifier """ + @abstractmethod def get_local_locations(self) -> Iterable[str]: """Returns a list of all the local locations""" + @abstractmethod def update(self): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. """ + @abstractmethod def get_identifier_dictionary( self, operating_system: Optional[str] = None, local_only: bool = False ) -> Dict[bytes, str]: @@ -142,12 +147,15 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): A dictionary of identifiers mapped to a location """ + @abstractmethod def get_identifier(self, location: str) -> Optional[bytes]: """Returns an identifier based on a specific location or None""" + @abstractmethod def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: """Returns all identifiers for a particular operating system""" + @abstractmethod def get_location_statistics( self, location: str ) -> Optional[Tuple[int, int, int, int]]: @@ -157,6 +165,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): A tuple of base_types, types, enums, symbols, or None is location not found """ + @abstractmethod def get_hash(self, location: str) -> Optional[str]: """Returns the hash of the JSON from within a location ISF""" diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 0cfaf5693..812b8ec59 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -11,7 +11,7 @@ expect to be in the context (such as particular layers or symboltables). import abc import logging import os -from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type +from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, Type from urllib import parse, request from volatility3.framework import constants, interfaces @@ -314,11 +314,11 @@ class TranslationLayerRequirement( def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: interfaces.configuration.ConfigSimpleType = None, optional: bool = False, - oses: List = None, - architectures: List = None, + oses: Optional[List] = None, + architectures: Optional[List[str]] = None, ) -> None: """Constructs a Translation Layer Requirement. @@ -526,18 +526,18 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): description: Optional[str] = None, default: bool = False, optional: bool = False, - component: Type[interfaces.configuration.VersionableInterface] = None, + component: Optional[Type[interfaces.configuration.VersionableInterface]] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: if version is None: raise TypeError("Version cannot be None") + if component is None: + raise TypeError("Component cannot be None") if description is None: description = f"Version {'.'.join(str(x) for x in version)} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) - if component is None: - raise TypeError("Component cannot be None") self._component: Type[interfaces.configuration.VersionableInterface] = component self._version = version @@ -546,7 +546,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): context: interfaces.context.ContextInterface, config_path: str, accumulator: Optional[ - List[interfaces.configuration.VersionableInterface] + Set[interfaces.configuration.VersionableInterface] ] = None, ) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type @@ -580,7 +580,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): ) if result: - result.update({config_path: self}) + result[config_path] = self return result context.config[interfaces.configuration.path_join(config_path, self.name)] = ( @@ -604,10 +604,10 @@ class PluginRequirement(VersionRequirement): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, optional: bool = False, - plugin: Type[interfaces.plugins.PluginInterface] = None, + plugin: Optional[Type[interfaces.plugins.PluginInterface]] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: super().__init__( @@ -627,7 +627,7 @@ class ModuleRequirement( def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, architectures: Optional[List[str]] = None, optional: bool = False, diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 5111b168a..f527544c0 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -229,7 +229,7 @@ class Module(interfaces.context.ModuleInterface): def object( self, object_type: str, - offset: int = None, + offset: Optional[int] = None, native_layer_name: Optional[str] = None, absolute: bool = False, **kwargs, diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index 0867b1608..4ac386fc0 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -42,7 +42,7 @@ class AutomagicInterface( priority = 10 """An ordering to indicate how soon this automagic should be run""" - exclusion_list = [] + exclusion_list: List[str] = [] """A list of plugin categories (typically operating systems) which the plugin will not operate on""" def __init__( diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index cbbf7e342..2e4f580a7 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -82,7 +82,7 @@ class HierarchicalDict(collections.abc.Mapping): def __init__( self, - initial_dict: Dict[str, "SimpleTypeRequirement"] = None, + initial_dict: Optional[Dict[str, "SimpleTypeRequirement"]] = None, separator: str = CONFIG_SEPARATOR, ) -> None: """ @@ -328,7 +328,7 @@ class RequirementInterface(metaclass=ABCMeta): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: ConfigSimpleType = None, optional: bool = False, ) -> None: @@ -618,7 +618,7 @@ class ConstructableRequirementInterface(RequirementInterface): self, context: "interfaces.context.ContextInterface", config_path: str, - requirement_dict: Dict[str, object] = None, + requirement_dict: Optional[Dict[str, object]] = None, ) -> Optional["interfaces.objects.ObjectInterface"]: """Constructs the class, handing args and the subrequirements as parameters to __init__""" @@ -652,6 +652,7 @@ class ConstructableRequirementInterface(RequirementInterface): class ConfigurableRequirementInterface(RequirementInterface): """Simple Abstract class to provide build_required_config.""" + @abstractmethod def build_configuration( self, context: "interfaces.context.ContextInterface", diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 8b5e816e8..a87e0f1e8 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -85,7 +85,7 @@ class ContextInterface(metaclass=ABCMeta): object_type: Union[str, "interfaces.objects.Template"], layer_name: str, offset: int, - native_layer_name: str = None, + native_layer_name: Optional[str] = None, **arguments, ) -> "interfaces.objects.ObjectInterface": """Object factory, takes a context, symbol, offset and optional @@ -114,6 +114,7 @@ class ContextInterface(metaclass=ABCMeta): """ return copy.deepcopy(self) + @abstractmethod def module( self, module_name: str, @@ -232,7 +233,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): def object( self, object_type: str, - offset: int = None, + offset: Optional[int] = None, native_layer_name: Optional[str] = None, absolute: bool = False, **kwargs, @@ -277,27 +278,35 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): symbol = self.get_symbol(name) return self.offset + symbol.address + @abstractmethod def get_type(self, name: str) -> "interfaces.objects.Template": """Returns a type from the module's symbol table.""" + @abstractmethod def get_symbol(self, name: str) -> "interfaces.symbols.SymbolInterface": """Returns a symbol object from the module's symbol table.""" + @abstractmethod def get_enumeration(self, name: str) -> "interfaces.objects.Template": """Returns an enumeration from the module's symbol table.""" + @abstractmethod def has_type(self, name: str) -> bool: """Determines whether a type is present in the module's symbol table.""" + @abstractmethod def has_symbol(self, name: str) -> bool: """Determines whether a symbol is present in the module's symbol table.""" + @abstractmethod def has_enumeration(self, name: str) -> bool: """Determines whether an enumeration is present in the module's symbol table.""" + @abstractmethod def symbols(self) -> List: """Lists the symbols contained in the symbol table for this module""" + @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: """Returns the symbols within table_name (or this module if not specified) that live at the specified absolute offset provided.""" @@ -343,6 +352,7 @@ class ModuleContainer(collections.abc.Mapping): def __iter__(self): return iter(self._modules) + @abstractmethod def free_module_name(self, prefix: str = "module") -> str: """Returns an unused table name to ensure no collision occurs when inserting a symbol table.""" diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 56798aca9..a90a78667 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -210,7 +210,7 @@ class DataLayerInterface( context: interfaces.context.ContextInterface, scanner: ScannerInterface, progress_callback: constants.ProgressCallback = None, - sections: Iterable[Tuple[int, int]] = None, + sections: Optional[Iterable[Tuple[int, int]]] = None, ) -> Iterable[Any]: """Scans a Translation layer by chunk. diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 51d25510d..23c90b13b 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -374,6 +374,7 @@ class Template: f"{self.__class__.__name__} object has no attribute {attr}" ) + @abc.abstractmethod def __call__( self, context: "interfaces.context.ContextInterface", diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index 7105274c0..e26164ee7 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -183,7 +183,7 @@ class TreeGrid(metaclass=ABCMeta): @abstractmethod def populate( self, - function: VisitorSignature = None, + function: Optional[VisitorSignature] = None, initial_accumulator: Any = None, fail_on_errors: bool = True, ) -> Optional[Exception]: @@ -235,7 +235,7 @@ class TreeGrid(metaclass=ABCMeta): node: Optional[TreeNode], function: VisitorSignature, initial_accumulator: _Type, - sort_key: ColumnSortKey = None, + sort_key: Optional[ColumnSortKey] = None, ) -> None: """Visits all the nodes in a tree, calling function on each one. diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index ead91fb4d..b8712e38d 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -256,6 +256,7 @@ class SymbolSpaceInterface(collections.abc.Mapping): """An interface for the container that holds all the symbol-containing tables for use within a context.""" + @abstractmethod def free_table_name(self, prefix: str = "layer") -> str: """Returns an unused table name to ensure no collision occurs when inserting a symbol table.""" diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index f54b44ff4..be9f1c39a 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -72,7 +72,7 @@ class MultiStringScanner(layers.ScannerInterface): return None for char in value: - trie[char] = trie.get(char, {}) + trie.setdefault(char, {}) trie = trie[char] # Mark the end of a string diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 5846da070..869d4dae6 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -152,7 +152,7 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): type_name: str, object_info: interfaces.objects.ObjectInformation, data_format: DataFormatInfo, - new_value: TUnion[int, float, bool, bytes, str] = None, + new_value: Optional[TUnion[int, float, bool, bytes, str]] = None, **kwargs, ) -> "PrimitiveObject": """Creates the appropriate class and returns it so that the native type @@ -601,7 +601,7 @@ class Enumeration(interfaces.objects.ObjectInterface, int): inverse_choices[v] = k return inverse_choices - def lookup(self, value: int = None) -> str: + def lookup(self, value: Optional[int] = None) -> str: """Looks up an individual value and returns the associated name. If multiple identifiers map to the same value, the first matching identifier will be returned @@ -690,7 +690,7 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): type_name: str, object_info: interfaces.objects.ObjectInformation, count: int = 0, - subtype: templates.ObjectTemplate = None, + subtype: Optional[templates.ObjectTemplate] = None, ) -> None: super().__init__(context=context, type_name=type_name, object_info=object_info) self._vol["count"] = count diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index a6d2e6538..82b8dcc67 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import datetime -from typing import Any, Callable, Iterable, List, Tuple +from typing import Any, Callable, Iterable, List, Optional, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -58,7 +58,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ] @classmethod - def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[Any], bool]: + def create_pid_filter( + cls, pid_list: Optional[List[int]] = None + ) -> Callable[[Any], bool]: """Constructs a filter function for process IDs. Args: diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 74d044ba9..8c5e5c1a5 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Dict, Iterable, List +from typing import Callable, Dict, Iterable, List, Optional from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -82,7 +82,9 @@ class PsList(interfaces.plugins.PluginInterface): return list_tasks @classmethod - def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[int], bool]: + def create_pid_filter( + cls, pid_list: Optional[List[int]] = None + ) -> Callable[[int], bool]: def filter_func(_): return False diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 4e483922b..0f4064d79 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -54,7 +54,9 @@ class Timeliner(interfaces.plugins.PluginInterface): self.automagics: Optional[List[interfaces.automagic.AutomagicInterface]] = None @classmethod - def get_usable_plugins(cls, selected_list: List[str] = None) -> List[Type]: + def get_usable_plugins( + cls, selected_list: Optional[List[str]] = None + ) -> List[Type]: # Initialize for the run plugin_list = list(framework.class_subclasses(TimeLinerInterface)) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index a3677ad34..85eb474a8 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import Generator, Iterable, List +from typing import Generator, Iterable, List, Optional from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -133,7 +133,7 @@ class Modules(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - pids: List[int] = None, + pids: Optional[List[int]] = None, ) -> Generator[str, None, None]: """Build a cache of possible virtual layers, in priority starting with the primary/kernel layer. Then keep one layer per session by cycling diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 85d5d14d1..678652624 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -96,7 +96,7 @@ class PEDump(interfaces.plugins.PluginInterface): pe_table_name: str, ldr_entry: interfaces.objects.ObjectInterface, open_method: Type[interfaces.plugins.FileHandlerInterface], - layer_name: str = None, + layer_name: Optional[str] = None, prefix: str = "", ) -> Optional[str]: """Extracts the PE file referenced an LDR_DATA_TABLE_ENTRY (DLL, kernel module) instance diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 8c56d202d..efde09638 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -183,7 +183,7 @@ class PoolScanner(plugins.PluginInterface): @staticmethod def builtin_constraints( - symbol_table: str, tags_filter: List[bytes] = None + symbol_table: str, tags_filter: Optional[List[bytes]] = None ) -> List[PoolConstraint]: """Get built-in PoolConstraints given a list of pool tags. diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index f262aeae6..579a235d8 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Iterator, List, Type +from typing import Callable, Iterator, List, Optional, Type from volatility3.framework import renderers, interfaces, layers, exceptions, constants from volatility3.framework.configuration import requirements @@ -114,7 +114,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def create_pid_filter( - cls, pid_list: List[int] = None, exclude: bool = False + cls, pid_list: Optional[List[int]] = None, exclude: bool = False ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process IDs. @@ -171,7 +171,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def create_name_filter( - cls, name_list: List[str] = None, exclude: bool = False + cls, name_list: Optional[List[str]] = None, exclude: bool = False ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process names. diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 86eb47300..cdf344ee6 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -89,7 +89,7 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, layer_name: str, - offset: int = None, + offset: Optional[int] = None, physical: bool = True, exclude: bool = False, ) -> Callable[[interfaces.objects.ObjectInterface], bool]: diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 4fe3f97fb..ed926805b 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import List, Sequence, Iterable, Tuple, Union +from typing import List, Optional, Sequence, Iterable, Tuple, Union from volatility3.framework import objects, renderers, exceptions, interfaces, constants from volatility3.framework.configuration import requirements @@ -51,7 +51,7 @@ class PrintKey(interfaces.plugins.PluginInterface): def key_iterator( cls, hive: RegistryHive, - node_path: Sequence[objects.StructType] = None, + node_path: Optional[Sequence[objects.StructType]] = None, recurse: bool = False, ) -> Iterable[ Tuple[ @@ -121,7 +121,7 @@ class PrintKey(interfaces.plugins.PluginInterface): def _printkey_iterator( self, hive: RegistryHive, - node_path: Sequence[objects.StructType] = None, + node_path: Optional[Sequence[objects.StructType]] = None, recurse: bool = False, ): """Method that wraps the more generic key_iterator, to provide output @@ -242,8 +242,8 @@ class PrintKey(interfaces.plugins.PluginInterface): self, layer_name: str, symbol_table: str, - hive_offsets: List[int] = None, - key: str = None, + hive_offsets: Optional[List[int]] = None, + key: Optional[str] = None, recurse: bool = False, ): for hive in hivelist.HiveList.list_hives( diff --git a/volatility3/framework/plugins/windows/scheduled_tasks.py b/volatility3/framework/plugins/windows/scheduled_tasks.py index 277a0d856..6dd5613c4 100644 --- a/volatility3/framework/plugins/windows/scheduled_tasks.py +++ b/volatility3/framework/plugins/windows/scheduled_tasks.py @@ -270,7 +270,6 @@ class _ScheduledTasksReader(io.BytesIO): return val def read_aligned_bstring_expand_sz(self) -> Optional[str]: - # type: () -> Optional[str] sz = self.read_aligned_u4() if sz is None: return None diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 39ce1135d..112e93751 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -214,7 +214,7 @@ class TreeGrid(interfaces.renderers.TreeGrid): def populate( self, - function: interfaces.renderers.VisitorSignature = None, + function: Optional[interfaces.renderers.VisitorSignature] = None, initial_accumulator: Any = None, fail_on_errors: bool = True, ) -> Optional[Exception]: diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index a8753bd4d..87f2288d7 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -53,10 +53,10 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} - def clear_symbol_cache(self, table_name: str = None) -> None: + def clear_symbol_cache(self, table_name: Optional[str] = None) -> None: """Clears the symbol cache for the specified table name. If no table name is specified, the caches of all symbol tables are cleared.""" - table_list: List[interfaces.symbols.BaseSymbolTableInterface] = list() + table_list: List[interfaces.symbols.BaseSymbolTableInterface] = [] if table_name is None: table_list = list(self._dict.values()) else: @@ -81,7 +81,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): yield table + constants.BANG + symbol_name def get_symbols_by_location( - self, offset: int, size: int = 0, table_name: str = None + self, offset: int, size: int = 0, table_name: Optional[str] = None ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = ( @@ -128,7 +128,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self, producer: str, validator: Callable[[Optional[Tuple], Optional[datetime.datetime]], bool], - tables: List[str] = None, + tables: Optional[List[str]] = None, ) -> bool: """Verifies the producer metadata and version of tables diff --git a/volatility3/framework/symbols/generic/__init__.py b/volatility3/framework/symbols/generic/__init__.py index 9d6da5aa4..7dd00fa75 100644 --- a/volatility3/framework/symbols/generic/__init__.py +++ b/volatility3/framework/symbols/generic/__init__.py @@ -4,7 +4,7 @@ import random import string -from typing import Union +from typing import Optional, Union from volatility3.framework import objects, interfaces @@ -14,8 +14,8 @@ class GenericIntelProcess(objects.StructType): self, context: interfaces.context.ContextInterface, dtb: Union[int, interfaces.objects.ObjectInterface], - config_prefix: str = None, - preferred_name: str = None, + config_prefix: Optional[str] = None, + preferred_name: Optional[str] = None, ) -> str: """Constructs a new layer based on the process's DirectoryTableBase.""" diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 8a28d732f..5b4aa22b8 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -86,7 +86,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): config_path: str, name: str, isf_url: str, - native_types: interfaces.symbols.NativeTableInterface = None, + native_types: Optional[interfaces.symbols.NativeTableInterface] = None, table_mapping: Optional[Dict[str, str]] = None, validate: bool = True, class_types: Optional[ @@ -319,7 +319,7 @@ class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass=ABCMeta): config_path: str, name: str, json_object: Any, - native_types: interfaces.symbols.NativeTableInterface = None, + native_types: Optional[interfaces.symbols.NativeTableInterface] = None, table_mapping: Optional[Dict[str, str]] = None, ) -> None: self._json_object = json_object diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7b025450c..4e2e80bc6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -308,7 +308,7 @@ class module(generic.GenericIntelProcess): class task_struct(generic.GenericIntelProcess): def add_process_layer( - self, config_prefix: str = None, preferred_name: str = None + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: """Constructs a new layer based on the process's DTB. diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index ee6dd10a3..dc54a8371 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Iterator, Any, Iterable, List, Tuple, Set +from typing import Iterator, Any, Iterable, List, Optional, Tuple, Set from volatility3.framework import interfaces, objects, exceptions, constants from volatility3.framework.symbols import intermed @@ -97,7 +97,7 @@ class MacUtilities(interfaces.configuration.VersionableInterface): context: interfaces.context.ContextInterface, handlers: Iterator[Any], target_address, - kernel_module_name: str = None, + kernel_module_name: Optional[str] = None, ): mod_name = "UNKNOWN" symbol_name = "N/A" diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index d2573fb95..cc700f209 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -18,7 +18,7 @@ class proc(generic.GenericIntelProcess): return self.task.dereference().cast("task") def add_process_layer( - self, config_prefix: str = None, preferred_name: str = None + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: """Constructs a new layer based on the process's DTB. diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 7e069e518..ea635f1f1 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -25,7 +25,7 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): return self._json_data.get("version", "") @property - def version(self) -> Optional[Tuple[int]]: + def version(self) -> Optional[Tuple[int, ...]]: """Returns the version of the ISF file producer""" version = self.version_string if not version: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 600f3e23f..d63f138b6 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -692,7 +692,9 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): return True - def add_process_layer(self, config_prefix: str = None, preferred_name: str = None): + def add_process_layer( + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None + ): """Constructs a new layer based on the process's DirectoryTableBase.""" parent_layer = self._context.layers[self.vol.layer_name] diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index 5a7847986..de5c8271b 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -362,7 +362,7 @@ class OBJECT_HEADER(objects.StructType): return True def get_object_type( - self, type_map: Dict[int, str], cookie: int = None + self, type_map: Dict[int, str], cookie: Optional[int] = None ) -> Optional[str]: """Across all Windows versions, the _OBJECT_HEADER embeds details on the type of object (i.e. process, file) but the way its embedded diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index ea2884bb2..248ef7d0c 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -984,7 +984,9 @@ if __name__ == "__main__": def __init__(self): self._max_message_len = 0 - def __call__(self, progress: Union[int, float], description: str = None): + def __call__( + self, progress: Union[int, float], description: Optional[str] = None + ): """A simple function for providing text-based feedback. .. warning:: Only for development use. diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 1a8644fa8..b5e8ca70a 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -36,7 +36,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): layer_name: str, offset: int, symbol_table_class: str = "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path: str = None, + config_path: Optional[str] = None, progress_callback: constants.ProgressCallback = None, ) -> Optional[str]: """Produces the name of a symbol table loaded from the offset for an MZ header @@ -388,8 +388,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, ) -> str: """Creates symbol table for a module in the specified layer_name. @@ -418,8 +418,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, create_module: bool = False, ) -> Tuple[Optional[str], Optional[str]]: if module_offset is None: @@ -478,8 +478,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, ) -> str: """Creates a module in the specified layer_name based on a pdb name. From cf7aabead44aa1aee02384c57860f7e3ba1a5a84 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 08:12:49 +0000 Subject: [PATCH 728/979] Change semi-colon to colon --- volatility3/framework/configuration/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 0cfaf5693..cc3ed847e 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -111,7 +111,7 @@ class ListRequirement(interfaces.configuration.RequirementInterface): Args: element_type: The (requirement) type of each element within the list - max_elements; The maximum number of acceptable elements this list can contain + max_elements: The maximum number of acceptable elements this list can contain min_elements: The minimum number of acceptable elements this list can contain """ super().__init__(*args, **kwargs) From 7464a3b883a50ee35a2e3c607d1906dd6d8c1807 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 18 Dec 2024 10:57:33 -0600 Subject: [PATCH 729/979] Windows Extensions: Fix potential AttributeErrors If `self.get_owner()` returns `None`, and the chained call to `is_valid()` is executed, an `AttributeError` will occur. This fixes two instances of this bug by intializing a local variable with the result of the `get_owner()` call, checking for `None`, and returning if that's the case. Also adds type-hints for these methods. --- .../symbols/windows/extensions/network.py | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 9b7573c2e..cfa2a7a30 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -4,7 +4,7 @@ import logging import socket -from typing import Dict, Tuple, List, Union +from typing import Dict, Tuple, List, Union, Optional from volatility3.framework import exceptions from volatility3.framework import objects, interfaces @@ -86,19 +86,29 @@ class _TCP_LISTENER(objects.StructType): except exceptions.InvalidAddressException: return None - def get_owner_pid(self): - if self.get_owner().is_valid(): - if self.get_owner().has_valid_member("UniqueProcessId"): - return self.get_owner().UniqueProcessId + def get_owner_pid(self) -> Optional[int]: + owner = self.get_owner() + + if owner is None: + return None + + if owner.is_valid(): + if owner.has_valid_member("UniqueProcessId"): + return owner.UniqueProcessId return None - def get_owner_procname(self): - if self.get_owner().is_valid(): - if self.get_owner().has_valid_member("ImageFileName"): - return self.get_owner().ImageFileName.cast( + def get_owner_procname(self) -> Optional[str]: + owner = self.get_owner() + + if owner is None: + return None + + if owner.is_valid(): + if owner.has_valid_member("ImageFileName"): + return owner.ImageFileName.cast( "string", - max_length=self.get_owner().ImageFileName.vol.count, + max_length=owner.ImageFileName.vol.count, errors="replace", ) From c865f4892c88d346d010a4d08ed75ecdb10a44c5 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:36:05 +0000 Subject: [PATCH 730/979] Slightly modify documentation --- doc/source/glossary.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index a9460b1a2..33e56883a 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -123,6 +123,11 @@ Page Table possible to use them as a way to map a particular address within a (potentially larger, but sparsely populated) virtual space to a concrete (and usually contiguous) physical space, through the process of :ref:`mapping`. +.. _Plugin: + +Plugin + Plugins are the "functions" of the volatility framework. They carry out algorithms on data stored in layers using objects constructed from symbols. Broadly, plugins take in a number of TranslationLayers (the data, which is a representation of part of an image, in a specified type described by templates) and outputs a TreeGrid. + .. _Pointer: Pointer From 1e2900d587acb1b752a1c8352bcf4fc112a9ec7a Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:39:03 +0000 Subject: [PATCH 731/979] Slightly modify documentation --- doc/source/glossary.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 33e56883a..04f1fb090 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -61,9 +61,7 @@ Map, mapping of the :ref:`Range`). Mappings can be seen as a mathematical function, and therefore volatility 3 attempts to use mathematical functional notation where possible. Within volatility a mapping is most often used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). - For further information, please see - `https://en.wikipedia.org/wiki/Function_(mathematics)`. - + For further information, please see `[Function (mathematics) in Wikipedia](https://en.wikipedia.org/wiki/Function_(mathematics))`. .. _Member: From 7eca407b6d8fc6123486c79121f0d6db2bf7dc8b Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 18 Dec 2024 11:34:25 -0600 Subject: [PATCH 732/979] Windows PEDump: Revert overwritten changes When #1364 was merged, it may not have been rebased onto the changes introduced in #1422, and they ended up overwritten to the old version. This reverts those changes. --- .../framework/plugins/windows/pedump.py | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 85d5d14d1..275775ddb 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -64,30 +64,27 @@ class PEDump(interfaces.plugins.PluginInterface): """ Returns the filename of the dump file or None """ - try: - file_handle = open_method(file_name) + with open_method(file_name) as file_handle: + try: + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base, + layer_name=layer_name, + ) - dos_header = context.object( - pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset=base, - layer_name=layer_name, - ) + for offset, data in dos_header.reconstruct(): + file_handle.seek(offset) + file_handle.write(data) + except ( + OSError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: + vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") + return None - for offset, data in dos_header.reconstruct(): - file_handle.seek(offset) - file_handle.write(data) - except ( - OSError, - exceptions.VolatilityException, - OverflowError, - ValueError, - ) as excp: - vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") - return None - finally: - file_handle.close() - - return file_handle.preferred_filename + return file_handle.preferred_filename @classmethod def dump_ldr_entry( From a9417edd2810e682474966d501786bfb8e3206ec Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:49:56 +0000 Subject: [PATCH 733/979] Slightly modify documentation --- doc/source/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 04f1fb090..d3bc9613a 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -61,7 +61,7 @@ Map, mapping of the :ref:`Range`). Mappings can be seen as a mathematical function, and therefore volatility 3 attempts to use mathematical functional notation where possible. Within volatility a mapping is most often used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). - For further information, please see `[Function (mathematics) in Wikipedia](https://en.wikipedia.org/wiki/Function_(mathematics))`. + For further information, please see `Function (mathematics) in Wikipedia_`. .. _Member: From 3c26955e34d68150e5d009557a66581fd7188bb4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 19 Dec 2024 12:02:16 +1100 Subject: [PATCH 734/979] xen_layer: fix potential uninitialized variable issue #1434 --- volatility3/framework/layers/xen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index e7aa0ccec..c0a5e1a7d 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -54,6 +54,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): segments = [] self._segment_headers = [] + segment_names = None for sindex in range(ehdr.e_shnum): shdr = self.context.object( From d14c777b4f2ba75ff06e1a39d4a23fa52473a5fa Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 19 Dec 2024 14:40:51 +1100 Subject: [PATCH 735/979] linux: vmcoreinfo layer: Update to use the new cache-manager --- volatility3/framework/automagic/linux.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index bb3bf2091..ce73ee969 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -224,19 +224,6 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): @staticmethod def _check_versions() -> bool: """Verify the versions of the required modules""" - - # Check SQlite cache version - sqlitecache_version_required = (1, 0, 0) - if not requirements.VersionRequirement.matches_required( - sqlitecache_version_required, symbol_cache.SqliteCache.version - ): - vollog.info( - "SQLiteCache version not suitable: required %s found %s", - sqlitecache_version_required, - symbol_cache.SqliteCache.version, - ) - return False - # Check VMCOREINFO API version vmcoreinfo_version_required = (1, 0, 0) if not requirements.VersionRequirement.matches_required( @@ -272,11 +259,9 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - identifiers_path = os.path.join( - constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + linux_banners = symbol_cache.load_cache_manager().get_identifier_dictionary( + operating_system="linux" ) - sqlite_cache = symbol_cache.SqliteCache(identifiers_path) - linux_banners = sqlite_cache.get_identifier_dictionary(operating_system="linux") if not linux_banners: # If we have no banners, don't bother scanning vollog.info( From d56f3a5537de12d4b188b2d2eeace213396feea5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 19 Dec 2024 14:49:23 +1100 Subject: [PATCH 736/979] linux: vmcoreinfo layer: Add review suggestions --- volatility3/framework/automagic/linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index ce73ee969..c2fc3306d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -339,8 +339,8 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): context.symbol_space.append(table) # Build the new layer - new_layer_name = context.layers.free_layer_name("IntelLayer") - config_path = join("IntelHelper", new_layer_name) + new_layer_name = context.layers.free_layer_name("primary") + config_path = join("vmcoreinfo", new_layer_name) kernel_banner = LinuxSymbolFinder.banner_config_key banner_str = banner.decode(encoding="latin-1") context.config[join(config_path, kernel_banner)] = banner_str From 7858af3d6d69e3054c1b2c3690e115257efc82b0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 19 Dec 2024 14:51:33 +1100 Subject: [PATCH 737/979] linux: vmcoreinfo layer: remove unused import --- volatility3/framework/automagic/linux.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index c2fc3306d..76c7f7ab3 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -2,7 +2,6 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import os import logging from typing import Optional, Tuple From c82d432b10258136ff0777dfec1fbf5844316132 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Thu, 19 Dec 2024 12:16:55 +0800 Subject: [PATCH 738/979] Typing fix --- volatility3/framework/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 754939460..a1925faef 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,7 +5,6 @@ # Check the python version to ensure it's suitable import glob import sys -from volatility3.framework import check_python_version as check_python_version import zipfile import importlib import inspect @@ -58,7 +57,7 @@ class NonInheritable: self.default_value = value self.cls = cls - def __get__(self, obj: Any, get_type: Type = Optional[None]) -> Any: + def __get__(self, obj: Any, get_type: Optional[Type] = None) -> Any: if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) From 7ed8b99e9845b2f982da4ef186bed137ba77af8f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 19 Dec 2024 15:50:17 +1100 Subject: [PATCH 739/979] linux: vmcoreinfo layer: Use the version class accessor instead of the internal class member --- volatility3/framework/automagic/linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 76c7f7ab3..58195744b 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -226,12 +226,12 @@ class LinuxIntelVMCOREINFOStacker(interfaces.automagic.StackerLayerInterface): # Check VMCOREINFO API version vmcoreinfo_version_required = (1, 0, 0) if not requirements.VersionRequirement.matches_required( - vmcoreinfo_version_required, linux.VMCoreInfo._version + vmcoreinfo_version_required, linux.VMCoreInfo.version ): vollog.info( "VMCOREINFO version not suitable: required %s found %s", vmcoreinfo_version_required, - linux.VMCoreInfo._version, + linux.VMCoreInfo.version, ) return False From 95e103d9ea6c556e93872f35cc1a83d8453ab94f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:35:34 +0000 Subject: [PATCH 740/979] Remove commented out import --- volatility3/framework/plugins/windows/shimcachemem.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index 59f33510d..b8e9b5bd7 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -17,8 +17,6 @@ from volatility3.framework.symbols.windows.extensions import pe, shimcache from volatility3.plugins import timeliner from volatility3.plugins.windows import modules, pslist, vadinfo -# from volatility3.plugins.windows import pslist, vadinfo, modules - vollog = logging.getLogger(__name__) From c56d9334f82f90e90d46e1026e43d5e209cf9466 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:00:27 +0000 Subject: [PATCH 741/979] Tweak comment --- volatility3/framework/plugins/windows/pe_symbols.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 002577241..21e657ab3 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -158,7 +158,7 @@ class PESymbolFinder: class PDBSymbolFinder(PESymbolFinder): """ - PESymbolFinder implementation for PDB modules + PESymbolFinder implementation for PDB modules """ def _do_get_address(self, name: str) -> Optional[int]: @@ -195,7 +195,7 @@ class PDBSymbolFinder(PESymbolFinder): class ExportSymbolFinder(PESymbolFinder): """ - PESymbolFinder implementation for PDB modules + PESymbolFinder implementation for PDB modules """ def _get_name(self, export: pefile.ExportData) -> Optional[str]: @@ -300,7 +300,7 @@ class PESymbols(interfaces.plugins.PluginInterface): base_address: int, ) -> Optional[pefile.PE]: """ - Attempts to pefile object from the bytes of the PE file + Attempts to create a pefile object from the bytes of the PE file Args: pe_table_name: name of the pe types table From 0e0c959cd0f3d4a7d14f5cd19a683609b64808f6 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:21:20 +0000 Subject: [PATCH 742/979] Swap two letters in a typo --- volatility3/framework/plugins/windows/psxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index e3ec216dd..b5ddd2ee5 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -25,7 +25,7 @@ class PsXView(plugins.PluginInterface): identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this plugin's output in a terminal.""" - # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality + # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the functionality # which the original plugin used to do it. # The sessions method is omitted because it begins with the list of processes found by Pslist anyway. From a11131b3d4767ec15619bab8084c4914ad528f75 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 14:59:58 +0000 Subject: [PATCH 743/979] Update how to write a simple plugin --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 39670a62d..84b921114 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -52,7 +52,7 @@ to be able to run properly. Any that are defined as optional need not necessari version = (2, 0, 0))] -This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how +This is a classmethod, so it can be called before the specific plugin object has been instantiated (in order to know how to instantiate the plugin). At the moment these requirements are fairly straightforward: :: From 4fe3db50df1a53d80d79ac100fdfaf577146fa86 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 16:32:30 +0000 Subject: [PATCH 744/979] Reorder requirements by type --- .../framework/plugins/windows/dlllist.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 57f19f620..35b9fb2dc 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -19,7 +19,7 @@ vollog = logging.getLogger(__name__) class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): - """Lists the loaded modules in a particular windows memory image.""" + """Lists the loaded DLLs in a particular windows memory image.""" _required_framework_version = (2, 0, 0) _version = (3, 0, 0) @@ -39,6 +39,9 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="psscan", component=psscan.PsScan, version=(1, 1, 0) ), + requirements.VersionRequirement( + name="pedump", component=pedump.PEDump, version=(1, 0, 0) + ), requirements.VersionRequirement( name="info", component=info.Info, version=(1, 0, 0) ), @@ -53,16 +56,16 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): description="Process offset in the physical address space", optional=True, ), - requirements.StringRequirement( - name="name", - description="Specify a regular expression to match dll name(s)", - optional=True, - ), requirements.IntRequirement( name="base", description="Specify a base virtual address in process memory", optional=True, ), + requirements.StringRequirement( + name="name", + description="Specify a regular expression to match dll name(s)", + optional=True, + ), requirements.BooleanRequirement( name="ignore-case", description="Specify case insensitivity for the regular expression name matching", @@ -75,9 +78,6 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): default=False, optional=True, ), - requirements.VersionRequirement( - name="pedump", component=pedump.PEDump, version=(1, 0, 0) - ), ] def _generator(self, procs): @@ -90,12 +90,15 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kuser = info.Info.get_kuser_structure( self.context, kernel.layer_name, kernel.symbol_table_name ) + nt_major_version = int(kuser.NtMajorVersion) nt_minor_version = int(kuser.NtMinorVersion) + # LoadTime only applies to versions higher or equal to Window 7 (6.1 and higher) dll_load_time_field = (nt_major_version > 6) or ( nt_major_version == 6 and nt_minor_version >= 1 ) + for proc in procs: proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() @@ -114,7 +117,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mod_re = re.compile(self.config["name"], flags) except re.error: vollog.debug( - "Error parsing regular expression: %s", self.config["name"] + f"Error parsing regular expression: {self.config["name"]}" ) return None From 59a5e85b504c48a381fb65f8fc2b42a84bf1bc9a Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:36:42 +0000 Subject: [PATCH 745/979] Reorder requirements by type --- volatility3/framework/plugins/windows/dlllist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 35b9fb2dc..65e337dfc 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -117,7 +117,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mod_re = re.compile(self.config["name"], flags) except re.error: vollog.debug( - f"Error parsing regular expression: {self.config["name"]}" + f'Error parsing regular expression: {self.config["name"]}' ) return None @@ -138,7 +138,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if dll_load_time_field: # Versions prior to 6.1 won't have the LoadTime attribute - # and 32bit version shouldn't have the Quadpart according to MSDN + # and 32-bit version shouldn't have the Quadpart according to MSDN try: DllLoadTime = conversion.wintime_to_datetime( entry.LoadTime.QuadPart From 8b35031d0f443184953d80263f2689e4dd0a059f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 00:37:31 +0000 Subject: [PATCH 746/979] Volshell: Bump linux.pslist plugin requirement --- volatility3/cli/volshell/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index c5e555ec7..72193201c 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -20,7 +20,7 @@ class Volshell(generic.Volshell): name="kernel", description="Linux kernel module" ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.IntRequirement( name="pid", description="Process ID", optional=True From 658d40335a15bc40d2ecb5679198d759858492b5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 13:06:14 +1100 Subject: [PATCH 747/979] testcases: Add basic volshell testcases for each OS image --- .github/workflows/test.yaml | 5 ++++ test/test_volatility.py | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dfc42499d..e07673175 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -42,6 +42,11 @@ jobs: - name: Testing... run: | + # VolShell + pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_windows_volshell -v + pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_linux_volshell -v + + # Volatility pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_windows -v pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_linux -v diff --git a/test/test_volatility.py b/test/test_volatility.py index 5bce07481..8ef6d8b70 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -54,13 +54,56 @@ def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]) return runvol(args, volatility, python) +def runvolshell(img, volshell, python, volshellargs=[], globalargs=[]): + args = ( + globalargs + + [ + "--single-location", + img, + "-q", + ] + + volshellargs + ) + + return runvol(args, volshell, python) + + # # TESTS # + +def basic_volshell_test(image, volatility, python): + # Basic VolShell test to verify requirements and ensure VolShell runs without crashing + + # FIXME: When the minimum Python version includes 3.12, replace the following with: + # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... + fd, filename = tempfile.mkstemp(suffix=".txt") + try: + with os.fdopen(fd, "w") as f: + f.write("exit()") + + rc, out, _err = runvolshell( + img=image, + volshell=volatility, + python=python, + volshellargs=["--script", filename], + ) + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + assert rc == 0 + assert out.count(b"\n") >= 4 + + # WINDOWS +def test_windows_volshell(image, volatility, python): + basic_volshell_test(image, volatility, python) + + def test_windows_pslist(image, volatility, python): rc, out, _err = runvol_plugin("windows.pslist.PsList", image, volatility, python) out = out.lower() @@ -332,6 +375,10 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): # LINUX +def test_linux_volshell(image, volatility, python): + basic_volshell_test(image, volatility, python) + + def test_linux_pslist(image, volatility, python): rc, out, _err = runvol_plugin("linux.pslist.PsList", image, volatility, python) @@ -770,6 +817,10 @@ def test_linux_hidden_modules(image, volatility, python): # MAC +def test_mac_volshell(image, volatility, python): + basic_volshell_test(image, volatility, python) + + def test_mac_pslist(image, volatility, python): rc, out, _err = runvol_plugin("mac.pslist.PsList", image, volatility, python) out = out.lower() From 90766f466f581561e17b2cc5e2c7d8eca56c56e6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 13:21:24 +1100 Subject: [PATCH 748/979] testcases: exclude volshell test from the volatility set --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e07673175..ce2722457 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -47,8 +47,8 @@ jobs: pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_linux_volshell -v # Volatility - pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_windows -v - pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_linux -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k "test_windows and not test_windows_volshell" -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k "test_linux and not test_linux_volshell" -v - name: Clean up post-test run: | From e6e0754fa523f9c0edb6a6e449bdbf08a16fa712 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Fri, 20 Dec 2024 10:46:54 +0800 Subject: [PATCH 749/979] Remove mypy overrides section --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cc09922e9..d695a4eac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,9 +69,6 @@ include = ["volatility3*"] mypy_path = "./stubs" show_traceback = true -[[tool.mypy.overrides]] -ignore_missing_imports = true - [tool.ruff] line-length = 88 target-version = "py38" From c317a45eb56a5da26ee50e2edc0d2e8c6821d262 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 14:31:40 +1100 Subject: [PATCH 750/979] testcases: Add missing operating system argument --- test/test_volatility.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 8ef6d8b70..f8a32fef3 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -73,7 +73,7 @@ def runvolshell(img, volshell, python, volshellargs=[], globalargs=[]): # -def basic_volshell_test(image, volatility, python): +def basic_volshell_test(image, volatility, python, globalargs): # Basic VolShell test to verify requirements and ensure VolShell runs without crashing # FIXME: When the minimum Python version includes 3.12, replace the following with: @@ -88,6 +88,7 @@ def basic_volshell_test(image, volatility, python): volshell=volatility, python=python, volshellargs=["--script", filename], + globalargs=globalargs, ) finally: with contextlib.suppress(FileNotFoundError): @@ -101,7 +102,7 @@ def basic_volshell_test(image, volatility, python): def test_windows_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python) + basic_volshell_test(image, volatility, python, globalargs=["-w"]) def test_windows_pslist(image, volatility, python): @@ -376,7 +377,7 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): def test_linux_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python) + basic_volshell_test(image, volatility, python, globalargs=["-l"]) def test_linux_pslist(image, volatility, python): @@ -818,7 +819,7 @@ def test_linux_hidden_modules(image, volatility, python): def test_mac_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python) + basic_volshell_test(image, volatility, python, globalargs=["-m"]) def test_mac_pslist(image, volatility, python): From d546c983f3012f3060aca2b2721fedcc48321370 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 14:33:36 +1100 Subject: [PATCH 751/979] testcases: Fix runvol* default list arguments --- test/test_volatility.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index f8a32fef3..47ef9769f 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -39,7 +39,9 @@ def runvol(args, volatility, python): return p.returncode, stdout, stderr -def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]): +def runvol_plugin(plugin, img, volatility, python, pluginargs=None, globalargs=None): + pluginargs = pluginargs or [] + globalargs = globalargs or [] args = ( globalargs + [ @@ -54,7 +56,9 @@ def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]) return runvol(args, volatility, python) -def runvolshell(img, volshell, python, volshellargs=[], globalargs=[]): +def runvolshell(img, volshell, python, volshellargs=None, globalargs=None): + volshellargs = volshellargs or [] + globalargs = globalargs or [] args = ( globalargs + [ From ef977efe62af8e4cc5baa96e8a20fb713ee7494f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 14:50:35 +1100 Subject: [PATCH 752/979] testcases: Improve volshell basic testcase calling ps() on each of them --- test/test_volatility.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 47ef9769f..bb7c9a851 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -80,12 +80,18 @@ def runvolshell(img, volshell, python, volshellargs=None, globalargs=None): def basic_volshell_test(image, volatility, python, globalargs): # Basic VolShell test to verify requirements and ensure VolShell runs without crashing + volshell_commands = [ + "print(ps())", + "exit()", + ] + # FIXME: When the minimum Python version includes 3.12, replace the following with: # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... fd, filename = tempfile.mkstemp(suffix=".txt") try: + volshell_script = "\n".join(volshell_commands) with os.fdopen(fd, "w") as f: - f.write("exit()") + f.write(volshell_script) rc, out, _err = runvolshell( img=image, @@ -101,12 +107,15 @@ def basic_volshell_test(image, volatility, python, globalargs): assert rc == 0 assert out.count(b"\n") >= 4 + return out + # WINDOWS def test_windows_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python, globalargs=["-w"]) + out = basic_volshell_test(image, volatility, python, globalargs=["-w"]) + assert out.count(b" 40 def test_windows_pslist(image, volatility, python): @@ -381,7 +390,8 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): def test_linux_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python, globalargs=["-l"]) + out = basic_volshell_test(image, volatility, python, globalargs=["-l"]) + assert out.count(b" 100 def test_linux_pslist(image, volatility, python): From 89564cca66a14f9a1707a6d3e056ef56e605b4f0 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 20 Dec 2024 05:48:42 +0000 Subject: [PATCH 753/979] Revert one f-string As part of the ruff linting we did recently all vollog messages should have explicitly been reverted back to %-formatting. --- volatility3/framework/plugins/windows/dlllist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 65e337dfc..1dafb6bf5 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -117,7 +117,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mod_re = re.compile(self.config["name"], flags) except re.error: vollog.debug( - f'Error parsing regular expression: {self.config["name"]}' + "Error parsing regular expression: %s", self.config["name"] ) return None From 15eb80b600e6a2114e232355a04cb1bbf5ce8972 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 20 Dec 2024 13:11:35 +0100 Subject: [PATCH 754/979] fix unbound page variable access --- volatility3/framework/plugins/windows/malfind.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 510719352..14362776b 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -120,8 +120,7 @@ class Malfind(interfaces.plugins.PluginInterface): vadinfo.winnt_protections, ) write_exec = "EXECUTE" in protection_string and "WRITE" in protection_string - dirty_page_check = False - + dirty_page = None if not write_exec: """ # Inspect "PAGE_EXECUTE_READ" VAD pages to detect @@ -135,12 +134,12 @@ class Malfind(interfaces.plugins.PluginInterface): try: # If we have a dirty page in a non writable "EXECUTE" region, it is suspicious. if proc_layer.is_dirty(page): - dirty_page_check = True + dirty_page = page break except exceptions.InvalidAddressException: # Abort as it is likely that other addresses in the same range will also fail. break - if not dirty_page_check: + if dirty_page is None: continue else: continue @@ -152,10 +151,10 @@ class Malfind(interfaces.plugins.PluginInterface): if cls.is_vad_empty(proc_layer, vad): continue - if dirty_page_check: + if dirty_page is not None: # Useful information to investigate the page content with volshell afterwards. vollog.warning( - f"[proc_id {proc_id}] Found suspicious DIRTY + {protection_string} page at {hex(page)}", + f"[proc_id {proc_id}] Found suspicious DIRTY + {protection_string} page at {hex(dirty_page)}", ) data = proc_layer.read(vad.get_start(), 64, pad=True) yield vad, data From 675ef6deea00cf4659bca50e0004dea3be43e8c1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:01:09 +0000 Subject: [PATCH 755/979] Generic: Fix up potential issue with isfinfo Fixes #1436 --- volatility3/framework/plugins/isfinfo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 78e78fb9e..1c2ac52e9 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -132,6 +132,7 @@ class IsfInfo(plugins.PluginInterface): valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning(f"Invalid ISF: {entry}") + continue yield ( 0, ( From f6f4c7b986c8c2adf7a73211f7aeb1375b00f255 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:11:42 +0000 Subject: [PATCH 756/979] Layers: Fix MSF page len on a possibly uninitialized variable Fixes #1441 --- volatility3/framework/layers/msf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 03e144e25..2b4fae963 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -194,7 +194,7 @@ class PdbMSFStream(linear.LinearlyMappedLayer): ) -> None: super().__init__(context, config_path, name, metadata) self._base_layer = self.config["base_layer"] - self._pages = self.config.get("pages", None) + self._pages = self.config.get("pages", []) self._pages_len = len(self._pages) if not self._pages: raise PDBFormatException(name, "Invalid/no pages specified") From eaca7b31bba1c9d8f934ad3e607fced26797abfd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:16:17 +0000 Subject: [PATCH 757/979] Layers: Fix vmware layer without a suitable meta Fixes #1442 --- volatility3/framework/layers/vmware.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 622ff0250..39fb21b63 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -57,6 +57,10 @@ class VmwareLayer(segmented.SegmentedLayer): ) meta_layer = self.context.layers.get(self._meta_layer, None) + if meta_layer is None: + raise exceptions.LayerException( + self._meta_layer, "VMware: Meta layer not found" + ) header_size = struct.calcsize(self.header_structure) data = meta_layer.read(0, header_size) magic, unknown, groupCount = struct.unpack(self.header_structure, data) From cc8fbd5b6824231d5f7bfb53a66560cb2e3fe3b1 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:34:52 +0000 Subject: [PATCH 758/979] Tweak a comment --- volatility3/framework/plugins/windows/svclist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index ea73247ce..a5825e1fe 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -41,7 +41,7 @@ class SvcList(svcscan.SvcScan): @classmethod def _get_exe_range(cls, proc) -> Optional[Tuple[int, int]]: """ - Returns a tuple of starting,ending address for + Returns a tuple of starting address and size of the the VAD containing services.exe """ From bed03dfbc7024d97c289b0c26f98d0627e105eee Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 21 Dec 2024 06:15:35 +0000 Subject: [PATCH 759/979] Refactor check of BasicType The intention is either a BasicType or a list where each element is only a BasicType (and not a list). --- volatility3/cli/volshell/generic.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 08132608b..a4b141c2d 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -554,11 +554,15 @@ class Volshell(interfaces.plugins.PluginInterface): del kwargs[argname] for keyword, val in kwargs.items(): - if not isinstance(val, (interfaces.configuration.BasicTypes, list)): - if all(isinstance(x, interfaces.configuration.BasicTypes) for x in val): - raise TypeError( - "Configurable values must be simple types (int, bool, str, bytes)" - ) + BasicType_or_list_of_BasicType = False # excludes list of lists + if isinstance(val, interfaces.configuration.BasicTypes): + BasicType_or_list_of_BasicType = True + if all(isinstance(x, interfaces.configuration.BasicTypes) for x in val): + BasicType_or_list_of_BasicType = True + if not BasicType_or_list_of_BasicType: + raise TypeError( + "Configurable values must be simple types (int, bool, str, bytes)" + ) self.context.config[config_path + "." + keyword] = val constructed = clazz(self.context, config_path, **constructor_args) From 88eb2aa88648180439e4d5311c9bf49ef2ae9363 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 21 Dec 2024 13:22:04 +0100 Subject: [PATCH 760/979] switch pillow to pyproject.toml --- pyproject.toml | 78 ++++++++++++++++++++++++++++++++++++++++++++---- requirements.txt | 29 ------------------ 2 files changed, 72 insertions(+), 35 deletions(-) delete mode 100644 requirements.txt diff --git a/pyproject.toml b/pyproject.toml index 2e1636a43..c8b2971e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,13 +8,55 @@ authors = [ ] requires-python = ">=3.8.0" license = { text = "VSL" } -dynamic = ["dependencies", "optional-dependencies", "version"] +dynamic = ["version"] + +dependencies = [ + "pefile>=2024.8.26", +] + +[project.optional-dependencies] +full = [ + "yara-python>=4.5.1,<5", + "capstone>=5.0.3,<6", + "pycryptodome>=3.21.0,<4", + "leechcorepyc>=2.19.2,<3; sys_platform != 'darwin'", + # https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst + # 10.0.0 dropped support for Python3.7 + # 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 + "pillow>=10.0.0,<11.0.0", +] + +cloud = [ + "gcsfs>=2024.10.0", + "s3fs>=2024.10.0", +] + +dev = [ + "volatility3[full,cloud]", + "jsonschema>=4.23.0,<5", + "pyinstaller>=6.11.0,<7", + "pyinstaller-hooks-contrib>=2024.9", +] + +test = [ + "volatility3[dev]", + "pytest>=8.3.3,<9", + "capstone>=5.0.3,<6", + "yara-x>=0.10.0,<1", +] + +docs = [ + "volatility3[dev]", + "sphinx>=8.0.0,<7", + "sphinx-autodoc-typehints>=2.5.0,<3", + "sphinx-rtd-theme>=3.0.1,<4", +] [project.urls] -Homepage = "https://github.com/volatilityfoundation/volatility3/" -"Bug Tracker" = "https://github.com/volatilityfoundation/volatility3/issues" -Documentation = "https://volatility3.readthedocs.io/" -"Source Code" = "https://github.com/volatilityfoundation/volatility3" +homepage = "https://github.com/volatilityfoundation/volatility3/" +documentation = "https://volatility3.readthedocs.io/" +repository = "https://github.com/volatilityfoundation/volatility3" +issues = "https://github.com/volatilityfoundation/volatility3/issues" [project.scripts] vol = "volatility3.cli:main" @@ -22,11 +64,35 @@ volshell = "volatility3.cli.volshell:main" [tool.setuptools.dynamic] version = { attr = "volatility3.framework.constants._version.PACKAGE_VERSION" } -dependencies = { file = "requirements-minimal.txt" } [tool.setuptools.packages.find] include = ["volatility3*"] +[tool.mypy] +mypy_path = "./stubs" +show_traceback = true + +[tool.mypy.overrides] +ignore_missing_imports = true + +[tool.ruff] +line-length = 88 +target-version = "py38" + +[tool.ruff.lint] +select = [ + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "G", # flake8-logging-format + "PIE", # flake8-pie + "UP", # pyupgrade +] + +ignore = [ + "E501", # ignore due to conflict with formatter +] + [build-system] requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 21c8f9a76..000000000 --- a/requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# Include the minimal requirements --r requirements-minimal.txt - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 - -# This is required for several plugins that perform malware analysis and disassemble code. -# It can also improve accuracy of Windows 8 and later memory samples. -# FIXME: Version 6.0.0 is incompatible (#1336) so we'll need an adaptor at some point -capstone>=3.0.5,<6.0.0 - -# This is required by plugins that decrypt passwords, password hashes, etc. -pycryptodome - -# This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0; sys_platform != 'darwin' - -# This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage -gcsfs>=2023.1.0 -s3fs>=2023.1.0 - -# This is required by plugins that manipulate pixels and images. -# https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst -# 10.0.0 dropped support for Python3.7 -# 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 -pillow>=10.0.0,<11.0.0 \ No newline at end of file From f6a54c5c48b6ba47a334956cb7994fecfcf14f0c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 21 Dec 2024 13:31:29 +0100 Subject: [PATCH 761/979] ruff fix --- volatility3/framework/plugins/linux/graphics/fbdev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 60e00d033..7144e081a 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -218,7 +218,7 @@ class Fbdev(interfaces.plugins.PluginInterface): fourcc = linux.LinuxUtilities.convert_fourcc_code(fb_info.var.grayscale) warn_msg = f"""Framebuffer "{id}" uses a FOURCC pixel format "{fourcc}" that isn't natively supported. You can try using ffmpeg to decode the raw buffer. Example usage: -"ffmpeg -pix_fmts" to list supported formats, then +"ffmpeg -pix_fmts" to list supported formats, then "ffmpeg -f rawvideo -video_size {fb_info.var.xres_virtual}x{fb_info.var.yres_virtual} -i .raw -pix_fmt output.png".""" vollog.warning(warn_msg) From a84d3611130b1ba42e1e3c317bc6f633adc669ab Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 21 Dec 2024 12:18:23 -0600 Subject: [PATCH 762/979] Add the suspended threads plugin from DEF CON 2024 --- .../plugins/windows/suspended_threads.py | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 volatility3/framework/plugins/windows/suspended_threads.py diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py new file mode 100644 index 000000000..2cc0673d7 --- /dev/null +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -0,0 +1,147 @@ +import logging + +from typing import Dict +from functools import partial + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +import volatility3.plugins.windows.pslist as pslist +import volatility3.plugins.windows.threads as threads +import volatility3.plugins.windows.pe_symbols as pe_symbols + +from volatility3.framework.objects import utility + +vollog = logging.getLogger(__name__) + + +class SuspendedThreads(interfaces.plugins.PluginInterface): + """Enumerates suspended threads.""" + + _required_framework_version = (2, 13, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="threads", component=threads.Threads, version=(1, 0, 0) + ), + ] + + def _generator(self): + """ + The goal of this plugin is to report on threads that are suspended + + Legitimate programs can start threads suspended but then will later resume them + + Subsets of malware techniques, such as EDR evasion and process hollowing, + create suspended threads and do not resume them. These are the threads that this + plugin is designed to catch. + + See the whitepaper from our DEF CON 2024 presentation for more details: + + https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + """ + kernel = self.context.modules[self.config["kernel"]] + + vads_cache: Dict[int, pe_symbols.PESymbols.ranges_type] = {} + + proc_modules = None + + # walk the threads of each process checking for suspended threads + for proc in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ): + for thread in threads.Threads.list_threads(kernel, proc): + try: + # we only care if the thread is suspended + if thread.Tcb.SuspendCount == 0: + continue + + # 4 == terminated + if thread.Tcb.State == 4: + continue + + owner_proc = thread.owning_process() + owner_proc_pid = thread.Cid.UniqueProcess + owner_proc_name = utility.array_to_string(owner_proc.ImageFileName) + thread_tid = thread.Cid.UniqueThread + thread_start_addr = thread.StartAddress + thread_win32_addr = thread.Win32StartAddress + except exceptions.InvalidAddressException: + continue + + # Nothing useful to report if a process doesn't have VADs.. Also a sign of smear/terminated + vads = pe_symbols.PESymbols.get_vads_for_process_cache(vads_cache, owner_proc) + if not vads: + continue + + # Only compute this if needed as its expensive and 99.9% of samples + # will not have suspended threads + if not proc_modules: + proc_modules = pe_symbols.PESymbols.get_process_modules( + self.context, kernel.layer_name, kernel.symbol_table_name, None + ) + + path_and_symbol = partial( + pe_symbols.PESymbols.path_and_symbol_for_address, + self.context, + self.config_path, + proc_modules, + ) + + start_file, start_sym = path_and_symbol(vads, thread_start_addr) + win32_file, win32_sym = path_and_symbol(vads, thread_win32_addr) + + # the only false positive found in mass scanning of samples + if start_file and start_file.endswith("\\WorkFoldersShell.dll"): + continue + + if win32_file and win32_file.endswith("\\WorkFoldersShell.dll"): + continue + + yield ( + 0, + ( + owner_proc_name, + owner_proc_pid, + thread_tid, + start_file or renderers.NotAvailableValue(), + start_sym or renderers.NotAvailableValue(), + format_hints.Hex(thread_start_addr), + win32_file or renderers.NotAvailableValue(), + win32_sym or renderers.NotAvailableValue(), + format_hints.Hex(thread_win32_addr), + ), + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("TID", int), + ("StartFile", str), + ("StartSymbol", str), + ("StartAddress", format_hints.Hex), + ("Win32StartFile", str), + ("Win32StartSymbol", str), + ("Win32StartAddress", format_hints.Hex), + ], + self._generator(), + ) + From d31ac276edb29721847f248c13953696c4a98a9c Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 21 Dec 2024 12:22:33 -0600 Subject: [PATCH 763/979] Add the suspended threads plugin from DEF CON 2024 --- volatility3/framework/plugins/windows/suspended_threads.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py index 2cc0673d7..6ddfecca7 100644 --- a/volatility3/framework/plugins/windows/suspended_threads.py +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -86,7 +86,9 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): continue # Nothing useful to report if a process doesn't have VADs.. Also a sign of smear/terminated - vads = pe_symbols.PESymbols.get_vads_for_process_cache(vads_cache, owner_proc) + vads = pe_symbols.PESymbols.get_vads_for_process_cache( + vads_cache, owner_proc + ) if not vads: continue @@ -144,4 +146,3 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): ], self._generator(), ) - From b9fa217d7980042aa7264efbdafee6ce4daa930f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 21 Dec 2024 18:48:12 +0000 Subject: [PATCH 764/979] Add a required framework version Added _required_framework_version and set it to the same value (2, 0, 0) as its plugin requirement of svcscan. Also tweaked one comment. --- volatility3/framework/plugins/windows/svclist.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index a5825e1fe..8a64084c5 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -18,6 +18,7 @@ vollog = logging.getLogger(__name__) class SvcList(svcscan.SvcScan): """Lists services contained with the services.exe doubly linked list of services""" + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) def __init__(self, *args, **kwargs): @@ -41,7 +42,7 @@ class SvcList(svcscan.SvcScan): @classmethod def _get_exe_range(cls, proc) -> Optional[Tuple[int, int]]: """ - Returns a tuple of starting address and size of the + Returns a tuple of starting address and size of the VAD containing services.exe """ From ea273fe878fd5724f1801fd709a805bfa92d7ce0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 21 Dec 2024 23:23:37 +0000 Subject: [PATCH 765/979] Volshell: Fix up shuffled imports --- volatility3/cli/volshell/generic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 90d73b4ac..2321408fe 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -11,6 +11,11 @@ import sys from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union from urllib import parse, request +from volatility3.cli import text_renderer, volshell +from volatility3.framework import exceptions, interfaces, objects, plugins, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import intel, physical, resources, scanners + try: import capstone @@ -18,11 +23,6 @@ try: except ImportError: has_capstone = False -from volatility3.cli import text_renderer, volshell -from volatility3.framework import exceptions, interfaces, objects, plugins, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.layers import intel, physical, resources, scanners - class Volshell(interfaces.plugins.PluginInterface): """Shell environment to directly interact with a memory image.""" From 7caf8c572a4629a2a235a974e6dfff112ccabecd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:03:26 +0100 Subject: [PATCH 766/979] PIL import graceful exit --- .../framework/plugins/linux/graphics/fbdev.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 7144e081a..28cae8000 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -4,9 +4,6 @@ import logging import io -# Image manipulation functions are kept in the plugin, -# to prevent a general exit on missing PIL (pillow) dependency. -from PIL import Image from dataclasses import dataclass from typing import Type, List, Dict, Tuple from volatility3.framework import constants, exceptions, interfaces @@ -16,6 +13,15 @@ from volatility3.framework.objects import utility from volatility3.framework.constants import architectures from volatility3.framework.symbols import linux +# Image manipulation functions are kept in the plugin, +# to prevent a general exit on missing PIL (pillow) dependency. +try: + from PIL import Image + + has_pil = True +except ImportError: + has_pil = False + vollog = logging.getLogger(__name__) @@ -101,7 +107,7 @@ class Fbdev(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, kernel_name: str, fb: Framebuffer, - ) -> Image.Image: + ): """Convert raw framebuffer pixels to an image. Args: @@ -238,6 +244,13 @@ You can try using ffmpeg to decode the raw buffer. Example usage: return fb def _generator(self): + + if not has_pil: + vollog.error( + "PIL (pillow) module is required to use this plugin. Please install it manually or through pyproject.toml." + ) + return None + kernel_name = self.config["kernel"] kernel = self.context.modules[kernel_name] From 1fef570022eb0ae13924ed321e5c09a24fe72563 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:15:45 +0100 Subject: [PATCH 767/979] restrict output to PNG, unify file handling --- .../framework/plugins/linux/graphics/fbdev.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 28cae8000..e82827944 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -160,15 +160,13 @@ class Fbdev(interfaces.plugins.PluginInterface): kernel_name: str, open_method: Type[interfaces.plugins.FileHandlerInterface], fb: Framebuffer, - convert_to_image: bool, - image_format: str = "PNG", + convert_to_png_image: bool, ) -> str: - """Dump a Framebuffer raw buffer to disk. + """Dump a Framebuffer buffer to disk. Args: fb: the relevant Framebuffer object convert_to_image: a boolean specifying if the buffer should be converted to an image - image_format: the target PIL image format (defaults to PNG) Returns: The filename of the dumped buffer. @@ -176,19 +174,19 @@ class Fbdev(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_name] kernel_layer = context.layers[kernel.layer_name] base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" - if convert_to_image: - image = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) - output = io.BytesIO() - image.save(output, image_format) - file_handle = open_method(f"{base_filename}.{image_format.lower()}") - file_handle.write(output.getvalue()) + if convert_to_png_image: + image_object = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) + raw_io_output = io.BytesIO() + image_object.save(raw_io_output, "PNG") + final_fb_buffer = raw_io_output.getvalue() + filename = f"{base_filename}.png" else: - raw_pixels = kernel_layer.read(fb.fb_info.screen_base, fb.size) - file_handle = open_method(f"{base_filename}.raw") - file_handle.write(raw_pixels) + final_fb_buffer = kernel_layer.read(fb.fb_info.screen_base, fb.size) + filename = f"{base_filename}.raw" - file_handle.close() - return file_handle.preferred_filename + with open_method(filename) as f: + f.write(final_fb_buffer) + return f.preferred_filename @classmethod def parse_fb_info( From 8d213284e642c545f44502d0fab3f026bc4fa0ff Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:23:04 +0100 Subject: [PATCH 768/979] handle NotAvailableValue in filename --- volatility3/framework/plugins/linux/graphics/fbdev.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index e82827944..7f7deee4e 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -173,7 +173,8 @@ class Fbdev(interfaces.plugins.PluginInterface): """ kernel = context.modules[kernel_name] kernel_layer = context.layers[kernel.layer_name] - base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" + id = "N-A" if isinstance(fb.id, NotAvailableValue) else fb.id + base_filename = f"{id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" if convert_to_png_image: image_object = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) raw_io_output = io.BytesIO() @@ -207,8 +208,7 @@ class Fbdev(interfaces.plugins.PluginInterface): - struct fb_var_screeninfo stores device independent changeable information about a frame buffer device, its current format and video mode, as well as other miscellaneous parameters. """ - # NotAvailableValue() messes with the filename output on disk - id = utility.array_to_string(fb_info.fix.id) or "N-A" + id = utility.array_to_string(fb_info.fix.id) or NotAvailableValue() color_fields = None # 0 = color, 1 = grayscale, >1 = FOURCC From f3d7647433a727a5bb7bc8c91fa3803ad44a6bf4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 15:48:29 +0100 Subject: [PATCH 769/979] unify Tainting parsing capabilities --- .../framework/symbols/linux/__init__.py | 121 ++++++++++++++++++ .../symbols/linux/extensions/__init__.py | 74 ++--------- 2 files changed, 131 insertions(+), 64 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0230a9c48..832b1de9b 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -11,6 +11,7 @@ from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions +from volatility3.framework.constants import linux as linux_constants class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -830,3 +831,123 @@ class PageCache: page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page + + +class Tainting: + """Tainted kernel and modules parsing capabilities. + + Relevant kernel functions: + - modules: module_flags_taint + - kernel: print_tainted + """ + + def __init__( + self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ): + self.kernel = context.modules[kernel_module_name] + + @property + def kernel_taint_flags_list( + self, + ) -> Optional[List[interfaces.objects.ObjectInterface]]: + if self.kernel.has_symbol("taint_flags"): + return list(self.kernel.object_from_symbol("taint_flags")) + return None + + def _module_flags_taint_pre_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if is_module and is_module != taint_flag.module: + continue + + if taints & taint_flag.shift: + taints_string += char + + return taints_string + + def _module_flags_taint_post_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self.kernel_taint_flags_list): + if is_module and is_module != taint_flag.module: + continue + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taints & (1 << i): + taints_string += c_true + elif c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + """Convert the taints value to a 1-1 character mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + s + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self.kernel_taint_flags_list: + return self._module_flags_taint_post_4_10_rc1(taints, is_module) + return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + + def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + """Convert the taints string to a 1-1 descriptor mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for character in self.get_taints_as_plain_string(taints, is_module): + taint_flag = linux_constants.TAINT_FLAGS.get(character) + if not taint_flag: + comprehensive_taints.append(f"") + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) + + return comprehensive_taints diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0ecf731f4..075a83ae8 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -279,76 +279,29 @@ class module(generic.GenericIntelProcess): return None - def _module_flags_taints_pre_4_10_rc1(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on statically defined taints mappings in the framework. - - Returns: - The raw taints string. - """ - taints_string = "" - for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if taint_flag.module and self.taints & taint_flag.shift: - taints_string += char - - return taints_string - - def _module_flags_taints_post_4_10_rc1(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on kernel symbol embedded taints definitions. - - struct taint_flag { - char c_true; /* character printed when tainted */ - char c_false; /* character printed when not tainted */ - bool module; /* also show as a per-module taint flag */ - }; - - Returns: - The raw taints string. - """ - taints_string = "" - for i, taint_flag in enumerate(self.taint_flags_list): - c_true = chr(taint_flag.c_true) - c_false = chr(taint_flag.c_false) - if taint_flag.module and (self.taints & (1 << i)): - taints_string += c_true - elif taint_flag.module and c_false != " ": - taints_string += c_false - - return taints_string - def get_taints_as_plain_string(self) -> str: """Convert the module's taints value to a 1-1 character mapping. + Convenient wrapper around framework's Tainting capabilities. Returns: The raw taints string. - - Documentation: - - module_flags_taint kernel function """ - - if self.taint_flags_list: - return self._module_flags_taints_post_4_10_rc1() - return self._module_flags_taints_pre_4_10_rc1() + return linux.Tainting( + self._context, + linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, + ).get_taints_as_plain_string(self.taints, True) def get_taints_parsed(self) -> List[str]: """Convert the module's taints string to a 1-1 descriptor mapping. + Convenient wrapper around framework's Tainting capabilities. Returns: A comprehensive (user-friendly) taint descriptor list. - - Documentation: - - module_flags_taint kernel function """ - comprehensive_taints = [] - for character in self.get_taints_as_plain_string(): - taint_flag = linux_constants.TAINT_FLAGS.get(character) - if not taint_flag: - comprehensive_taints.append(f"") - elif taint_flag.when_present: - comprehensive_taints.append(taint_flag.desc) - - return comprehensive_taints + return linux.Tainting( + self._context, + linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, + ).get_taints_parsed(self.taints, True) @property def section_symtab(self): @@ -376,13 +329,6 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("Unable to get strtab") - @property - def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: - kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - if kernel.has_symbol("taint_flags"): - return list(kernel.object_from_symbol("taint_flags")) - return None - class task_struct(generic.GenericIntelProcess): def add_process_layer( From 9d4dd010a7eb6212effc43dd8d57f86e127684f2 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:28:40 +0000 Subject: [PATCH 770/979] Reformat how to write a simple plugin --- doc/source/simple-plugin.rst | 93 ++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 84b921114..aa8ec3a7e 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -41,15 +41,24 @@ to be able to run properly. Any that are defined as optional need not necessari @classmethod def get_requirements(cls): - return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True), - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (2, 0, 0))] + return [ + requirements.ModuleRequirement( + name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"] + ), + requirements.ListRequirement( + name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True + ), + requirements.PluginRequirement( + name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0) + ), + ] This is a classmethod, so it can be called before the specific plugin object has been instantiated (in order to know how @@ -57,8 +66,11 @@ to instantiate the plugin). At the moment these requirements are fairly straigh :: - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"] + ), This requirement specifies the need for a particular submodule. Each module requires a :py:class:`TranslationLayer ` and a @@ -85,9 +97,11 @@ not be requested directly from the user. :: - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.TranslationLayerRequirement( + name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"] + ), This requirement indicates that the plugin will operate on a single :py:class:`TranslationLayer `. The name of the @@ -110,8 +124,10 @@ not be requested directly from the user. :: - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + requirements.SymbolTableRequirement( + name = "nt_symbols", + description = "Windows kernel symbols" + ), This requirement specifies the need for a particular :py:class:`SymbolTable ` @@ -127,10 +143,12 @@ not be requested directly from the user. :: - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True), + requirements.ListRequirement( + name = 'pid', + description = 'Filter on specific process IDs', + element_type = int, + optional = True + ), The next requirement is a List Requirement, populated by integers. The description will be presented to the user to describe what the value represents. The optional flag indicates that the plugin can function without the ``pid`` value @@ -138,9 +156,11 @@ being defined within the configuration tree at all. :: - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (2, 0, 0))] + requirements.PluginRequirement( + name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0) + ) This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements on that plugin. The version is specified in terms of Semantic Versioning meaning that, to be compatible, the major @@ -180,16 +200,21 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) kernel = self.context.modules[self.config['kernel']] - return renderers.TreeGrid([("PID", int), - ("Process", str), - ("Base", format_hints.Hex), - ("Size", format_hints.Hex), - ("Name", str), - ("Path", str)], - self._generator(pslist.PsList.list_processes(self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Base", format_hints.Hex), + ("Size", format_hints.Hex), + ("Name", str), + ("Path", str), + ], + self._generator( + pslist.PsList.list_processes( + self.context, kernel.layer_name, kernel.symbol_table_name, filter_func = filter_func + ) + ) + ) In this instance, the plugin constructs a filter (using the PsList plugin's *classmethod* for creating filters). It checks the plugin's configuration for the ``pid`` value, and passes it in as a list if it finds it, or None if @@ -281,5 +306,3 @@ such as ``!_UNICODE``) and the parameters to that type. Since the cast value must populate a string typed column, it had to be a Python string (such as being cast to the native type string) and could not have been a special Structure such as ``_UNICODE``. For the format hint columns, the format hint type must be used to ensure the error checking does not fail. - - From df37f0a909255410bd5df31cd4d16bdabb1aa9e8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:34:17 +0000 Subject: [PATCH 771/979] Reformat how to write a simple plugin --- doc/source/simple-plugin.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index aa8ec3a7e..07d9e1467 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -211,7 +211,10 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. ], self._generator( pslist.PsList.list_processes( - self.context, kernel.layer_name, kernel.symbol_table_name, filter_func = filter_func + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func = filter_func ) ) ) From 0bb09191aae08b2a1b481fef4fbd565e07a7d91f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 23 Dec 2024 21:28:58 +0100 Subject: [PATCH 772/979] file output failure results in UnreadableValue --- .../framework/plugins/linux/graphics/fbdev.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 7f7deee4e..ab4289cf1 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -8,7 +8,12 @@ from dataclasses import dataclass from typing import Type, List, Dict, Tuple from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.renderers import ( + format_hints, + TreeGrid, + NotAvailableValue, + UnreadableValue, +) from volatility3.framework.objects import utility from volatility3.framework.constants import architectures from volatility3.framework.symbols import linux @@ -280,11 +285,12 @@ You can try using ffmpeg to decode the raw buffer. Example usage: file_output = self.dump_fb( self.context, kernel_name, self.open, fb, bool(fb.color_fields) ) + file_output = str(file_output) except exceptions.InvalidAddressException as excp: vollog.error( f'Layer {excp.layer_name} failed to read address {hex(excp.invalid_address)} when dumping framebuffer "{fb.id}".' ) - file_output = "Error" + file_output = UnreadableValue() try: fb_device_name = utility.pointer_to_string( @@ -303,7 +309,7 @@ You can try using ffmpeg to decode the raw buffer. Example usage: f"{fb.xres_virtual}x{fb.yres_virtual}", fb.bpp, "RUNNING" if fb.fb_info.state == 0 else "SUSPENDED", - str(file_output), + file_output, ), ) From ea2757c06ce23d9a24e01e2ce7823e4980889ee8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 24 Dec 2024 11:45:23 +0100 Subject: [PATCH 773/979] minor version bump --- volatility3/framework/constants/_version.py | 2 +- volatility3/framework/plugins/linux/graphics/fbdev.py | 3 +++ volatility3/framework/symbols/linux/__init__.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 11edc07d8..9ca2d0a5b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 13 # Number of changes that only add to the interface +VERSION_MINOR = 14 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index ab4289cf1..7b644eccf 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -60,6 +60,9 @@ class Fbdev(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=architectures.LINUX_ARCHS, ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 2, 0) + ), requirements.BooleanRequirement( name="dump", description="Dump framebuffers", diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index ba223f979..5aa27b964 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -76,7 +76,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 1, 1) + _version = (2, 2, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) From 79fe1c50dfcda812cd9d4b307b271ddcc3c26bd9 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 24 Dec 2024 19:42:05 +0000 Subject: [PATCH 774/979] Tweak configuration.py Create a tuple directly and replace random.choice by random.choices. --- volatility3/framework/interfaces/configuration.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index 2e4f580a7..a376fa813 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -53,7 +53,7 @@ ConfigSimpleType = Optional[Union[SimpleTypes, List[SimpleTypes]]] def path_join(*args) -> str: """Joins configuration paths together.""" # If a path element (particularly the first) is empty, then remove it from the list - args = tuple([arg for arg in args if arg]) + args = tuple(arg for arg in args if arg) return CONFIG_SEPARATOR.join(args) @@ -772,8 +772,7 @@ class ConfigurableInterface(metaclass=ABCMeta): str: The newly generated full configuration path """ random_config_dict = "".join( - random.SystemRandom().choice(string.ascii_uppercase + string.digits) - for _ in range(8) + random.SystemRandom().choices(string.ascii_uppercase + string.digits, k=8) ) new_config_path = path_join(base_config_path, random_config_dict) # TODO: Check that the new_config_path is empty, although it's not critical if it's not since the values are merged in From 4c430d2ec464b3e1fdf8ddd9b51fdf4525078814 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 26 Dec 2024 07:27:45 +0000 Subject: [PATCH 775/979] Use BasicTypes variable This removes a "float", which should be excluded. --- volatility3/framework/interfaces/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index a376fa813..b6f4f889c 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -779,9 +779,9 @@ class ConfigurableInterface(metaclass=ABCMeta): # This should check that each k corresponds to a requirement and each v is of the appropriate type # This would require knowledge of the new configurable itself to verify, and they should do validation in the - # constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a simple type + # constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a basic type for k, v in kwargs.items(): - if not isinstance(v, (int, str, bool, float, bytes)): + if not isinstance(v, BasicTypes): raise TypeError( "Config values passed to make_subconfig can only be simple types" ) From 834b7d0f072984f232dfc7880c0214b40df424fb Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 28 Dec 2024 14:58:12 +0000 Subject: [PATCH 776/979] Make ETHREAD year check dynamic Change upper bound year check of ETHREAD to be a decade from now. Makes consistent with EPROCESS. --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d63f138b6..5ec84f95f 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -519,7 +519,8 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False - if not (1998 < ctime.year < 2030): + current_year = datetime.datetime.now().year + if not (1998 < ctime.year < current_year + 10): return False except exceptions.InvalidAddressException: From 37873f9e1593fad055b0319085694d67a9568f4b Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 28 Dec 2024 16:58:24 +0000 Subject: [PATCH 777/979] Remove superfluous spaces in intermed.py --- volatility3/framework/symbols/intermed.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 5b4aa22b8..6802af7d6 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -101,7 +101,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): Args: context: The volatility context for the symbol table config_path: The configuration path for the symbol table - name: The name for the symbol table (this is used in symbols e.g. table!symbol ) + name: The name for the symbol table (this is used in symbols e.g. table!symbol) isf_url: The URL pointing to the ISF file location native_types: The NativeSymbolTable that contains the native types for this symbol table table_mapping: A dictionary linking names referenced in the file with symbol tables in the context @@ -111,7 +111,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): """ # Check there are no obvious errors # Open the file and test the version - self._versions = dict([(x.version, x) for x in class_subclasses(ISFormatTable)]) + self._versions = dict((x.version, x) for x in class_subclasses(ISFormatTable)) with resources.ResourceAccessor().open(isf_url) as fp: reader = codecs.getreader("utf-8") json_object = json.load(reader(fp)) # type: ignore @@ -166,9 +166,9 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): format. An interface version such as Major.Minor.Patch means that Major - of the provider must be equal to that of the consumer, and the + of the provider must be equal to that of the consumer, and the provider (the JSON in this instance) must have a greater minor - (indicating that only additive changes have been made) than + (indicating that only additive changes have been made) than the consumer (in this case, the file reader). """ major, minor, patch = (int(x) for x in version.split(".")) From 1bd031b9a86677f6b234e13f93ab2ad9feeb2cc8 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 28 Dec 2024 22:42:25 +0000 Subject: [PATCH 778/979] Prevent infinite loops in device enumeration extensions #1483 --- .../symbols/windows/extensions/__init__.py | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d63f138b6..6dec08c38 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -405,11 +405,24 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the attached device's objects""" - device = self.AttachedDevice.dereference() - while device: - yield device - device = device.AttachedDevice.dereference() + seen = set() + try: + device = self.AttachedDevice.dereference() + except exceptions.InvalidAddressException: + return + + while device: + if device.vol.offset in seen: + break + seen.add(device.vol.offset) + + yield device + + try: + device = device.AttachedDevice.dereference() + except exceptions.InvalidAddressException: + return class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" @@ -421,10 +434,24 @@ class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" - device = self.DeviceObject.dereference() + seen = set() + + try: + device = self.DeviceObject.dereference() + except exceptions.InvalidAddressException: + return + while device: + if device.vol.offset in seen: + return + seen.add(device.vol.offset) + yield device - device = device.NextDevice.dereference() + + try: + device = device.NextDevice.dereference() + except exceptions.InvalidAddressException: + return def is_valid(self) -> bool: """Determine if the object is valid.""" From bf7f1ca91ed88bf482b19bd2558d79aa70bb5c0e Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 28 Dec 2024 22:43:56 +0000 Subject: [PATCH 779/979] Prevent infinite loops in device enumeration extensions #1483 --- volatility3/framework/symbols/windows/extensions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 6dec08c38..c38d47f73 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -424,6 +424,7 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): except exceptions.InvalidAddressException: return + class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" From e64af61efa1a37f6d5c91e34d5375219b1544ee3 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:13:18 +0000 Subject: [PATCH 780/979] Do not analyze processes without VADs #1470 --- volatility3/framework/plugins/windows/direct_system_calls.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index b0c162f46..183e4095c 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -433,6 +433,8 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] vads = self.get_vad_maps(proc) + if not vads: + continue # for each valid process, look for malicious syscall invocations for address, vad_path in self._get_rule_hits( From 33855cf920a8ef76d2bfa414779b7cf96c8c8def Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:20:02 +0000 Subject: [PATCH 781/979] Significantly improve the smear/error handling in the netstat plugin --- .../framework/plugins/windows/netstat.py | 157 +++++++++++++----- .../symbols/windows/extensions/network.py | 8 +- 2 files changed, 120 insertions(+), 45 deletions(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index a1521a8c6..3408c0a3a 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -111,8 +111,21 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): The list of indices at which a 1 was found. """ ret = [] + # This value is broken in many samples and was causing essentially infinite loops + # Testing showed that 8192 is the current size across all Windows versions + # We give some leeway in case it increases in later versions, while still keeping it sane + # The problematic samples had values that looked like addresses, so in the billions + if bitmap_size_in_byte > 8192 * 10: + return ret + for idx in range(bitmap_size_in_byte): - current_byte = context.layers[layer_name].read(bitmap_offset + idx, 1)[0] + try: + current_byte = context.layers[layer_name].read(bitmap_offset + idx, 1)[ + 0 + ] + except exceptions.InvalidAddressException: + continue + current_offs = idx * 8 for bit in range(8): if current_byte & (1 << bit) != 0: @@ -154,32 +167,37 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) else: # invalid argument. - return None + return vollog.debug(f"Current Port: {port}") # the given port serves as a shifted index into the port pool lists list_index = port >> 8 truncated_port = port & 0xFF - # constructing port_pool object here so callers don't have to - port_pool = context.object( - net_symbol_table + constants.BANG + "_INET_PORT_POOL", - layer_name=layer_name, - offset=port_pool_addr, - ) + try: + # constructing port_pool object here so callers don't have to + port_pool = context.object( + net_symbol_table + constants.BANG + "_INET_PORT_POOL", + layer_name=layer_name, + offset=port_pool_addr, + ) + # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) + inpa = port_pool.PortAssignments[list_index] - # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) - inpa = port_pool.PortAssignments[list_index] - - # then parse the port assignment list (`_PORT_ASSIGNMENT_LIST`) and grab the correct entry - assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] + # then parse the port assignment list (`_PORT_ASSIGNMENT_LIST`) and grab the correct entry + assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] + except exceptions.InvalidAddressException: + return if not assignment: - return None + return # the value within assignment.Entry is a) masked and b) points inside of the network object # first decode the pointer - netw_inside = cls._decode_pointer(assignment.Entry) + try: + netw_inside = cls._decode_pointer(assignment.Entry) + except exceptions.InvalidAddressException: + return if netw_inside: # if the value is valid, calculate the actual object address by subtracting the offset @@ -188,16 +206,30 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) yield curr_obj + try: + next_obj_address = cls._decode_pointer(curr_obj.Next) + except exceptions.InvalidAddressException: + return + # if the same port is used on different interfaces multiple objects are created # those can be found by following the pointer within the object's `Next` field until it is empty - while curr_obj.Next: - curr_obj = context.object( - obj_name, - layer_name=layer_name, - offset=cls._decode_pointer(curr_obj.Next) - ptr_offset, - ) + while next_obj_address: + try: + curr_obj = context.object( + obj_name, + layer_name=layer_name, + offset=next_obj_address - ptr_offset, + ) + except exceptions.InvalidAddressException: + return + yield curr_obj + try: + next_obj_address = cls._decode_pointer(curr_obj.Next) + except exceptions.InvalidAddressException: + return + @classmethod def get_tcpip_module( cls, @@ -243,16 +275,25 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): The hash table entries which are _not_ empty """ # we are looking for entries whose values are not their own address + # smear sanity check from mass testing + if ht_length > 4096: + return + for index in range(ht_length): current_addr = ht_offset + index * alignment - current_pointer = context.object( - net_symbol_table + constants.BANG + "pointer", - layer_name=layer_name, - offset=current_addr, - ) + try: + current_pointer = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=current_addr, + ) + except exceptions.InvalidAddressException: + continue + # check if addr of pointer is equal to the value pointed to if current_pointer.vol.offset == current_pointer: continue + yield current_pointer @classmethod @@ -292,11 +333,15 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): tcpip_symbol_table + constants.BANG + "PartitionCount" ).address - part_table_addr = context.object( - net_symbol_table + constants.BANG + "pointer", - layer_name=layer_name, - offset=tcpip_module_offset + part_table_symbol, - ) + try: + part_table_addr = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=tcpip_module_offset + part_table_symbol, + ) + except exceptions.InvalidAddressException: + vollog.debug(f"`PartitionTable` not present in memory.") + return # part_table is the actual partition table offset and consists out of a dynamic amount of _PARTITION objects part_table = context.object( @@ -304,10 +349,18 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): layer_name=layer_name, offset=part_table_addr, ) - part_count = int.from_bytes( - context.layers[layer_name].read(tcpip_module_offset + part_count_symbol, 1), - "little", - ) + + try: + part_count = int.from_bytes( + context.layers[layer_name].read( + tcpip_module_offset + part_count_symbol, 1 + ), + "little", + ) + except exceptions.InvalidAddressException: + vollog.debug(f"`PartitionCount` not present in memory.") + return + part_table.Partitions.count = part_count vollog.debug( @@ -316,9 +369,21 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): entry_offset = context.symbol_space.get_type(obj_name).relative_child_offset( "ListEntry" ) - for ctr, partition in enumerate(part_table.Partitions): + + try: + partitions = part_table.Partitions + except exceptions.InvalidAddressException: + vollog.debug("Partitions member not present in memory") + return + + for ctr, partition in enumerate(partitions): vollog.debug(f"Parsing partition {ctr}") - if partition.Endpoints.NumEntries > 0: + try: + num_entries = partition.Endpoints.NumEntries + except exceptions.InvalidAddressException: + continue + + if num_entries > 0: for endpoint_entry in cls.parse_hashtable( context, layer_name, @@ -402,6 +467,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): upp_symbol = context.symbol_space.get_symbol( tcpip_symbol_table + constants.BANG + "UdpPortPool" ).address + upp_addr = context.object( net_symbol_table + constants.BANG + "pointer", layer_name=layer_name, @@ -498,13 +564,16 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # then, towards the UDP and TCP port pools # first, find their addresses - upp_addr, tpp_addr = cls.find_port_pools( - context, - layer_name, - net_symbol_table, - tcpip_symbol_table, - tcpip_module_offset, - ) + try: + upp_addr, tpp_addr = cls.find_port_pools( + context, + layer_name, + net_symbol_table, + tcpip_symbol_table, + tcpip_module_offset, + ) + except (exceptions.SymbolError, exceptions.InvalidAddressException): + vollog.debug("Unable to reconstruct port pools") # create port pool objects at the detected address and parse the port bitmap upp_obj = context.object( diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 478deab6b..e41ac6a05 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -219,7 +219,13 @@ class _TCP_ENDPOINT(_TCP_LISTENER): return None def is_valid(self): - if self.State not in self.State.choices.values(): + # netstat calls this before validating the object itself + try: + state = self.State + except exceptions.InvalidAddressException: + return False + + if state not in state.choices.values(): vollog.debug( f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid tcp state {self.State}" ) From 61d6a92f8f32e4fe81d951fc35fd7f36e80a2146 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:25:17 +0000 Subject: [PATCH 782/979] Significantly improve the smear/error handling in the netstat plugin --- volatility3/framework/plugins/windows/netstat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 3408c0a3a..902be5fc8 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -340,7 +340,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset=tcpip_module_offset + part_table_symbol, ) except exceptions.InvalidAddressException: - vollog.debug(f"`PartitionTable` not present in memory.") + vollog.debug("`PartitionTable` not present in memory.") return # part_table is the actual partition table offset and consists out of a dynamic amount of _PARTITION objects @@ -358,7 +358,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): "little", ) except exceptions.InvalidAddressException: - vollog.debug(f"`PartitionCount` not present in memory.") + vollog.debug("`PartitionCount` not present in memory.") return part_table.Partitions.count = part_count From 65f602965b14a76dba1596e35a71ab1762257ea7 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:47:48 +0000 Subject: [PATCH 783/979] Address feedback --- volatility3/framework/plugins/windows/suspended_threads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py index 6ddfecca7..cec51ed37 100644 --- a/volatility3/framework/plugins/windows/suspended_threads.py +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -1,7 +1,7 @@ import logging from typing import Dict -from functools import partial +import functools from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements @@ -99,7 +99,7 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, None ) - path_and_symbol = partial( + path_and_symbol = functools.partial( pe_symbols.PESymbols.path_and_symbol_for_address, self.context, self.config_path, From 52a643d5b7f6c57e67acf39eed7c1feb2a0e9dbe Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 29 Dec 2024 20:29:03 +0000 Subject: [PATCH 784/979] Use enumerate for readability --- volatility3/framework/renderers/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 112e93751..093edf8cc 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -83,8 +83,7 @@ class TreeNode(interfaces.renderers.TreeNode): raise TypeError( "Values must be a list of objects made up of simple types and number the same as the columns" ) - for index in range(len(self._treegrid.columns)): - column = self._treegrid.columns[index] + for index, column in enumerate(self._treegrid.columns): val = values[index] if not isinstance(val, (column.type, interfaces.renderers.BaseAbsentValue)): raise TypeError( @@ -413,8 +412,7 @@ class ColumnSortKey(interfaces.renderers.ColumnSortKey): _index = None self._type = None self.ascending = ascending - for i in range(len(treegrid.columns)): - column = treegrid.columns[i] + for i, column in enumerate(treegrid.columns): if column.name.lower() == column_name.lower(): _index = i self._type = column.type From 28ff910d6280edc0dfa21b3b0585a1ab07de9279 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 30 Dec 2024 10:58:38 +0000 Subject: [PATCH 785/979] Use rsplit instead of split Since we want to split rightmost only. --- volatility3/plugins/windows/registry/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/__init__.py b/volatility3/plugins/windows/registry/__init__.py index 8915cdfad..aeeaa87f2 100644 --- a/volatility3/plugins/windows/registry/__init__.py +++ b/volatility3/plugins/windows/registry/__init__.py @@ -15,5 +15,5 @@ import os import sys # This is necessary to ensure the core plugins are available, whilst still be overridable -parent_module, module_name = ".".join(__name__.split(".")[:-1]), __name__.split(".")[-1] +parent_module, module_name = __name__.rsplit(".", maxsplit=1) __path__ = [os.path.join(x, module_name) for x in sys.modules[parent_module].__path__] From e9d9345cef488067e7035aa485ff11ed665c4414 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 09:37:40 -0600 Subject: [PATCH 786/979] Windows Cachedump: Handle uncaught InvalidAddressException --- volatility3/framework/plugins/windows/cachedump.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index 6e667984a..6c730e6ae 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -8,7 +8,7 @@ from typing import Tuple from Crypto.Cipher import ARC4, AES from Crypto.Hash import HMAC -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions @@ -140,9 +140,14 @@ class Cachedump(interfaces.plugins.PluginInterface): if cache_item.Name == "NL$Control": continue - data = sechive.read(cache_item.Data + 4, cache_item.DataLength) - if data is None: + try: + data = sechive.read(cache_item.Data + 4, cache_item.DataLength) + except exceptions.InvalidAddressException: continue + + if not data: + continue + ( uname_len, domain_len, From 2153b742a1dbde57b369adb34fedc5e05e5eb40c Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 30 Dec 2024 18:06:04 +0000 Subject: [PATCH 787/979] Fix uncheck read() call and remove variable that would not be definied if exception triggers --- .../plugins/windows/skeleton_key_check.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index f5d7e1b3a..b103bc831 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -289,7 +289,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException as excp: vollog.debug( - f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" + f"Invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None, None @@ -431,15 +431,20 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): # we do not want to fail just because the count is not in memory # 16 was the size on samples I tested, so I chose it as the default + count = 16 + if target_address: - count = int.from_bytes( - self.context.layers[proc_layer_name].read( - target_address, 4 - ), - "little", - ) - else: - count = 16 + try: + count = int.from_bytes( + self.context.layers[proc_layer_name].read( + target_address, 4 + ), + "little", + ) + except exceptions.InvalidAddressException: + vollog.debug( + "Unable to read `cCsystems`. Defaulting to 16." + ) found_count = True From a3844e8bc54f9d597d459005830e733bd73d7256 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 30 Dec 2024 18:08:30 +0000 Subject: [PATCH 788/979] Fix uncheck read() call and remove variable that would not be definied if exception triggers --- volatility3/framework/plugins/windows/skeleton_key_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index b103bc831..6ae07381a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -282,7 +282,6 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): for proc in proc_list: try: - proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() return proc, proc_layer_name From 5af5363c461eab6ae1661665429a1794a2a712fb Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 13:50:05 -0600 Subject: [PATCH 789/979] Windows Handles: Work in fixes from @attrc These changes fix bugs encountered during regression testing related to virtual offset validation and string length checks. --- volatility3/framework/plugins/windows/handles.py | 8 ++++++++ .../framework/symbols/windows/extensions/pool.py | 11 ++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 62eceb973..e3845b376 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -226,6 +226,14 @@ class Handles(interfaces.plugins.PluginInterface): masked_offset = offset & layer_object.maximum_address for entry in table: + # This triggered a backtrace in many testing samples + # in the level == 0 path + # The code above this calls `is_valid` on the `offset` + # It is sent but then does not validate `entry` before + # sending it to `_get_item` + if not self.context.layers[virtual].is_valid(entry.vol.offset): + continue + if level > 0: yield from self._make_handle_array(entry, level - 1, depth) depth += 1 diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index de5c8271b..ff65acdeb 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -376,7 +376,16 @@ class OBJECT_HEADER(objects.StructType): try: # vista and earlier have a Type member - self._vol["object_header_object_type"] = self.Type.Name.String + length = self.Type.member("Name").Length + if length == 0 or length > 128: + string = None + else: + string = self.Type.Name.String + if len(string) == 0 or len(string) > 128: + string = None + + self._vol["object_header_object_type"] = string + except AttributeError: # windows 7 and later have a TypeIndex, but windows 10 # further encodes the index value with nt1!ObHeaderCookie From 3eeb10be2916bb7988d296c7a85785ffb5a7f25e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 10:10:03 -0600 Subject: [PATCH 790/979] Windows Registry: Handle uncaught exceptions A number of calls to `get_key` across multiple plugins are not made within a `try/except` block that handles `registry.RegistryFormatException` - the calls are either unprotected or only check for `KeyError`. This adds the required `try/except` blocks, or updates the existing ones as needed. --- volatility3/framework/plugins/windows/amcache.py | 10 +++++----- volatility3/framework/plugins/windows/hashdump.py | 2 +- volatility3/framework/plugins/windows/lsadump.py | 7 +++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/amcache.py b/volatility3/framework/plugins/windows/amcache.py index 1e918d61c..46a742233 100644 --- a/volatility3/framework/plugins/windows/amcache.py +++ b/volatility3/framework/plugins/windows/amcache.py @@ -543,7 +543,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\InventoryDriverBinary") # type: ignore ) ) - except KeyError: + except (KeyError, registry.RegistryFormatException): # Registry key not found pass @@ -554,7 +554,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\Programs") ) # type: ignore } - except KeyError: + except (KeyError, registry.RegistryFormatException): programs = {} try: @@ -564,7 +564,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), key=_entry_sort_key, ) - except KeyError: + except (KeyError, registry.RegistryFormatException): files = [] for program_id, file_entries in itertools.groupby( @@ -593,7 +593,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\InventoryApplication") # type: ignore ) ) - except KeyError: + except (KeyError, registry.RegistryFormatException): programs = {} try: @@ -603,7 +603,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), key=_entry_sort_key, ) - except KeyError: + except (KeyError, registry.RegistryFormatException): files = [] for program_id, file_entries in itertools.groupby( diff --git a/volatility3/framework/plugins/windows/hashdump.py b/volatility3/framework/plugins/windows/hashdump.py index 0c98ab8ca..621b0ae53 100644 --- a/volatility3/framework/plugins/windows/hashdump.py +++ b/volatility3/framework/plugins/windows/hashdump.py @@ -332,7 +332,7 @@ class Hashdump(interfaces.plugins.PluginInterface): try: if hive: result = hive.get_key(key) - except KeyError: + except (KeyError, registry.RegistryFormatException): vollog.info( f"Unable to load the required registry key {hive.get_name()}\\{key} from this memory image" ) diff --git a/volatility3/framework/plugins/windows/lsadump.py b/volatility3/framework/plugins/windows/lsadump.py index da8dee325..f3925f2a2 100644 --- a/volatility3/framework/plugins/windows/lsadump.py +++ b/volatility3/framework/plugins/windows/lsadump.py @@ -8,7 +8,7 @@ from typing import Optional from Crypto.Cipher import ARC4, DES, AES from Crypto.Hash import MD5, SHA256 -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions @@ -81,7 +81,10 @@ class Lsadump(interfaces.plugins.PluginInterface): if not enc_reg_value: return None - obf_lsa_key = sechive.read(enc_reg_value.Data + 4, enc_reg_value.DataLength) + try: + obf_lsa_key = sechive.read(enc_reg_value.Data + 4, enc_reg_value.DataLength) + except exceptions.InvalidAddressException: + return None if not obf_lsa_key: return None From f3294ef5f12a6b036989585b88105fde62634ce2 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 12:58:44 -0600 Subject: [PATCH 791/979] Windows Registry: Handle possible exception in get_node Encountered a `SwappedInvalidAddressException` within the call to `cast` due to an underlying call to `read`. --- volatility3/framework/layers/registry.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index cc364ad50..ee7286e1e 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -140,7 +140,14 @@ class RegistryHive(linear.LinearlyMappedLayer): """Returns the appropriate Node, interpreted from the Cell based on its Signature.""" cell = self.get_cell(cell_offset) - signature = cell.cast("string", max_length=2, encoding="latin-1") + try: + signature = cell.cast("string", max_length=2, encoding="latin-1") + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read cell signature for cell at {cell.vol.offset:x}" + ) + return cell + if signature == "nk": return cell.u.KeyNode elif signature == "sk": From 21077f909f6bda2dec6a3900c7e9ec53268b8213 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 30 Dec 2024 22:28:47 +0000 Subject: [PATCH 792/979] Sort imports and swap two assignments --- volatility3/cli/text_filter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 6bd6878a5..b6f019da9 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -1,7 +1,8 @@ import logging -from typing import Any, List, Optional -from volatility3.framework import constants, interfaces import re +from typing import Any, List, Optional + +from volatility3.framework import constants, interfaces vollog = logging.getLogger(__name__) @@ -67,8 +68,8 @@ class ColumnFilter: ) -> None: self.column_num = column_num self.pattern = pattern - self.exclude = exclude self.regex = regex + self.exclude = exclude def find(self, item) -> bool: """Identifies whether an item is found in the appropriate column""" From 3288ac971397500f61501b86a99678592cbd4128 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 14:01:40 -0600 Subject: [PATCH 793/979] Windows Handles: Handle possibly invalid memory accesses Any number of member accesses here can raise an `InvalidAddressException`; each is now checked, and `None` returned if any `InvalidAddressException` occurs. --- .../framework/plugins/windows/handles.py | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index e3845b376..85b16d2d4 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -68,7 +68,12 @@ class Handles(interfaces.plugins.PluginInterface): if not self.context.layers[virtual].is_valid(handle_table_entry.Object): return None fast_ref = handle_table_entry.Object.cast("_EX_FAST_REF") - object_header = fast_ref.dereference().cast("_OBJECT_HEADER") + + try: + object_header = fast_ref.dereference().cast("_OBJECT_HEADER") + except exceptions.InvalidAddressException: + return None + object_header.GrantedAccess = handle_table_entry.GrantedAccess except AttributeError: # starting with windows 8 @@ -77,16 +82,26 @@ class Handles(interfaces.plugins.PluginInterface): ) if is_64bit: - if handle_table_entry.ObjectPointerBits == 0: + try: + pointer_bits = handle_table_entry.ObjectPointerBits + except exceptions.InvalidAddressException: return None - offset = handle_table_entry.ObjectPointerBits << 4 + if pointer_bits == 0: + return None + + offset = pointer_bits << 4 else: - if handle_table_entry.InfoTable == 0: + try: + info_table = handle_table_entry.InfoTable + except exceptions.InvalidAddressException: return None - offset = handle_table_entry.InfoTable & ~7 + if info_table == 0: + return None + + offset = info_table & ~7 # print("LowValue: {0:#x} Magic: {1:#x} Offset: {2:#x}".format(handle_table_entry.InfoTable, magic, offset)) object_header = self.context.object( @@ -94,7 +109,10 @@ class Handles(interfaces.plugins.PluginInterface): virtual, offset=offset, ) - object_header.GrantedAccess = handle_table_entry.GrantedAccessBits + try: + object_header.GrantedAccess = handle_table_entry.GrantedAccessBits + except exceptions.InvalidAddressException: + return None object_header.HandleValue = handle_value return object_header From 9a5365e681e971f0e58b23ed42195df09dced6e3 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 16:59:40 -0600 Subject: [PATCH 794/979] Windows Handles: Fix unbound local in exception handler This fixes an unbound local used in a debug message; If the exception is raised during the dereference operation, the `objct` variable may be uninitialized. This uses the offset of `ptr` instead. --- volatility3/framework/plugins/windows/handles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 85b16d2d4..38ccfbfbc 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -178,7 +178,7 @@ class Handles(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, - f"Cannot access _OBJECT_HEADER Name at {objt.vol.offset:#x}", + f"Cannot access _OBJECT_HEADER Name at {ptr.vol.offset:#x}", ) continue From 263c87611b51f1cb9710c2ec71d1f41eb98e9c77 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 31 Dec 2024 10:43:27 -0600 Subject: [PATCH 795/979] Windows Registry: Handle exceptions in read calls These calls to `.read()` can raise an `InvalidAddressException`. Instead of propagating this exception to the caller, this adds debug logging, and pads the data will null bytes. Also updates the docstring for `decode_data()` to indicate that it can raise `TypeError` and `ValueError`. --- .../symbols/windows/extensions/registry.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 9e2f8df3b..97dd7390d 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -276,7 +276,16 @@ class CM_KEY_VALUE(objects.StructType): return RegValueTypes(self.Type) def decode_data(self) -> Union[int, bytes]: - """Properly decodes the data associated with the value node""" + """ + Properly decodes the data associated with the value node. + + If an InvalidAddressException occurs when reading data from the + underlying RegistryHive layer, the data will be padded with null bytes + of the same length. + + Raises ValueError if the data cannot be read + Raises TypeError if the class was not instantiated on a RegistryHive layer + """ # Determine if the data is stored inline datalen = self.DataLength data = b"" @@ -310,14 +319,26 @@ class CM_KEY_VALUE(objects.StructType): and block_offset < layer.maximum_address ): amount = min(BIG_DATA_MAXLEN, datalen) - data += layer.read( - offset=layer.get_cell(block_offset).vol.offset, length=amount - ) + try: + data += layer.read( + offset=layer.get_cell(block_offset).vol.offset, + length=amount, + ) + except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {amount:x} bytes of data, padding with {amount:x}" + ) datalen -= amount else: # Suspect Data actually points to a Cell, # but the length at the start could be negative so just adding 4 to jump past it - data = layer.read(self.Data + 4, datalen) + try: + data = layer.read(self.Data + 4, datalen) + except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {datalen:x} bytes of data, returning {datalen:x} null bytes" + ) + data = b"\x00" * datalen if self.get_type() == RegValueTypes.REG_DWORD: if len(data) != struct.calcsize(" Date: Tue, 31 Dec 2024 11:11:57 -0600 Subject: [PATCH 796/979] Windows Registry: Update docstrings + exceptions This updates the docstrings on several methods to indicate that they may raise an exception. --- .../symbols/windows/extensions/registry.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 97dd7390d..e53338855 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -159,6 +159,11 @@ class CM_KEY_NODE(objects.StructType): """Extension to allow traversal of registry keys.""" def get_volatile(self) -> bool: + """ + Returns a bool indicating whether or not the key is volatile. + + Raises ValueError if the key was not instantiated on a RegistryHive layer + """ if not isinstance(self._context.layers[self.vol.layer_name], RegistryHive): raise ValueError( "Cannot determine volatility of registry key without an offset in a RegistryHive layer" @@ -166,7 +171,10 @@ class CM_KEY_NODE(objects.StructType): return bool(self.vol.offset & 0x80000000) def get_subkeys(self) -> Iterator["CM_KEY_NODE"]: - """Returns a list of the key nodes.""" + """Returns a list of the key nodes. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") @@ -222,7 +230,10 @@ class CM_KEY_NODE(objects.StructType): yield from self._get_subkeys_recursive(hive, subnode) def get_values(self) -> Iterator["CM_KEY_VALUE"]: - """Returns a list of the Value nodes for a key.""" + """Returns a list of the Value nodes for a key. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") @@ -251,6 +262,11 @@ class CM_KEY_NODE(objects.StructType): return self.Name.cast("string", max_length=namelength, encoding="latin-1") def get_key_path(self) -> str: + """ + Returns the full path to this registry key. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ reg = self._context.layers[self.vol.layer_name] if not isinstance(reg, RegistryHive): raise TypeError("Key was not instantiated on a RegistryHive layer") From fa67f10d3183cd743ca7e44b66d152141a34b2fb Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 31 Dec 2024 11:40:41 -0600 Subject: [PATCH 797/979] Windows Registry: Catch RegistryInvalidIndex refs #1484 This catches uncaught exceptions when casting the cell to a string in `get_node`. --- volatility3/framework/layers/registry.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index cc364ad50..c684ccd40 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -140,7 +140,13 @@ class RegistryHive(linear.LinearlyMappedLayer): """Returns the appropriate Node, interpreted from the Cell based on its Signature.""" cell = self.get_cell(cell_offset) - signature = cell.cast("string", max_length=2, encoding="latin-1") + try: + signature = cell.cast("string", max_length=2, encoding="latin-1") + except (RegistryInvalidIndex, exceptions.InvalidAddressException): + vollog.debug( + f"Failed to get cell signature for cell (0x{cell.vol.offset:x})" + ) + return cell if signature == "nk": return cell.u.KeyNode elif signature == "sk": From c6209800bdc810627f8757881f32e1cf0cfb6f17 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 13:23:02 +0000 Subject: [PATCH 798/979] Core: Fix up issues when resolving merge --- volatility3/framework/constants/_version.py | 2 +- volatility3/framework/interfaces/context.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 02402c5c9..694375538 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 15 # Number of changes that only add to the interface +VERSION_MINOR = 14 # Number of changes that only add to the interface VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 30840a5b9..0b2ae0cc9 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -306,6 +306,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): @abstractmethod def symbols(self) -> List: """Lists the symbols contained in the symbol table for this module""" + raise NotImplementedError("Symbols property has not been implemented.") @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: From 443f7afc9c95c5738cd9eec841ca834860914759 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 13:59:20 +0000 Subject: [PATCH 799/979] Core: Fix code scanning issue concerning equality --- volatility3/framework/contexts/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index a9ec4ac69..e1fb56d94 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -11,6 +11,7 @@ without them interfering with each other. import functools import hashlib import logging +import re from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, interfaces, symbols, exceptions @@ -387,7 +388,6 @@ class ModuleCollection(interfaces.context.ModuleContainer): contents.""" def __init__(self, modules: Optional[List[SizedModule]] = None) -> None: - self._prefix_count = {} self._modules: Dict[str, SizedModule] = {} super().__init__(modules) @@ -408,13 +408,12 @@ class ModuleCollection(interfaces.context.ModuleContainer): def free_module_name(self, prefix: str = "module") -> str: """Returns an unused module name""" - if prefix not in self._prefix_count: - self._prefix_count[prefix] = 1 + existing_names = [name for name in self if re.match(rf"^{prefix}[0-9]*$", name)] + if not existing_names: return prefix - count = self._prefix_count[prefix] + count = len(existing_names) while prefix + str(count) in self: count += 1 - self._prefix_count[prefix] = count return prefix + str(count) @property From 6be039a7e5ddf23f918f073428cbab7e3604f4e7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 14:05:02 +0000 Subject: [PATCH 800/979] Core: Fix black error --- volatility3/framework/interfaces/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 0b2ae0cc9..48c066e96 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -306,7 +306,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): @abstractmethod def symbols(self) -> List: """Lists the symbols contained in the symbol table for this module""" - raise NotImplementedError("Symbols property has not been implemented.") + raise NotImplementedError("Symbols property has not been implemented.") @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: From 9c02f0d12a13fb77db8fb326f6f68dc31fceec1f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 15:22:09 +0000 Subject: [PATCH 801/979] Linux: Fix kmsf f-strings Closes #1496 --- volatility3/framework/plugins/linux/kmsg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d66e3b9ca..c1d09aff8 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return f"{nsec / 1000000000:lu}.{(nsec % 1000000000) / 1000:06lu}" + return f"{nsec / 1000000000}.{(nsec % 1000000000) / 1000:06}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = f"{caller_name}({caller_id & ~0x80000000:u})" + caller = f"{caller_name}({int(caller_id & ~0x80000000)})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: From ac3e76665b7a44b6c5dbc18e633814bd2371ff75 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 15:29:07 +0000 Subject: [PATCH 802/979] Linux: Fix kmsg unguarded read of msg.len --- volatility3/framework/plugins/linux/kmsg.py | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d66e3b9ca..67114c087 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -317,23 +317,27 @@ class Kmsg_3_5_to_3_11(ABCKmsg): while cur_idx < end_idx: msg_offset = log_buf_ptr + cur_idx # type: ignore msg = self.vmlinux.object(object_type=log_struct_name, offset=msg_offset) - if msg.len == 0: - # As per kernel/printk.c: - # A length == 0 for the next message indicates a wrap-around to - # the beginning of the buffer. - cur_idx = 0 - end_idx = log_next_idx - else: - facility, level, timestamp, caller = self.get_prefix(msg) - level_txt = self.get_level_text(level) - facility_txt = self.get_facility_text(facility) + try: + if msg.len == 0: + # As per kernel/printk.c: + # A length == 0 for the next message indicates a wrap-around to + # the beginning of the buffer. + cur_idx = 0 + end_idx = log_next_idx + else: + facility, level, timestamp, caller = self.get_prefix(msg) + level_txt = self.get_level_text(level) + facility_txt = self.get_facility_text(facility) - for line in self.get_log_lines(msg): - yield facility_txt, level_txt, timestamp, caller, line - for line in self.get_dict_lines(msg): - yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_log_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_dict_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line - cur_idx += msg.len + cur_idx += msg.len + except exceptions.InvalidAddressException: + vollog.warning("Kmsg buffer msg length could not be read") + return class Kmsg_3_11_to_5_10(Kmsg_3_5_to_3_11): From c8e67e526a831dcd05b59fa0adeeda8937c4f81a Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 1 Jan 2025 22:33:25 -0600 Subject: [PATCH 803/979] Convert ValueError to TypeError All other methods in this class raise a `TypeError` if the hive was not instantiated on a registry layer; this changes makes this method consistent with the convention used in the others. All `except` blocks checking for `ValueError` have been audited to ensure that this doesn't break exception handling in existing code within the framework. This also includes a minor version bump because: 1. RegistryHives are currently only instantiated one way, which is through the `hivelist` plugin. `hivelist` uses the correct layers when instantiating the hives. 2. Because there is currently a single source for registry hives, and it's unlikely that a hive from that source will ever be created on the wrong layer, it's unlikely that the existing `ValueError` is being raised anywhere within the framework's code. 3. It seems unlikely that consumers of this framework would be instantiating registry hives independent of the `hivelist` plugin, given that they would effectively have to duplicate the `hivelist` code to do so. For these reasons, we're going to do a minor version bump, even though an argument can be made that this warrants a major version bump according to the SemVer rules. This is a one-off and does not indicate any change in the way that we typically update version numbers. --- volatility3/framework/constants/_version.py | 2 +- .../framework/symbols/windows/extensions/registry.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 9ca2d0a5b..2f0c53093 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 14 # Number of changes that only add to the interface +VERSION_MINOR = 15 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index e53338855..c9544a8ba 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -162,12 +162,10 @@ class CM_KEY_NODE(objects.StructType): """ Returns a bool indicating whether or not the key is volatile. - Raises ValueError if the key was not instantiated on a RegistryHive layer + Raises TypeError if the key was not instantiated on a RegistryHive layer """ if not isinstance(self._context.layers[self.vol.layer_name], RegistryHive): - raise ValueError( - "Cannot determine volatility of registry key without an offset in a RegistryHive layer" - ) + raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") return bool(self.vol.offset & 0x80000000) def get_subkeys(self) -> Iterator["CM_KEY_NODE"]: From ac299b0cfc1b12d431c370ce0217d39d0b5cd111 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 2 Jan 2025 17:58:51 +1100 Subject: [PATCH 804/979] linux: module symbols: Fixes and improve code. - Fixed an issue causing the generation of an invalid, extra symbol. - Reuse ELF sym API instead of reimplemented it - Updated the function to return the symbol index, enabling the use of additional module tables. - Ensured the ELF symbol object has `cached_strtab` set, allowing retrieval of critical symbol information like names. - Added typing hints --- .../framework/constants/linux/__init__.py | 3 + .../symbols/linux/extensions/__init__.py | 73 ++++++++++--------- .../framework/symbols/linux/extensions/elf.py | 34 +++++---- 3 files changed, 61 insertions(+), 49 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 6e49e6f37..f3a13f2a5 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -352,3 +352,6 @@ NSEC_PER_SEC = 1e9 MODULE_MAXIMUM_CORE_SIZE = 20000000 MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 MODULE_MINIMUM_SIZE = 4096 + +# Kallsyms +KSYM_NAME_LEN = 512 diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 34d0fcba9..28fbb3fc5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -213,30 +213,37 @@ class module(generic.GenericIntelProcess): ) return elf_table_name - def get_symbols(self): + def get_symbols( + self, + ) -> Iterable[Tuple[int, interfaces.objects.ObjectInterface]]: """Get symbols of the module Yields: - A symbol object + A tuple containing the ELF symbol index and the corresponding ELF symbol object """ - if not hasattr(self, "_elf_table_name"): - self._elf_table_name = self.get_elf_table_name() - if symbols.symbol_table_is_64bit(self._context, self.get_symbol_table_name()): - prefix = "Elf64_" - else: - prefix = "Elf32_" - syms = self._context.object( - self.get_symbol_table_name() + constants.BANG + "array", + if not self.section_strtab or self.num_symtab < 1: + return None + + elf_table_name = self.get_elf_table_name() + symbol_table_name = self.get_symbol_table_name() + + is_64bit = symbols.symbol_table_is_64bit(self._context, symbol_table_name) + sym_name = "Elf64_Sym" if is_64bit else "Elf32_Sym" + sym_type = self._context.symbol_space.get_type( + elf_table_name + constants.BANG + sym_name + ) + elf_syms = self._context.object( + symbol_table_name + constants.BANG + "array", layer_name=self.vol.layer_name, offset=self.section_symtab, - subtype=self._context.symbol_space.get_type( - self._elf_table_name + constants.BANG + prefix + "Sym" - ), - count=self.num_symtab + 1, + subtype=sym_type, + count=self.num_symtab, ) - if self.section_strtab: - yield from syms + for elf_sym_num, elf_sym_obj in enumerate(elf_syms): + # Prepare the symbol object for methods like get_name() + elf_sym_obj.cached_strtab = self.section_strtab + yield elf_sym_num, elf_sym_obj def get_symbols_names_and_addresses(self) -> Iterable[Tuple[str, int]]: """Get names and addresses for each symbol of the module @@ -244,34 +251,25 @@ class module(generic.GenericIntelProcess): Yields: A tuple for each symbol containing the symbol name and its corresponding value """ - - for sym in self.get_symbols(): - sym_arr = self._context.object( - self.get_symbol_table_name() + constants.BANG + "array", - layer_name=self.vol.native_layer_name, - offset=self.section_strtab + sym.st_name, - ) - try: - sym_name = utility.array_to_string( - sym_arr, 512 - ) # 512 is the value of KSYM_NAME_LEN kernel constant - except exceptions.InvalidAddressException: + layer = self._context.layers[self.vol.layer_name] + for _sym_num, sym in self.get_symbols(): + sym_name = sym.get_name() + if not sym_name: continue - if sym_name != "": - # Normalize sym.st_value offset, which is an address pointing to the symbol value - mask = self._context.layers[self.vol.layer_name].address_mask - sym_address = sym.st_value & mask - yield (sym_name, sym_address) - def get_symbol(self, wanted_sym_name): - """Get symbol value for a given symbol name""" + # Normalize sym.st_value offset, which is an address pointing to the symbol value + sym_address = sym.st_value & layer.address_mask + yield (sym_name, sym_address) + + def get_symbol(self, wanted_sym_name) -> Optional[int]: + """Get symbol address for a given symbol name""" for sym_name, sym_address in self.get_symbols_names_and_addresses(): if wanted_sym_name == sym_name: return sym_address return None - def get_symbol_by_address(self, wanted_sym_address): + def get_symbol_by_address(self, wanted_sym_address) -> Optional[str]: """Get symbol name for a given symbol address""" for sym_name, sym_address in self.get_symbols_names_and_addresses(): if wanted_sym_address == sym_address: @@ -285,6 +283,7 @@ class module(generic.GenericIntelProcess): return self.kallsyms.symtab elif self.has_member("symtab"): return self.symtab + raise AttributeError("Unable to get symtab") @property @@ -293,6 +292,7 @@ class module(generic.GenericIntelProcess): return int(self.kallsyms.num_symtab) elif self.has_member("num_symtab"): return int(self.member("num_symtab")) + raise AttributeError("Unable to determine number of symbols") @property @@ -303,6 +303,7 @@ class module(generic.GenericIntelProcess): # Older kernels elif self.has_member("strtab"): return self.strtab + raise AttributeError("Unable to get strtab") diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index eadcbbae0..4de73d952 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -2,13 +2,14 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Dict, Tuple +from typing import Dict, Tuple, Optional import logging from volatility3.framework import constants from volatility3.framework.constants.linux import ( ELF_IDENT, ELF_CLASS, + KSYM_NAME_LEN, ) from volatility3.framework import objects, interfaces, exceptions @@ -328,22 +329,29 @@ class elf_sym(objects.StructType): def cached_strtab(self, cached_strtab): self._cached_strtab = cached_strtab - def get_name(self): + def get_name(self, max_size=KSYM_NAME_LEN) -> Optional[str]: + """Returns the symbol name + + Args: + max_size: Maximum length for a symbol name string. Defaults to KSYM_NAME_LEN (512 bytes). + + Returns: + The symbol name + """ + addr = self._cached_strtab + self.st_name - # Just get the first 255 characters, it should be enough for a symbol name - name_bytes = self._context.layers[self.vol.layer_name].read(addr, 255, pad=True) - - if name_bytes: - idx = name_bytes.find(b"\x00") - if idx != -1: - name_bytes = name_bytes[:idx] - return name_bytes.decode("utf-8", errors="ignore") - else: - # If we cannot read the name from the address space, - # we return None. + layer = self._context.layers[self.vol.layer_name] + name_bytes = layer.read(addr, max_size, pad=True) + if not name_bytes: return None + idx = name_bytes.find(b"\x00") + if idx != -1: + name_bytes = name_bytes[:idx] + + return name_bytes.decode("utf-8", errors="ignore") + class elf_phdr(objects.StructType): """An elf program header""" From 97b93abe438bf32b32067bd962a19e07d9917406 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 2 Jan 2025 11:26:21 +0000 Subject: [PATCH 805/979] Linux: Remove unnecessary int cast --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index c1d09aff8..894ca575f 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = f"{caller_name}({int(caller_id & ~0x80000000)})" + caller = f"{caller_name}({caller_id & ~0x80000000})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: From 7278bb244f58f36b346d254512e393cde6f63871 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 14:47:18 +0100 Subject: [PATCH 806/979] move get_flags_list at bottom --- .../framework/symbols/linux/extensions/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 34d0fcba9..9546fcf82 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2628,6 +2628,19 @@ class page(objects.StructType): page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) return page_data + def get_flags_list(self) -> List[str]: + """Returns a list of page flags + + Returns: + List of page flags + """ + flags = [] + for name, value in self.pageflags_enum.items(): + if self.flags & (1 << value) != 0: + flags.append(name) + + return flags + class IDR(objects.StructType): IDR_BITS = 8 From b61ba66223a866a29a119eb17f44fba250ecc01e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 14:51:23 +0100 Subject: [PATCH 807/979] multi-architecture vmemmap_start calculation --- .../symbols/linux/extensions/__init__.py | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9546fcf82..b02f80433 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -15,7 +15,7 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion from volatility3.framework.constants import linux as linux_constants -from volatility3.framework.layers import linear +from volatility3.framework.layers import linear, intel from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf @@ -2525,16 +2525,13 @@ class address_space(objects.StructType): class page(objects.StructType): - @property - @functools.lru_cache + @functools.cached_property def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values Returns: A dictionary with the pageflags enumeration key/values """ - # FIXME: It would be even better to use @functools.cached_property instead, - # however, this requires Python +3.8 try: pageflags_enum = self._context.symbol_space.get_enumeration( self.get_symbol_table_name() + constants.BANG + "pageflags" @@ -2548,24 +2545,12 @@ class page(objects.StructType): return pageflags_enum - def get_flags_list(self) -> List[str]: - """Returns a list of page flags + @functools.cached_property + def _intel_vmemmap_start(self) -> int: + """Determine the start of the struct page array, for Intel systems. Returns: - List of page flags - """ - flags = [] - for name, value in self.pageflags_enum.items(): - if self.flags & (1 << value) != 0: - flags.append(name) - - return flags - - def to_paddr(self) -> int: - """Converts a page's virtual address to its physical address using the current physical memory model. - - Returns: - int: page physical address + int: vmemmap_start address """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] @@ -2605,13 +2590,39 @@ class page(objects.StructType): "Something went wrong, we shouldn't be here" ) - page_type_size = vmlinux.get_type("page").size + return vmemmap_start + + def _intel_to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current Intel memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] pagec = vmlinux_layer.canonicalize(self.vol.offset) - pfn = (pagec - vmemmap_start) // page_type_size + pfn = (pagec - self._intel_vmemmap_start) // vmlinux.get_type("page").size page_paddr = pfn * vmlinux_layer.page_size return page_paddr + def to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current CPU memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + if isinstance(vmlinux_layer, intel.Intel): + page_paddr = self._intel_to_paddr() + else: + raise exceptions.LayerException( + f"Architecture {type(vmlinux_layer)} vmemmap_start calculation isn't currently supported." + ) + + return page_paddr + def get_content(self) -> Union[str, None]: """Returns the page content @@ -2620,7 +2631,10 @@ class page(objects.StructType): """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] - physical_layer = vmlinux.context.layers["memory_layer"] + physical_layer_name = self._context.layers[self.vol.layer_name].config.get( + "memory_layer", self.vol.layer_name + ) + physical_layer = self._context.layers[physical_layer_name] page_paddr = self.to_paddr() if not page_paddr: return None From dda104bd62b9f5f7b9c0208832c6d788c0ebd2ea Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:37:14 +0100 Subject: [PATCH 808/979] move out Tainting capabilities --- .../framework/symbols/linux/__init__.py | 121 ------------------ 1 file changed, 121 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 832b1de9b..0230a9c48 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -11,7 +11,6 @@ from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions -from volatility3.framework.constants import linux as linux_constants class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -831,123 +830,3 @@ class PageCache: page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page - - -class Tainting: - """Tainted kernel and modules parsing capabilities. - - Relevant kernel functions: - - modules: module_flags_taint - - kernel: print_tainted - """ - - def __init__( - self, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - ): - self.kernel = context.modules[kernel_module_name] - - @property - def kernel_taint_flags_list( - self, - ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self.kernel.has_symbol("taint_flags"): - return list(self.kernel.object_from_symbol("taint_flags")) - return None - - def _module_flags_taint_pre_4_10_rc1( - self, taints: int, is_module: bool = False - ) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on statically defined taints mappings in the framework. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - The raw taints string. - """ - taints_string = "" - for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if is_module and is_module != taint_flag.module: - continue - - if taints & taint_flag.shift: - taints_string += char - - return taints_string - - def _module_flags_taint_post_4_10_rc1( - self, taints: int, is_module: bool = False - ) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on kernel symbol embedded taints definitions. - - struct taint_flag { - char c_true; /* character printed when tainted */ - char c_false; /* character printed when not tainted */ - bool module; /* also show as a per-module taint flag */ - }; - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - The raw taints string. - """ - taints_string = "" - for i, taint_flag in enumerate(self.kernel_taint_flags_list): - if is_module and is_module != taint_flag.module: - continue - c_true = chr(taint_flag.c_true) - c_false = chr(taint_flag.c_false) - if taints & (1 << i): - taints_string += c_true - elif c_false != " ": - taints_string += c_false - - return taints_string - - def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: - """Convert the taints value to a 1-1 character mapping. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - s - Returns: - The raw taints string. - - Documentation: - - module_flags_taint kernel function - """ - - if self.kernel_taint_flags_list: - return self._module_flags_taint_post_4_10_rc1(taints, is_module) - return self._module_flags_taint_pre_4_10_rc1(taints, is_module) - - def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: - """Convert the taints string to a 1-1 descriptor mapping. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - A comprehensive (user-friendly) taint descriptor list. - - Documentation: - - module_flags_taint kernel function - """ - comprehensive_taints = [] - for character in self.get_taints_as_plain_string(taints, is_module): - taint_flag = linux_constants.TAINT_FLAGS.get(character) - if not taint_flag: - comprehensive_taints.append(f"") - elif taint_flag.when_present: - comprehensive_taints.append(taint_flag.desc) - - return comprehensive_taints From 2a5f38ebad48e0d729b3b22caac84bd4209f20a2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:39:07 +0100 Subject: [PATCH 809/979] introduce versioned Linux utilities --- .../framework/symbols/linux/utilities/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 volatility3/framework/symbols/linux/utilities/__init__.py diff --git a/volatility3/framework/symbols/linux/utilities/__init__.py b/volatility3/framework/symbols/linux/utilities/__init__.py new file mode 100644 index 000000000..4225d444b --- /dev/null +++ b/volatility3/framework/symbols/linux/utilities/__init__.py @@ -0,0 +1,11 @@ +from volatility3 import framework +from volatility3.framework import interfaces + + +class LinuxUtilityInterface(interfaces.configuration.VersionableInterface): + """Class with multiple useful Linux functions surrounding a specific piece of functionality.""" + + _version = (2, 1, 1) + _required_framework_version = (2, 0, 0) + + framework.require_interface_version(*_required_framework_version) From 8bc62598f4530bdf2fb99aeb725e5b8f3e0d8cd5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:39:57 +0100 Subject: [PATCH 810/979] initial tainting utilities --- .../symbols/linux/utilities/tainting.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 volatility3/framework/symbols/linux/utilities/tainting.py diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py new file mode 100644 index 000000000..e6d75a963 --- /dev/null +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -0,0 +1,130 @@ +from volatility3 import framework +from volatility3.framework import interfaces +from volatility3.framework.symbols.linux.utilities import LinuxUtilityInterface +from volatility3.framework.constants import linux as linux_constants +from typing import List, Optional + + +class Tainting(LinuxUtilityInterface): + """Tainted kernel and modules parsing capabilities. + + Relevant Linux kernel functions: + - modules: module_flags_taint + - kernel: print_tainted + """ + + _version = (1, 0, 0) + _required_framework_version = (2, 14, 0) + + framework.require_interface_version(*_required_framework_version) + + def __init__( + self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ): + self.kernel = context.modules[kernel_module_name] + + @property + def _kernel_taint_flags_list( + self, + ) -> Optional[List[interfaces.objects.ObjectInterface]]: + if self.kernel.has_symbol("taint_flags"): + return list(self.kernel.object_from_symbol("taint_flags")) + return None + + def _module_flags_taint_pre_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if is_module and is_module != taint_flag.module: + continue + + if taints & taint_flag.shift: + taints_string += char + + return taints_string + + def _module_flags_taint_post_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self._kernel_taint_flags_list): + if is_module and is_module != taint_flag.module: + continue + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taints & (1 << i): + taints_string += c_true + elif c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + """Convert the taints value to a 1-1 character mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + s + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self._kernel_taint_flags_list: + return self._module_flags_taint_post_4_10_rc1(taints, is_module) + return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + + def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + """Convert the taints string to a 1-1 descriptor mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for character in self.get_taints_as_plain_string(taints, is_module): + taint_flag = linux_constants.TAINT_FLAGS.get(character) + if not taint_flag: + comprehensive_taints.append(f"") + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) + + return comprehensive_taints From 3105a31964a1a36420281bd995d983a81b161e97 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:40:49 +0100 Subject: [PATCH 811/979] leverage Tainting from separated Linux utilities --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 075a83ae8..ac07d2def 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -19,7 +19,7 @@ from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf - +from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -286,7 +286,7 @@ class module(generic.GenericIntelProcess): Returns: The raw taints string. """ - return linux.Tainting( + return tainting.Tainting( self._context, linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, ).get_taints_as_plain_string(self.taints, True) @@ -298,7 +298,7 @@ class module(generic.GenericIntelProcess): Returns: A comprehensive (user-friendly) taint descriptor list. """ - return linux.Tainting( + return tainting.Tainting( self._context, linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, ).get_taints_parsed(self.taints, True) From 6e4213e321b96dd8f0b35df6c87aa8426698a42c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:41:37 +0100 Subject: [PATCH 812/979] update tainting requirements to new versioned utilities --- volatility3/framework/plugins/linux/modxview.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index d79f5e7a9..c97864a87 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -9,6 +9,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue from volatility3.framework.symbols.linux import extensions from volatility3.framework.constants import architectures +from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -18,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 11, 0) + _required_framework_version = (2, 14, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -28,6 +29,9 @@ class Modxview(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=architectures.LINUX_ARCHS, ), + requirements.VersionRequirement( + name="linux-tainting", component=tainting.Tainting, version=(1, 0, 0) + ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), From 0a3502697cca4dac2ac2f39896ecfaaef507ac9b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:42:16 +0100 Subject: [PATCH 813/979] 2.13.0 -> 2.14.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 11edc07d8..9ca2d0a5b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 13 # Number of changes that only add to the interface +VERSION_MINOR = 14 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 89b8da8c39fe14f699d711c95a8311ec1e21331e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:48:22 +0100 Subject: [PATCH 814/979] make self.kernel private and call parent __init__ --- .../framework/symbols/linux/utilities/tainting.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index e6d75a963..603215961 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -22,15 +22,18 @@ class Tainting(LinuxUtilityInterface): self, context: interfaces.context.ContextInterface, kernel_module_name: str, + *args, + **kwargs, ): - self.kernel = context.modules[kernel_module_name] + super().__init__(*args, **kwargs) + self._kernel = context.modules[kernel_module_name] @property def _kernel_taint_flags_list( self, ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self.kernel.has_symbol("taint_flags"): - return list(self.kernel.object_from_symbol("taint_flags")) + if self._kernel.has_symbol("taint_flags"): + return list(self._kernel.object_from_symbol("taint_flags")) return None def _module_flags_taint_pre_4_10_rc1( From d2bb5c9f31d7f01fe2e343867c0c7c1926b3ac50 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 3 Jan 2025 10:01:37 +1100 Subject: [PATCH 815/979] linux: fix kmsg fstring bug introduced in #1502 --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 894ca575f..30f67b319 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return f"{nsec / 1000000000}.{(nsec % 1000000000) / 1000:06}" + return f"{nsec // 1000000000}.{(nsec % 1000000000) // 1000:06}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info From 4a34b988d1e4cb02e33e555c3e2ed63d808e5028 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 3 Jan 2025 13:21:09 +0100 Subject: [PATCH 816/979] minor readability adjustments --- .../framework/symbols/linux/utilities/tainting.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 603215961..f7f6c83ec 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -51,7 +51,7 @@ class Tainting(LinuxUtilityInterface): """ taints_string = "" for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if is_module and is_module != taint_flag.module: + if is_module and not taint_flag.module: continue if taints & taint_flag.shift: @@ -79,12 +79,12 @@ class Tainting(LinuxUtilityInterface): The raw taints string. """ taints_string = "" - for i, taint_flag in enumerate(self._kernel_taint_flags_list): - if is_module and is_module != taint_flag.module: + for taint_bit, taint_flag in enumerate(self._kernel_taint_flags_list): + if is_module and not taint_flag.module: continue c_true = chr(taint_flag.c_true) c_false = chr(taint_flag.c_false) - if taints & (1 << i): + if taints & (1 << taint_bit): taints_string += c_true elif c_false != " ": taints_string += c_false @@ -97,7 +97,6 @@ class Tainting(LinuxUtilityInterface): Args: taints: The taints value, represented by an integer is_module: Indicates if the taints value is associated with a built-in/LKM module - s Returns: The raw taints string. From 38c5cc168f93a4d1a5cab2a6c9b071cf32e22fc2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 3 Jan 2025 13:23:27 +0100 Subject: [PATCH 817/979] bump framework req to 2.16.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- volatility3/framework/symbols/linux/utilities/tainting.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index c97864a87..042930740 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 14, 0) + _required_framework_version = (2, 16, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index f7f6c83ec..fc2f94109 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -14,7 +14,7 @@ class Tainting(LinuxUtilityInterface): """ _version = (1, 0, 0) - _required_framework_version = (2, 14, 0) + _required_framework_version = (2, 16, 0) framework.require_interface_version(*_required_framework_version) From 32ca62bbb1205b11be0338e741e3046d503153a8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:20:35 +0000 Subject: [PATCH 818/979] Make f-string slightly more readable --- .../framework/plugins/windows/shimcachemem.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index b8e9b5bd7..9d968c30a 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -305,14 +305,14 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf If a number of validity checks are passed, this method will return the `SHIM_CACHE_HEAD` object. Otherwise, `None` is returned. """ - # print("checking RTL_AVL_TABLE at offset %s" % hex(offset)) + # Check RTL_AVL_TABLE at offset rtl_avl_table = context.object( symbol_table + constants.BANG + "_RTL_AVL_TABLE", layer_name, offset ) if not rtl_avl_table.is_valid(mod_page_start, mod_page_end): return None - vollog.debug(f"Candidate RTL_AVL_TABLE found at offset {hex(offset)}") + vollog.debug(f"Candidate RTL_AVL_TABLE found at offset {offset:#x}") ersrc_size = context.symbol_space.get_type( kernel_symbol_table + constants.BANG + "_ERESOURCE" @@ -324,13 +324,13 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf # 0x20 if context.symbol_space.get_type("pointer").size == 8 else 0x10 ) vollog.debug( - f"ERESOURCE size: {hex(ersrc_size)}, ERESOURCE alignment: {hex(ersrc_alignment)}" + f"ERESOURCE size: {ersrc_size:#x}, ERESOURCE alignment: {ersrc_alignment:#x}" ) eresource_rel_off = ersrc_size + ((offset - ersrc_size) % ersrc_alignment) eresource_offset = offset - eresource_rel_off - vollog.debug(f"Constructing ERESOURCE at {hex(eresource_offset)}") + vollog.debug(f"Constructing ERESOURCE at {eresource_offset:#x}") eresource = context.object( kernel_symbol_table + constants.BANG + "_ERESOURCE", layer_name, @@ -408,8 +408,8 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf # iterate over ahcache kernel module's .data section in search of *two* SHIM handles shim_heads = [] - vollog.debug(f"PAGE offset: {hex(mod_page_offset)}") - vollog.debug(f".data offset: {hex(data_sec_offset)}") + vollog.debug(f"PAGE offset: {mod_page_offset:#x}") + vollog.debug(f".data offset: {data_sec_offset:#x}") handle_type = context.symbol_space.get_type( shimcache_symbol_table + constants.BANG + "SHIM_CACHE_HANDLE" @@ -419,7 +419,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf data_sec_offset + data_sec_size, 8 if symbols.symbol_table_is_64bit(context, nt_symbol_table) else 4, ): - vollog.debug(f"Building shim handle pointer at {hex(offset)}") + vollog.debug(f"Building shim handle pointer at {offset:#x}") shim_handle = context.object( object_type=shimcache_symbol_table + constants.BANG + "pointer", layer_name=kernel_layer_name, @@ -430,7 +430,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf if shim_handle.is_valid(mod_page_offset, mod_page_offset + mod_page_size): if shim_handle.head is not None: vollog.debug( - f"Found valid shim handle @ {hex(shim_handle.vol.offset)}" + f"Found valid shim handle @ {shim_handle.vol.offset:#x}" ) shim_heads.append(shim_handle.head) if len(shim_heads) == 2: @@ -440,7 +440,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf vollog.debug("Failed to identify two valid SHIM_CACHE_HANDLE structures") return - # On Windows 8 x64, the frist cache contains the shim cache + # On Windows 8 x64, the first cache contains the shim cache. # On Windows 8 x86, 8.1 x86/x64, and 10, the second cache contains the shim cache. if ( not symbols.symbol_table_is_64bit(context, nt_symbol_table) From 03049f789559af5c4cdb56f343460178b52220f9 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 3 Jan 2025 18:40:52 +0000 Subject: [PATCH 819/979] Add missing exception handling in env var recovery. Prevent backtraces --- volatility3/framework/plugins/linux/envars.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 8cdbfe493..04b75c8a8 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -5,7 +5,7 @@ import logging from typing import Iterable, Tuple -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -58,10 +58,16 @@ class Envars(plugins.PluginInterface): Tuples of (key, value) representing each environment variable. """ - task_name = utility.array_to_string(task.comm) + # This ensures the `task` is valid as well as its + # memory mapping structures + try: + task_name = utility.array_to_string(task.comm) + env_start = task.mm.env_start + env_end = task.mm.env_end + except exceptions.InvalidAddressException: + return None + task_pid = task.pid - env_start = task.mm.env_start - env_end = task.mm.env_end env_area_size = env_end - env_start if not (0 < env_area_size <= env_area_max_size): vollog.debug( From 8ba60a2aaddf86e4cbd065c95d2553ce221db183 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 4 Jan 2025 16:49:02 +0000 Subject: [PATCH 820/979] Change add_process_layer to return None instead of throwing an exception as it was meant to be designed --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b02f80433..df1c00e3d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -324,9 +324,11 @@ class task_struct(generic.GenericIntelProcess): raise TypeError( "Parent layer is not a translation layer, unable to construct process layer" ) - dtb, layer_name = parent_layer.translate(pgd) - if not dtb: + try: + dtb, layer_name = parent_layer.translate(pgd) + except exceptions.InvalidAddressException: return None + if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.pid}" # Add the constructed layer and return the name From 5f1d318c715311ed12d67bde5a87a8a78e0d3bf0 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 13:39:00 +0000 Subject: [PATCH 821/979] Tiny comment changes --- volatility3/framework/plugins/windows/cmdscan.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 9645ee507..3dc70d649 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -67,6 +67,7 @@ class CmdScan(interfaces.plugins.PluginInterface): Args: conhost_proc: the process object for conhost.exe + size_filter: filter (keep) vads less than this size (bytes) Returns: A list of tuples of: @@ -100,7 +101,7 @@ class CmdScan(interfaces.plugins.PluginInterface): kernel_symbol_table_name: The name of the table containing the kernel symbols config_path: The config path where to find symbol files procs: list of process objects - max_history: an initial set of CommandHistorySize values + max_history: An initial set of CommandHistorySize values Returns: The conhost process object, the command history structure, a dictionary of properties for @@ -227,7 +228,6 @@ class CmdScan(interfaces.plugins.PluginInterface): "data": command_history.CommandCountMax, } ) - command_history_properties.append( { "level": 1, @@ -236,6 +236,7 @@ class CmdScan(interfaces.plugins.PluginInterface): "data": "", } ) + for ( cmd_index, bucket_cmd, @@ -352,7 +353,7 @@ class CmdScan(interfaces.plugins.PluginInterface): def _conhost_proc_filter(self, proc: interfaces.objects.ObjectInterface): """ - Used to filter to only conhost.exe processes + Used to filter only conhost.exe processes """ process_name = utility.array_to_string(proc.ImageFileName) From ab60add9933ee3863c3f2329d2c99af314b5b453 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 17:14:21 +0000 Subject: [PATCH 822/979] Update case insensitive check Update link and use casefold() instead of lower(). --- volatility3/framework/layers/registry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index c684ccd40..6d85da982 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -192,9 +192,9 @@ class RegistryHive(linear.LinearlyMappedLayer): while key_array and node_key: subkeys = node_key[-1].get_subkeys() for subkey in subkeys: - # registry keys are not case sensitive so compare lowercase - # https://msdn.microsoft.com/en-us/library/windows/desktop/ms724946(v=vs.85).aspx - if subkey.get_name().lower() == key_array[0].lower(): + # registry keys are not case sensitive so compare likewise + # https://learn.microsoft.com/en-gb/windows/win32/sysinfo/structure-of-the-registry + if subkey.get_name().casefold() == key_array[0].casefold(): node_key = node_key + [subkey] found_key, key_array = found_key + [key_array[0]], key_array[1:] break From 8f4f576e93a7594666f0e58f8ae73cce5538902c Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 21:09:15 +0000 Subject: [PATCH 823/979] Update case insensitive check Update link and use casefold() instead of lower(). --- volatility3/framework/layers/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 6d85da982..21e1a938e 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -193,7 +193,7 @@ class RegistryHive(linear.LinearlyMappedLayer): subkeys = node_key[-1].get_subkeys() for subkey in subkeys: # registry keys are not case sensitive so compare likewise - # https://learn.microsoft.com/en-gb/windows/win32/sysinfo/structure-of-the-registry + # https://learn.microsoft.com/en-us/windows/win32/sysinfo/structure-of-the-registry if subkey.get_name().casefold() == key_array[0].casefold(): node_key = node_key + [subkey] found_key, key_array = found_key + [key_array[0]], key_array[1:] From 94ec7d89c09b2a276e79fc4c7561828340d5712a Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 21:55:58 +0000 Subject: [PATCH 824/979] Tiny comment changes --- volatility3/framework/plugins/windows/cmdscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 3dc70d649..0cd0addb2 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -67,7 +67,7 @@ class CmdScan(interfaces.plugins.PluginInterface): Args: conhost_proc: the process object for conhost.exe - size_filter: filter (keep) vads less than this size (bytes) + size_filter: size above which vads will not be returned Returns: A list of tuples of: @@ -100,7 +100,7 @@ class CmdScan(interfaces.plugins.PluginInterface): kernel_layer_name: The name of the layer on which to operate kernel_symbol_table_name: The name of the table containing the kernel symbols config_path: The config path where to find symbol files - procs: list of process objects + procs: List of process objects max_history: An initial set of CommandHistorySize values Returns: From a7b4e2fb45bef981eb54c44a5e0cef87b879058f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:02:15 +0100 Subject: [PATCH 825/979] version check_modules --- volatility3/framework/plugins/linux/check_modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/check_modules.py b/volatility3/framework/plugins/linux/check_modules.py index 9b3594c5e..0ed638d9c 100644 --- a/volatility3/framework/plugins/linux/check_modules.py +++ b/volatility3/framework/plugins/linux/check_modules.py @@ -18,6 +18,7 @@ vollog = logging.getLogger(__name__) class Check_modules(plugins.PluginInterface): """Compares module list to sysfs info, if available""" + _version = (1, 0, 0) _required_framework_version = (2, 0, 0) @classmethod From 2d262e7acf5c9aabb32240c01cd57890b7d57647 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:20:56 +0100 Subject: [PATCH 826/979] cut unnecessary intermediate LinuxUtilityInterface --- .../framework/symbols/linux/utilities/__init__.py | 11 ----------- .../framework/symbols/linux/utilities/tainting.py | 5 ++--- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/__init__.py b/volatility3/framework/symbols/linux/utilities/__init__.py index 4225d444b..e69de29bb 100644 --- a/volatility3/framework/symbols/linux/utilities/__init__.py +++ b/volatility3/framework/symbols/linux/utilities/__init__.py @@ -1,11 +0,0 @@ -from volatility3 import framework -from volatility3.framework import interfaces - - -class LinuxUtilityInterface(interfaces.configuration.VersionableInterface): - """Class with multiple useful Linux functions surrounding a specific piece of functionality.""" - - _version = (2, 1, 1) - _required_framework_version = (2, 0, 0) - - framework.require_interface_version(*_required_framework_version) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index fc2f94109..29d7d2b5b 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -1,11 +1,10 @@ from volatility3 import framework from volatility3.framework import interfaces -from volatility3.framework.symbols.linux.utilities import LinuxUtilityInterface from volatility3.framework.constants import linux as linux_constants from typing import List, Optional -class Tainting(LinuxUtilityInterface): +class Tainting(interfaces.configuration.VersionableInterface): """Tainted kernel and modules parsing capabilities. Relevant Linux kernel functions: @@ -14,7 +13,7 @@ class Tainting(LinuxUtilityInterface): """ _version = (1, 0, 0) - _required_framework_version = (2, 16, 0) + _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) From 5c70356c27aedc931c03a8e633952b126ef5254b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:23:16 +0100 Subject: [PATCH 827/979] version check_modules requirement --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 042930740..b44f84c7d 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -38,7 +38,7 @@ class Modxview(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="check_modules", plugin=check_modules.Check_modules, - version=(0, 0, 0), + version=(1, 0, 0), ), requirements.PluginRequirement( name="hidden_modules", From 302f9fdf5ba1c24d07d2fce3d0f7c87c3e6bd1f2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:24:18 +0100 Subject: [PATCH 828/979] cut unnecessary plugin runner functions --- .../framework/plugins/linux/modxview.py | 78 ++++++------------- 1 file changed, 25 insertions(+), 53 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index b44f84c7d..a247bc2cc 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -53,54 +53,6 @@ class Modxview(interfaces.plugins.PluginInterface): ), ] - @classmethod - def run_lsmod( - cls, context: interfaces.context.ContextInterface, kernel_name: str - ) -> List[extensions.module]: - """Wrapper for the lsmod plugin.""" - return list(lsmod.Lsmod.list_modules(context, kernel_name)) - - @classmethod - def run_check_modules( - cls, - context: interfaces.context.ContextInterface, - kernel_name: str, - ) -> List[extensions.module]: - """Wrapper for the check_modules plugin. - Here, we extract the /sys/module/ list.""" - kernel = context.modules[kernel_name] - sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( - context, kernel_name - ) - - # Convert get_kset_modules() offsets back to module objects - return [ - kernel.object(object_type="module", offset=m_offset, absolute=True) - for m_offset in sysfs_modules.values() - ] - - @classmethod - def run_hidden_modules( - cls, - context: interfaces.context.ContextInterface, - kernel_name: str, - known_modules_addresses: Set[int], - ) -> List[extensions.module]: - """Wrapper for the hidden_modules plugin.""" - modules_memory_boundaries = ( - hidden_modules.Hidden_modules.get_modules_memory_boundaries( - context, kernel_name - ) - ) - return list( - hidden_modules.Hidden_modules.get_hidden_modules( - context, - kernel_name, - known_modules_addresses, - modules_memory_boundaries, - ) - ) - @classmethod def flatten_run_modules_results( cls, run_results: Dict[str, List[extensions.module]], deduplicate: bool = True @@ -140,15 +92,35 @@ class Modxview(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_name] run_results = {} - run_results["lsmod"] = cls.run_lsmod(context, kernel_name) - run_results["check_modules"] = cls.run_check_modules(context, kernel_name) + # lsmod + run_results["lsmod"] = list(lsmod.Lsmod.list_modules(context, kernel_name)) + # check_modules + sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( + context, kernel_name + ) + ## Convert get_kset_modules() offsets back to module objects + run_results["check_modules"] = [ + kernel.object(object_type="module", offset=m_offset, absolute=True) + for m_offset in sysfs_modules.values() + ] + # hidden_modules if run_hidden_modules: - known_module_addresses = set( + known_modules_addresses = set( context.layers[kernel.layer_name].canonicalize(module.vol.offset) for module in run_results["lsmod"] + run_results["check_modules"] ) - run_results["hidden_modules"] = cls.run_hidden_modules( - context, kernel_name, known_module_addresses + modules_memory_boundaries = ( + hidden_modules.Hidden_modules.get_modules_memory_boundaries( + context, kernel_name + ) + ) + run_results["hidden_modules"] = list( + hidden_modules.Hidden_modules.get_hidden_modules( + context, + kernel_name, + known_modules_addresses, + modules_memory_boundaries, + ) ) return run_results From 4115c26e7cc119a68aa33fff7f5b8a730b5b2c69 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:24:44 +0100 Subject: [PATCH 829/979] bump framework req to 2.18.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index a247bc2cc..34f5bac8f 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 16, 0) + _required_framework_version = (2, 18, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From bd82f4f33d860cb067e379600da5bbc74f9e2247 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:25:07 +0100 Subject: [PATCH 830/979] 2.16.0 -> 2.18.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 24f96fa89..832b2a5ba 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 16 # Number of changes that only add to the interface +VERSION_MINOR = 18 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From d956742db98610dfa94678ffb98d53c5b6bcd161 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:31:39 +0100 Subject: [PATCH 831/979] remove typing.Set import --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 34f5bac8f..3655200e8 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import List, Dict, Set, Iterator +from typing import List, Dict, Iterator from volatility3.plugins.linux import lsmod, check_modules, hidden_modules from volatility3.framework import interfaces from volatility3.framework.configuration import requirements From b5bc54cfaed91f4d615790305c80ce802658dafe Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:18:34 +0000 Subject: [PATCH 832/979] Use in-place subtraction Also tweak comments. --- volatility3/framework/renderers/conversion.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index e48684b31..f848b2dad 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -18,7 +18,7 @@ def wintime_to_datetime( unix_time = wintime // 10000000 if unix_time == 0: return renderers.NotApplicableValue() - unix_time = unix_time - 11644473600 + unix_time -= 11644473600 try: return datetime.datetime.fromtimestamp(unix_time, datetime.timezone.utc) # Windows sometimes throws OSErrors rather than ValueError/OverflowError when it can't convert a value @@ -71,7 +71,7 @@ def round(addr: int, align: int, up: bool = False) -> int: Args: addr: the address align: the alignment value - up: Whether to round up or not + up: whether to round up or not Returns: The aligned address @@ -122,11 +122,12 @@ def convert_port(port_as_integer): def convert_network_four_tuple(family, four_tuple): - """Converts the connection four_tuple: (source ip, source port, dest ip, - dest port) + """Converts the connection four_tuple: + + (source ip, source port, dest ip, dest port) into their string equivalents. IP addresses are expected as a tuple - of unsigned shorts Ports are converted to proper endianness as well + of unsigned shorts. Ports are converted to proper endianness as well. """ if family == socket.AF_INET: From 43ac6c4d6271c928d9bcdaf6407e01e1c96d7cf9 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 7 Jan 2025 10:24:46 -0600 Subject: [PATCH 833/979] Fix copy-pasted module docstrings This updates the module docstrings for 5 modules that duplicate the docstring from the `proc` module. This was presumably the result of using the `proc` module as a template for the others. --- volatility3/framework/plugins/linux/bash.py | 4 ++-- volatility3/framework/plugins/linux/check_afinfo.py | 4 ++-- volatility3/framework/plugins/linux/check_syscall.py | 3 +-- volatility3/framework/plugins/linux/elfs.py | 4 ++-- volatility3/framework/plugins/linux/lsmod.py | 3 +-- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index 056e3cd51..8acfeb848 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that recovers bash command history +from bash process memory.""" import datetime import struct diff --git a/volatility3/framework/plugins/linux/check_afinfo.py b/volatility3/framework/plugins/linux/check_afinfo.py index 201a443f7..7aa3cbdd2 100644 --- a/volatility3/framework/plugins/linux/check_afinfo.py +++ b/volatility3/framework/plugins/linux/check_afinfo.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that verifies the operation function +pointers of network protocols.""" import logging from typing import List diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 3537a9fa1..13d312f2f 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -1,8 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that checks the system call table for hooks.""" import contextlib import logging from typing import List diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 2fd740941..0d1c9c2dd 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin for enumerating memory-mapped +ELF files across all processes.""" import logging from typing import List, Optional, Type diff --git a/volatility3/framework/plugins/linux/lsmod.py b/volatility3/framework/plugins/linux/lsmod.py index 49e990e93..e9a2a7137 100644 --- a/volatility3/framework/plugins/linux/lsmod.py +++ b/volatility3/framework/plugins/linux/lsmod.py @@ -1,8 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that lists loaded kernel modules.""" import logging from typing import List, Iterable From 32cb6e11f6abe86ce5284e1a618bae9ab1cd4a5f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 7 Jan 2025 19:41:02 +0000 Subject: [PATCH 834/979] Change one letter of a typo --- volatility3/framework/plugins/windows/driverscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/driverscan.py b/volatility3/framework/plugins/windows/driverscan.py index 24d81c3d5..d388ffbb7 100644 --- a/volatility3/framework/plugins/windows/driverscan.py +++ b/volatility3/framework/plugins/windows/driverscan.py @@ -64,7 +64,7 @@ class DriverScan(interfaces.plugins.PluginInterface): names associated with a driver Args: - driver: A Eriver object + driver: A Driver object Returns: A tuple of strings of (driver name, service key, driver alt. name) From 0860441c2fc5a5a97902a7582473d8462c457bc3 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 17:11:02 +0000 Subject: [PATCH 835/979] Core: Improve speed to JSONSchema validation --- pyproject.toml | 3 +-- volatility3/framework/plugins/isfinfo.py | 2 +- volatility3/schemas/__init__.py | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86e3921d2..af22cbe0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,10 +33,9 @@ cloud = [ dev = [ "volatility3[full,cloud]", - "jsonschema>=4.23.0,<5", + "fastjsonschema>=2.21.1,<3", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", - "types-jsonschema>=4.23.0,<5", ] test = [ diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 1c2ac52e9..34b0a5653 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -97,7 +97,7 @@ class IsfInfo(plugins.PluginInterface): if filter_item in isf_file: filtered_list.append(isf_file) - if find_spec("jsonschema") and self.config["validate"]: + if find_spec("fastjsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False" diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 90cfaba48..3964e29a6 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -14,6 +14,8 @@ vollog = logging.getLogger(__name__) cached_validation_filepath = os.path.join(constants.CACHE_PATH, "valid_isf.hashcache") +validators = {} + def load_cached_validations() -> Set[str]: """Loads up the list of successfully cached json objects, so we don't need @@ -92,7 +94,12 @@ def valid( if input_hash in cached_validations and use_cache: return True try: - import jsonschema + import fastjsonschema + + schema_key = json.dumps(schema, sort_keys=True) + if schema_key not in validators: + validator = fastjsonschema.compile(schema) + validators[schema_key] = validator except ImportError: vollog.info("Dependency for validation unavailable: jsonschema") vollog.debug("All validations will report success, even with malformed input") @@ -100,10 +107,13 @@ def valid( try: vollog.debug("Validating JSON against schema...") - jsonschema.validate(input, schema) + validators[schema_key](input) + import pdb + + pdb.set_trace() cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") - except jsonschema.exceptions.SchemaError: + except fastjsonschema.JsonSchemaValueException: vollog.debug("Schema validation error", exc_info=True) return False From e676e6179a3cd9c8d6afb838560d7a9e4d3a5420 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:08:22 +0000 Subject: [PATCH 836/979] Swap fastjsonschema for jsonschema because of date-time validation issues --- volatility3/schemas/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 3964e29a6..b94e0831d 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -94,11 +94,13 @@ def valid( if input_hash in cached_validations and use_cache: return True try: - import fastjsonschema + import jsonschema schema_key = json.dumps(schema, sort_keys=True) if schema_key not in validators: - validator = fastjsonschema.compile(schema) + validator_class = jsonschema.validators.validator_for(schema) + validator_class.check_schema(schema) + validator = validator_class(schema) validators[schema_key] = validator except ImportError: vollog.info("Dependency for validation unavailable: jsonschema") @@ -107,10 +109,7 @@ def valid( try: vollog.debug("Validating JSON against schema...") - validators[schema_key](input) - import pdb - - pdb.set_trace() + validators[schema_key].validate(input) cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") except fastjsonschema.JsonSchemaValueException: From 21decf13708d882369cc1f8fa2884f5a8ae5494d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:12:36 +0000 Subject: [PATCH 837/979] Core: Put the dependencies back for jsonschema --- pyproject.toml | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af22cbe0a..fbdbf0a8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,15 @@ [project] name = "volatility3" description = "Memory forensics framework" -keywords = ["volatility", "memory", "forensics", "framework", "windows", "linux", "volshell"] +keywords = [ + "volatility", + "memory", + "forensics", + "framework", + "windows", + "linux", + "volshell", +] readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, @@ -10,9 +18,7 @@ requires-python = ">=3.8.0" license = { text = "VSL" } dynamic = ["version"] -dependencies = [ - "pefile>=2024.8.26", -] +dependencies = ["pefile>=2024.8.26"] [project.optional-dependencies] full = [ @@ -26,16 +32,14 @@ full = [ "pillow>=10.0.0,<11.0.0", ] -cloud = [ - "gcsfs>=2024.10.0", - "s3fs>=2024.10.0", -] +cloud = ["gcsfs>=2024.10.0", "s3fs>=2024.10.0"] dev = [ "volatility3[full,cloud]", - "fastjsonschema>=2.21.1,<3", + "jsonschema>=4.23.0,<5", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", + "types-jsonschema>=4.23.0,<5", ] test = [ @@ -78,16 +82,16 @@ target-version = "py38" [tool.ruff.lint] select = [ - "F", # pyflakes - "E", # pycodestyle errors - "W", # pycodestyle warnings - "G", # flake8-logging-format - "PIE", # flake8-pie - "UP", # pyupgrade + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "G", # flake8-logging-format + "PIE", # flake8-pie + "UP", # pyupgrade ] ignore = [ - "E501", # ignore due to conflict with formatter + "E501", # ignore due to conflict with formatter ] [build-system] From d07d31047b2ed49a570aa4b3698f6a64145c2d83 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:24:48 +0000 Subject: [PATCH 838/979] Core: Revert the exception catching too --- volatility3/schemas/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index b94e0831d..e894def9f 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -112,7 +112,7 @@ def valid( validators[schema_key].validate(input) cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") - except fastjsonschema.JsonSchemaValueException: + except jsonschema.exceptions.SchemaError: vollog.debug("Schema validation error", exc_info=True) return False From 585901105275a015a3c4326e486f4e2a52d8eb12 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 12:35:04 +0100 Subject: [PATCH 839/979] introduce customizable plugin arparse epilog --- volatility3/cli/__init__.py | 3 +++ volatility3/framework/interfaces/plugins.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 6172a17f3..87caaece6 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -368,6 +368,9 @@ class CommandLine: help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, ) + epilog = getattr(plugin_list[plugin], "_argparse_epilog", None) + if epilog is not None: + plugin_parser.epilog = epilog self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) ### diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index f763815a6..6cd72f02e 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -112,6 +112,8 @@ class PluginInterface( # Be careful with inheritance around this (We default to requiring a version which doesn't exist, so it must be set) _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" + _argparse_epilog: str = None + """Display additional description of the plugin after the description of the arguments. See: https://docs.python.org/3/library/argparse.html#epilog""" def __init__( self, From 530617a700e259f69d53f62f08ccc3382bcdd057 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:52:54 +0000 Subject: [PATCH 840/979] Small readability improvements --- volatility3/framework/automagic/mac.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index f3679d160..a883028d2 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -101,7 +101,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVVV, - f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}", + f"Skipping invalid idlepml4_ptr: {idlepml4_ptr:#x}", ) continue @@ -112,7 +112,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if tmp_dtb % 4096: vollog.log( constants.LOGLEVEL_VVV, - f"Skipping non-page aligned DTB: 0x{tmp_dtb:0x}", + f"Skipping non-page aligned DTB: {tmp_dtb:#x}", ) continue @@ -136,7 +136,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): new_layer.config["kernel_virtual_offset"] = kaslr_shift if new_layer and dtb: - vollog.debug(f"DTB was found at: 0x{dtb:0x}") + vollog.debug(f"DTB was found at: {dtb:#x}") return new_layer vollog.debug("No suitable mac banner could be matched") return None @@ -182,7 +182,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) tmp_aslr_shift = offset - cls.virtual_to_physical_address( version_json_address @@ -208,7 +208,6 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): continue aslr_shift = tmp_aslr_shift & 0xFFFFFFFF - break vollog.log(constants.LOGLEVEL_VVVV, f"Mac find_aslr returned: {aslr_shift:0x}") @@ -219,9 +218,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): """Converts a virtual mac address to a physical one (does not account of ASLR)""" if addr > 0xFFFFFF8000000000: - addr = addr - 0xFFFFFF8000000000 + addr -= 0xFFFFFF8000000000 else: - addr = addr - 0xFF8000000000 + addr -= 0xFF8000000000 return addr From a7661d45e78b10bc736946425055755c9627d111 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 10 Jan 2025 09:21:21 -0600 Subject: [PATCH 841/979] Windows: Certificates - handle uncaught RegistryFormatException Changes variable import to module import, and catches an unhandled `RegistryFormatException` in certificates.py --- .../plugins/windows/registry/certificates.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 8587b3719..a83badb90 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,11 +1,11 @@ import contextlib import logging import struct -from typing import List, Iterator, Optional, Tuple, Type +from typing import Iterator, List, Optional, Tuple, Type from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes +from volatility3.framework.symbols.windows.extensions import registry from volatility3.plugins.windows.registry import hivelist, printkey vollog = logging.getLogger(__name__) @@ -81,7 +81,11 @@ class Certificates(interfaces.plugins.PluginInterface): "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", ]: - with contextlib.suppress(KeyError, exceptions.InvalidAddressException): + with contextlib.suppress( + KeyError, + registry.RegistryFormatException, + exceptions.InvalidAddressException, + ): # Walk it node_path = hive.get_key(top_key, return_list=True) for ( @@ -92,7 +96,11 @@ class Certificates(interfaces.plugins.PluginInterface): _volatility, node, ) in printkey.PrintKey.key_iterator(hive, node_path, recurse=True): - if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": + if ( + not is_key + and registry.RegValueTypes(node.Type) + == registry.RegValueTypes.REG_BINARY + ): name, certificate_data = self.parse_data(node.decode_data()) unique_key_offset = ( key_path.casefold().index(top_key.casefold()) From 96eca6e0162a77699c2befcce6df16f7deac4d23 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 19:38:07 +0100 Subject: [PATCH 842/979] more compact _argparse_epilog --- volatility3/cli/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 87caaece6..37923362a 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -368,9 +368,9 @@ class CommandLine: help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, ) - epilog = getattr(plugin_list[plugin], "_argparse_epilog", None) - if epilog is not None: - plugin_parser.epilog = epilog + plugin_parser.epilog = getattr( + plugin_list[plugin], "_argparse_epilog", None + ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) ### From 615d1d5a2e85dcd2f9d65493690a474c15f691cd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 19:49:25 +0100 Subject: [PATCH 843/979] more compact _argparse_epilog --- volatility3/cli/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 37923362a..fde4fcc6d 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -367,9 +367,7 @@ class CommandLine: plugin, help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, - ) - plugin_parser.epilog = getattr( - plugin_list[plugin], "_argparse_epilog", None + epilog=getattr(plugin_list[plugin], "_argparse_epilog", None), ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) From a26ff8fa6e6ba03a6ea3ebe6c5f3b38b3a4d8851 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 10 Jan 2025 19:03:51 +0000 Subject: [PATCH 844/979] Small readability improvements --- volatility3/framework/automagic/mac.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index a883028d2..3b16eb353 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -184,12 +184,12 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): for offset, banner in offset_generator: banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) - tmp_aslr_shift = offset - cls.virtual_to_physical_address( + aslr_shift = offset - cls.virtual_to_physical_address( version_json_address ) major_string = context.layers[layer_name].read( - version_major_phys_offset + tmp_aslr_shift, 4 + version_major_phys_offset + aslr_shift, 4 ) major = struct.unpack(" Date: Fri, 10 Jan 2025 19:08:56 +0000 Subject: [PATCH 845/979] Small readability improvements --- volatility3/framework/automagic/mac.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 3b16eb353..94c259463 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -184,9 +184,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): for offset, banner in offset_generator: banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) - aslr_shift = offset - cls.virtual_to_physical_address( - version_json_address - ) + aslr_shift = offset - cls.virtual_to_physical_address(version_json_address) major_string = context.layers[layer_name].read( version_major_phys_offset + aslr_shift, 4 From 1cf0232d25fa6dffa21ae3c281e1f568bbb280ab Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 21:19:31 +0100 Subject: [PATCH 846/979] less specific argparse epilog reference --- volatility3/cli/__init__.py | 2 +- volatility3/framework/interfaces/plugins.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index fde4fcc6d..82a2a4205 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -367,7 +367,7 @@ class CommandLine: plugin, help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, - epilog=getattr(plugin_list[plugin], "_argparse_epilog", None), + epilog=plugin_list[plugin].additional_description, ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 6cd72f02e..7ad78d0ba 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -112,7 +112,7 @@ class PluginInterface( # Be careful with inheritance around this (We default to requiring a version which doesn't exist, so it must be set) _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" - _argparse_epilog: str = None + additional_description: str = None """Display additional description of the plugin after the description of the arguments. See: https://docs.python.org/3/library/argparse.html#epilog""" def __init__( From 7913fb2bb0aac4cc390ce6e42ad6621115f0ae7c Mon Sep 17 00:00:00 2001 From: ikelos Date: Fri, 10 Jan 2025 21:07:08 +0000 Subject: [PATCH 847/979] Revert "Small readability improvements" --- volatility3/framework/automagic/mac.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 94c259463..f3679d160 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -101,7 +101,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVVV, - f"Skipping invalid idlepml4_ptr: {idlepml4_ptr:#x}", + f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}", ) continue @@ -112,7 +112,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if tmp_dtb % 4096: vollog.log( constants.LOGLEVEL_VVV, - f"Skipping non-page aligned DTB: {tmp_dtb:#x}", + f"Skipping non-page aligned DTB: 0x{tmp_dtb:0x}", ) continue @@ -136,7 +136,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): new_layer.config["kernel_virtual_offset"] = kaslr_shift if new_layer and dtb: - vollog.debug(f"DTB was found at: {dtb:#x}") + vollog.debug(f"DTB was found at: 0x{dtb:0x}") return new_layer vollog.debug("No suitable mac banner could be matched") return None @@ -182,12 +182,14 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) - aslr_shift = offset - cls.virtual_to_physical_address(version_json_address) + tmp_aslr_shift = offset - cls.virtual_to_physical_address( + version_json_address + ) major_string = context.layers[layer_name].read( - version_major_phys_offset + aslr_shift, 4 + version_major_phys_offset + tmp_aslr_shift, 4 ) major = struct.unpack(" 0xFFFFFF8000000000: - addr -= 0xFFFFFF8000000000 + addr = addr - 0xFFFFFF8000000000 else: - addr -= 0xFF8000000000 + addr = addr - 0xFF8000000000 return addr From 884237534142ec10ba6e7386eedc06ef30d277d0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 23:28:02 +0100 Subject: [PATCH 848/979] 2.15.0 -> 2.16.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 2f0c53093..24f96fa89 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 15 # Number of changes that only add to the interface +VERSION_MINOR = 16 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 0e4e7518447837b9c7f0f30203155b3a3fee0c3a Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 11 Jan 2025 14:05:36 +0100 Subject: [PATCH 849/979] stateless classmethods --- .../symbols/linux/utilities/tainting.py | 67 +++++++++++++------ 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 29d7d2b5b..552f51b98 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -17,26 +17,22 @@ class Tainting(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) - def __init__( - self, + @classmethod + def _get_kernel_taint_flags_list( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self._kernel = context.modules[kernel_module_name] - - @property - def _kernel_taint_flags_list( - self, ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self._kernel.has_symbol("taint_flags"): - return list(self._kernel.object_from_symbol("taint_flags")) + kernel = context.modules[kernel_module_name] + if kernel.has_symbol("taint_flags"): + return list(kernel.object_from_symbol("taint_flags")) return None + @classmethod def _module_flags_taint_pre_4_10_rc1( - self, taints: int, is_module: bool = False + cls, + taints: int, + is_module: bool = False, ) -> str: """Convert the module's taints value to a 1-1 character mapping. Relies on statically defined taints mappings in the framework. @@ -58,8 +54,13 @@ class Tainting(interfaces.configuration.VersionableInterface): return taints_string + @classmethod def _module_flags_taint_post_4_10_rc1( - self, taints: int, is_module: bool = False + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, ) -> str: """Convert the module's taints value to a 1-1 character mapping. Relies on kernel symbol embedded taints definitions. @@ -78,7 +79,9 @@ class Tainting(interfaces.configuration.VersionableInterface): The raw taints string. """ taints_string = "" - for taint_bit, taint_flag in enumerate(self._kernel_taint_flags_list): + for taint_bit, taint_flag in enumerate( + cls._get_kernel_taint_flags_list(context, kernel_module_name) + ): if is_module and not taint_flag.module: continue c_true = chr(taint_flag.c_true) @@ -90,7 +93,14 @@ class Tainting(interfaces.configuration.VersionableInterface): return taints_string - def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + @classmethod + def get_taints_as_plain_string( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, + ) -> str: """Convert the taints value to a 1-1 character mapping. Args: @@ -103,11 +113,22 @@ class Tainting(interfaces.configuration.VersionableInterface): - module_flags_taint kernel function """ - if self._kernel_taint_flags_list: - return self._module_flags_taint_post_4_10_rc1(taints, is_module) - return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + if cls._get_kernel_taint_flags_list(context, kernel_module_name): + return cls._module_flags_taint_post_4_10_rc1( + context, kernel_module_name, taints, is_module + ) + return cls._module_flags_taint_pre_4_10_rc1( + context, kernel_module_name, taints, is_module + ) - def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + @classmethod + def get_taints_parsed( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, + ) -> List[str]: """Convert the taints string to a 1-1 descriptor mapping. Args: @@ -121,7 +142,9 @@ class Tainting(interfaces.configuration.VersionableInterface): - module_flags_taint kernel function """ comprehensive_taints = [] - for character in self.get_taints_as_plain_string(taints, is_module): + for character in cls.get_taints_as_plain_string( + context, kernel_module_name, taints, is_module + ): taint_flag = linux_constants.TAINT_FLAGS.get(character) if not taint_flag: comprehensive_taints.append(f"") From 6817d2c765fb5117a8ec6adb92cf343d37a92595 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 16:00:50 +1100 Subject: [PATCH 850/979] linux: ensure process listing functions yield only valid tasks --- volatility3/framework/plugins/linux/pslist.py | 5 ++- .../symbols/linux/extensions/__init__.py | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 37cf000fc..931acf29a 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -34,7 +34,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 13, 0) - _version = (4, 0, 0) + _version = (4, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -250,6 +250,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Note that the init_task itself is not yielded, since "ps" also never shows it. for task in init_task.tasks: + if not task.is_valid(): + continue + if filter_func(task): continue diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..a50b8ae09 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -307,6 +307,36 @@ class module(generic.GenericIntelProcess): class task_struct(generic.GenericIntelProcess): + def is_valid(self) -> bool: + layer = self._context.layers[self.vol.layer_name] + # Make sure the entire task content is readable + if not layer.is_valid(self.vol.offset, self.vol.size): + return False + + if self.pid < 0: + return False + + if not (self.signal and self.signal.is_readable()): + return False + + if not (self.nsproxy and self.nsproxy.is_readable()): + return False + + if not (self.real_parent and self.real_parent.is_readable()): + return False + + if self.active_mm and not self.active_mm.is_readable(): + return False + + if self.mm: + if not self.mm.is_readable(): + return False + + if self.mm != self.active_mm: + return False + + return True + def add_process_layer( self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: @@ -401,6 +431,8 @@ class task_struct(generic.GenericIntelProcess): tasks_iterable = self._get_tasks_iterable() threads_seen = set([self.vol.offset]) for task in tasks_iterable: + if not task.is_valid(): + continue if task.vol.offset not in threads_seen: threads_seen.add(task.vol.offset) yield task From 093b12b7cdf4a1623a5d534309f0673c0311cc6b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 16:52:45 +1100 Subject: [PATCH 851/979] Linux and Windows: Ensure linked list object extensions consistently yield valid entries --- .../symbols/linux/extensions/__init__.py | 42 ++++++++++------- .../symbols/windows/extensions/__init__.py | 47 +++++++++---------- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..2065b3bb4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1209,35 +1209,43 @@ class list_head(objects.StructType, collections.abc.Iterable): Objects of the type specified via the "symbol_type" argument. """ - layer = layer or self.vol.layer_name + layer_name = layer or self.vol.layer_name + + trans_layer = self._context.layers[layer_name] + if not trans_layer.is_valid(self.vol.offset): + return None relative_offset = self._context.symbol_space.get_type( symbol_type ).relative_child_offset(member) - direction = "prev" - if forward: - direction = "next" - try: - link = getattr(self, direction).dereference() - except exceptions.InvalidAddressException: + direction = "next" if forward else "prev" + + link_ptr = getattr(self, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() + if not sentinel: - yield self._context.object( - symbol_type, layer, offset=self.vol.offset - relative_offset - ) + obj_offset = self.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + + yield self._context.object(symbol_type, layer_name, offset=obj_offset) + seen = {self.vol.offset} while link.vol.offset not in seen: - obj = self._context.object( - symbol_type, layer, offset=link.vol.offset - relative_offset - ) - yield obj + obj_offset = link.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + + yield self._context.object(symbol_type, layer_name, offset=obj_offset) seen.add(link.vol.offset) - try: - link = getattr(link, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(link, direction) + if not (link_ptr and link_ptr.is_readable()): break + link = link_ptr.dereference() def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index f12fd3f5b..214002f49 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -962,56 +962,55 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): ) -> Iterator[interfaces.objects.ObjectInterface]: """Returns an iterator of the entries in the list.""" - layer = layer or self.vol.layer_name + layer_name = layer or self.vol.layer_name + native_layer_name = layer_name or self.vol.native_layer_name + + trans_layer = self._context.layers[layer_name] + if not trans_layer.is_valid(self.vol.offset): + return None relative_offset = self._context.symbol_space.get_type( symbol_type ).relative_child_offset(member) - direction = "Blink" - if forward: - direction = "Flink" + direction = "Flink" if forward else "Blink" - trans_layer = self._context.layers[layer] - - try: - is_valid = trans_layer.is_valid(self.vol.offset) - if not is_valid: - return None - - link = getattr(self, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(self, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() if not sentinel: + obj_offset = self.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + yield self._context.object( symbol_type, - layer, - offset=self.vol.offset - relative_offset, - native_layer_name=layer or self.vol.native_layer_name, + layer_name, + offset=obj_offset, + native_layer_name=native_layer_name, ) seen = {self.vol.offset} while link.vol.offset not in seen: obj_offset = link.vol.offset - relative_offset - if not trans_layer.is_valid(obj_offset): return None - obj = self._context.object( + yield self._context.object( symbol_type, - layer, + layer_name, offset=obj_offset, - native_layer_name=layer or self.vol.native_layer_name, + native_layer_name=native_layer_name, ) - yield obj seen.add(link.vol.offset) - try: - link = getattr(link, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(link, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) From 0d9715136cc9cc96637996b2bb027a76b8b5e87a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:13:44 +1100 Subject: [PATCH 852/979] Linux: Ensure VMA enumration functions yield only valid objects consistently --- .../symbols/linux/extensions/__init__.py | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..61562270a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -811,23 +811,30 @@ class mm_struct(objects.StructType): def _get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or - _get_mmap_iter() automatically as required.""" + _get_mmap_iter() automatically as required. + + Yields: + vm_area_struct objects + """ if not self.has_member("mmap"): raise AttributeError( "_get_mmap_iter called on mm_struct where no mmap member exists." ) - if not self.mmap: + vma_pointer = self.mmap + if not (vma_pointer and vma_pointer.is_readable()): return None - yield self.mmap + vma_object = vma_pointer.dereference() + yield vma_object - seen = {self.mmap.vol.offset} - link = self.mmap.vm_next + seen = {vma_pointer} + vma_pointer = vma_pointer.vm_next - while link != 0 and link.vol.offset not in seen: - yield link - seen.add(link.vol.offset) - link = link.vm_next + while vma_pointer and vma_pointer.is_readable() and vma_pointer not in seen: + vma_object = vma_pointer.dereference() + yield vma_object + seen.add(vma_pointer) + vma_pointer = vma_pointer.vm_next # TODO: As of version 3.0.0 this method should be removed def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: @@ -842,7 +849,11 @@ class mm_struct(objects.StructType): def _get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mm_mt member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or - get_mmap_iter() automatically as required.""" + get_mmap_iter() automatically as required. + + Yields: + vm_area_struct objects + """ if not self.has_member("mm_mt"): raise AttributeError( @@ -850,20 +861,27 @@ class mm_struct(objects.StructType): ) symbol_table_name = self.get_symbol_table_name() for vma_pointer in self.mm_mt.get_slot_iter(): - # convert pointer to vm_area_struct and yield - vma = self._context.object( + # Convert pointer to vm_area_struct and yield + vma_object = self._context.object( symbol_table_name + constants.BANG + "vm_area_struct", layer_name=self.vol.native_layer_name, offset=vma_pointer, ) - yield vma + yield vma_object def get_vma_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns an iterator for the VMAs in an mm_struct. Automatically choosing the mmap or mm_mt as required.""" + """Returns an iterator for the VMAs in an mm_struct. + Automatically choosing the mmap or mm_mt as required. + + Yields: + vm_area_struct objects + """ if self.has_member("mmap"): + # kernels < 6.1 yield from self._get_mmap_iter() elif self.has_member("mm_mt"): + # kernels >= 6.1 d4af56c5c7c6781ca6ca8075e2cf5bc119ed33d1 yield from self._get_maple_tree_iter() else: raise AttributeError("Unable to find mmap or mm_mt in mm_struct") From 35fa9321b3bdac0bc3b8097148eeb6872cade138 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:44:26 +1100 Subject: [PATCH 853/979] Linux: file struct: Remove `f_dentry` and `f_vfsmnt`, as they were preprocessor macro shortcuts, not actual members of the type. --- .../framework/symbols/linux/extensions/__init__.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..22c28f6fa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1152,19 +1152,15 @@ class struct_file(objects.StructType): """Returns a pointer to the dentry associated with this file""" if self.has_member("f_path"): return self.f_path.dentry - elif self.has_member("f_dentry"): - return self.f_dentry - else: - raise AttributeError("Unable to find file -> dentry") + + raise AttributeError("Unable to find file -> dentry") def get_vfsmnt(self) -> interfaces.objects.ObjectInterface: """Returns the fs (vfsmount) where this file is mounted""" if self.has_member("f_path"): return self.f_path.mnt - elif self.has_member("f_vfsmnt"): - return self.f_vfsmnt - else: - raise AttributeError("Unable to find file -> vfs mount") + + raise AttributeError("Unable to find file -> vfs mount") def get_inode(self) -> interfaces.objects.ObjectInterface: """Returns an inode associated with this file""" From fba8f05c8075e07ccecbaca6299ef106127623e3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:48:53 +1100 Subject: [PATCH 854/979] linux: Rename variable to clarify pointer type and avoid confusion with the 'dentry' class. --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 22c28f6fa..4d89764f3 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1390,9 +1390,9 @@ class mount(objects.StructType): A dentry pointer """ vfsmnt = self.get_vfsmnt_current() - dentry = vfsmnt.mnt_root + dentry_pointer = vfsmnt.mnt_root - return dentry + return dentry_pointer def get_dentry_parent(self): """Returns the parent root of the mounted tree From 4b10d658509faaf42b21c67ec543b9bd34f57f84 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:51:30 +1100 Subject: [PATCH 855/979] linux: minor docstring improvements --- .../framework/symbols/linux/__init__.py | 7 +-- .../symbols/linux/extensions/__init__.py | 46 ++++++++++--------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 5aa27b964..93ff35a06 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -106,8 +106,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): Args: task (task_struct): A reference task mnt (vfsmount or mount): A mounted filesystem or a mount point. - - kernels < 3.3.8 type is 'vfsmount' - - kernels >= 3.3.8 type is 'mount' + - kernels < 3.3 type is 'vfsmount' + - kernels >= 3.3 type is 'mount' Returns: str: Pathname of the mount point relative to the task's root directory. @@ -129,7 +129,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): rdentry (dentry *): A pointer to the root dentry rmnt (vfsmount *): A pointer to the root vfsmount dentry (dentry *): A pointer to the dentry - vfsmnt (vfsmount *): A pointer to the vfsmount + vfsmnt (vfsmount/vfsmount *): A vfsmount object (kernels >= 3.3) or a + vfsmount pointer (kernels < 3.3) Returns: str: Pathname of the mount point or file diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 4d89764f3..6d341653b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1500,16 +1500,18 @@ class vfsmount(objects.StructType): ) def _is_kernel_prior_to_struct_mount(self) -> bool: - """Helper to distinguish between kernels prior to version 3.3.8 that - lacked the 'mount' structure and later versions that have it. + """Helper to distinguish between kernels prior to version 3.3 which lacked the + 'mount' struct, versus later versions that include it. + See 7d6fec45a5131918b51dcd76da52f2ec86a85be6. - The 'mnt_parent' member was moved from struct 'vfsmount' to struct - 'mount' when the latter was introduced. + # Following that commit, also in kernel version 3.3 (3376f34fff5be9954fd9a9c4fd68f4a0a36d480e), + # the 'mnt_parent' member was relocated from the 'vfsmount' struct to the newly + # introduced 'mount' struct. Alternatively, vmlinux.has_type('mount') can be used here but it is faster. Returns: - bool: 'True' if the kernel + 'True' if the kernel lacks the 'mount' struct, typically indicating kernel < 3.3. """ return self.has_member("mnt_parent") @@ -1517,22 +1519,21 @@ class vfsmount(objects.StructType): def is_equal(self, vfsmount_ptr) -> bool: """Helper to make sure it is comparing two pointers to 'vfsmount'. - Depending on the kernel version, the calling object (self) could be - a 'vfsmount \\*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust - in the framework "auto" dereferencing ability to assure that when we - reach this point 'self' will be a 'vfsmount' already and self.vol.offset - a 'vfsmount \\*' and not a 'vfsmount \\*\\*'. The argument must be a 'vfsmount \\*'. + Depending on the kernel version, see 3376f34fff5be9954fd9a9c4fd68f4a0a36d480e, + the calling object (self) could be a 'vfsmount *' (<3.3) or a 'vfsmount' (>=3.3). + This way we trust in the framework "auto" dereferencing ability to assure that + when we reach this point 'self' will be a 'vfsmount' already and self.vol.offset + a 'vfsmount *' and not a 'vfsmount **'. The argument must be a 'vfsmount *'. Typically, it's called from do_get_path(). Args: - vfsmount_ptr (vfsmount *): A pointer to a 'vfsmount' + vfsmount_ptr: A pointer to a 'vfsmount' Raises: - exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \\*' + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount *' Returns: - bool: 'True' if the given argument points to the the same 'vfsmount' - as 'self'. + 'True' if the given argument points to the same 'vfsmount' as 'self'. """ if isinstance(vfsmount_ptr, objects.Pointer): return self.vol.offset == vfsmount_ptr @@ -1541,13 +1542,14 @@ class vfsmount(objects.StructType): "Unexpected argument type. It has to be a 'vfsmount *'" ) - def _get_real_mnt(self): + def _get_real_mnt(self) -> interfaces.objects.ObjectInterface: """Gets the struct 'mount' containing this 'vfsmount'. - It should be only called from kernels >= 3.3.8 when 'struct mount' was introduced. + It should be only called from kernels >= 3.3 when 'struct mount' was introduced. + See 7d6fec45a5131918b51dcd76da52f2ec86a85be6 Returns: - mount: the struct 'mount' containing this 'vfsmount'. + The 'mount' object containing this 'vfsmount'. """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) return linux.LinuxUtilities.container_of( @@ -1566,8 +1568,8 @@ class vfsmount(objects.StructType): """Gets the parent fs (vfsmount) to where it's mounted on Returns: - For kernels < 3.3.8: A vfsmount pointer - For kernels >= 3.3.8: A vfsmount object + For kernels < 3.3: A vfsmount pointer + For kernels >= 3.3: A vfsmount object """ if self._is_kernel_prior_to_struct_mount(): return self.get_mnt_parent() @@ -1600,8 +1602,8 @@ class vfsmount(objects.StructType): """Gets the mnt_parent member. Returns: - For kernels < 3.3.8: A vfsmount pointer - For kernels >= 3.3.8: A mount pointer + For kernels < 3.3: A vfsmount pointer + For kernels >= 3.3: A mount pointer """ if self._is_kernel_prior_to_struct_mount(): return self.mnt_parent @@ -1672,8 +1674,10 @@ class kobject(objects.StructType): class mnt_namespace(objects.StructType): def get_inode(self): if self.has_member("proc_inum"): + # 98f842e675f96ffac96e6c50315790912b2812be 3.8 <= kernels < 3.19 return self.proc_inum elif self.has_member("ns") and self.ns.has_member("inum"): + # kernels >= 3.19 435d5f4bb2ccba3b791d9ef61d2590e30b8e806e return self.ns.inum else: raise AttributeError("Unable to find mnt_namespace inode") From d4803a3884343f475a90a14de4be7fa947e561c5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:58:21 +1100 Subject: [PATCH 856/979] Linux: Ensure mount API consistently returns valid mountpoints and path names --- .../framework/plugins/linux/mountinfo.py | 6 ++++-- .../framework/symbols/linux/__init__.py | 21 ++++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index b4f80e4f5..5a0d39f31 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -36,7 +36,7 @@ class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 2, 0) - _version = (1, 2, 3) + _version = (1, 3, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -152,9 +152,11 @@ class MountInfo(plugins.PluginInterface): if not ( task and task.fs - and task.fs.root + and task.fs.is_readable() and task.nsproxy + and task.nsproxy.is_readable() and task.nsproxy.mnt_ns + and task.nsproxy.mnt_ns.is_readable() ): # This task doesn't have all the information required. # It should be a kernel < 2.6.30 diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 93ff35a06..03f4e501a 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -76,7 +76,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 2, 0) + _version = (2, 3, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -121,7 +121,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt) @classmethod - def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> Union[None, str]: + def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> str: """Returns a pathname of the mount point or file It mimics the Linux kernel prepend_path function. @@ -136,8 +136,19 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): str: Pathname of the mount point or file """ + if not (rdentry and rdentry.is_readable() and rmnt and rmnt.is_readable()): + return "" + + if isinstance(vfsmnt, objects.Pointer) and not (rmnt and rmnt.is_readable()): + # vfsmnt can be the vfsmount object itself (>=3.3) or a vfsmount * (<3.3) + return "" + path_reversed = [] - while dentry != rdentry or not vfsmnt.is_equal(rmnt): + while ( + dentry + and dentry.is_readable() + and (dentry != rdentry or not vfsmnt.is_equal(rmnt)) + ): if dentry == vfsmnt.get_mnt_root() or dentry.is_root(): # Escaped? if dentry != vfsmnt.get_mnt_root(): @@ -450,6 +461,10 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): type_dec = vmlinux.get_type(type_name) member_offset = type_dec.relative_child_offset(member_name) container_addr = addr - member_offset + layer = vmlinux.context.layers[vmlinux.layer_name] + if not layer.is_valid(container_addr): + return None + return vmlinux.object( object_type=type_name, offset=container_addr, absolute=True ) From 1707e0a89ce88696f8585734587cc0f300b160ad Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 18:01:31 +1100 Subject: [PATCH 857/979] Fix array_to_string helper method: If called with other object than array and a count value, it will end up with an AttributeError exception --- volatility3/framework/objects/utility.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/objects/utility.py b/volatility3/framework/objects/utility.py index b241ed56a..0bc285517 100644 --- a/volatility3/framework/objects/utility.py +++ b/volatility3/framework/objects/utility.py @@ -33,11 +33,12 @@ def array_to_string( ) -> interfaces.objects.ObjectInterface: """Takes a volatility Array of characters and returns a string.""" # TODO: Consider checking the Array's target is a native char - if count is None: - count = array.vol.count if not isinstance(array, objects.Array): raise TypeError("Array_to_string takes an Array of char") + if count is None: + count = array.vol.count + return array.cast("string", max_length=count, errors=errors) @@ -45,8 +46,10 @@ def pointer_to_string(pointer: "objects.Pointer", count: int, errors: str = "rep """Takes a volatility Pointer to characters and returns a string.""" if not isinstance(pointer, objects.Pointer): raise TypeError("pointer_to_string takes a Pointer") + if count < 1: raise ValueError("pointer_to_string requires a positive count") + char = pointer.dereference() return char.cast("string", max_length=count, errors=errors) From 77ad6f0d831a33cfbc037012a8f2bec0c57520ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 18:25:04 +1100 Subject: [PATCH 858/979] linux: remove unused import --- volatility3/framework/symbols/linux/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 03f4e501a..931c461b9 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -4,7 +4,7 @@ import math import contextlib from abc import ABC, abstractmethod -from typing import Iterator, List, Tuple, Optional, Union +from typing import Iterator, List, Tuple, Optional from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects From 8bc04529350c3ce5a927099a72bc4e419a049db5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Jan 2025 11:21:23 +1100 Subject: [PATCH 859/979] linux: Improve compatibility with ancient kernels --- .../symbols/linux/extensions/__init__.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index a50b8ae09..4a1a263dd 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -316,16 +316,26 @@ class task_struct(generic.GenericIntelProcess): if self.pid < 0: return False - if not (self.signal and self.signal.is_readable()): + if self.has_member("signal") and not ( + self.signal and self.signal.is_readable() + ): return False - if not (self.nsproxy and self.nsproxy.is_readable()): + if self.has_member("nsproxy") and not ( + self.nsproxy and self.nsproxy.is_readable() + ): return False - if not (self.real_parent and self.real_parent.is_readable()): + if self.has_member("real_parent") and not ( + self.real_parent and self.real_parent.is_readable() + ): return False - if self.active_mm and not self.active_mm.is_readable(): + if ( + self.has_member("active_mm") + and self.active_mm + and not self.active_mm.is_readable() + ): return False if self.mm: From 7fc2af5b4ecf4b1ced5c71357d164981b05ed309 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Jan 2025 11:39:48 +1100 Subject: [PATCH 860/979] linux: Add an additional quick check before validating pointer readability --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 4a1a263dd..e2c9454f6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -313,7 +313,7 @@ class task_struct(generic.GenericIntelProcess): if not layer.is_valid(self.vol.offset, self.vol.size): return False - if self.pid < 0: + if self.pid < 0 or self.tgid < 0: return False if self.has_member("signal") and not ( From d21fdb4211c800404acd863b71b8302022d090e4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 16:52:45 +1100 Subject: [PATCH 861/979] Linux: pagecache: Harden Page Cache API to consistently yield valid entries --- .../framework/plugins/linux/pagecache.py | 14 ++++++-- .../framework/symbols/linux/__init__.py | 9 +++--- .../symbols/linux/extensions/__init__.py | 32 +++++++++++++------ 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 382268515..430190970 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -104,7 +104,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -253,6 +253,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): if not root_inode.is_valid(): continue + if not (root_inode.i_mapping and root_inode.i_mapping.is_readable()): + # Retrieving data from the page cache requires a valid address space + continue + # Inode already processed? if root_inode_ptr in seen_inodes: continue @@ -284,6 +288,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): if not file_inode.is_valid(): continue + if not (file_inode.i_mapping and file_inode.i_mapping.is_readable()): + # Retrieving data from the page cache requires a valid address space + continue + # Inode already processed? if file_inode_ptr in seen_inodes: continue @@ -316,10 +324,12 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): if self.config["find"]: if inode_in.path == self.config["find"]: inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) break # Only the first match else: inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) def generate_timeline(self): @@ -389,7 +399,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 5aa27b964..3265dfd37 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -838,11 +838,12 @@ class PageCache: Yields: Page objects """ - + layer = self.vmlinux.context.layers[self.vmlinux.layer_name] for page_addr in self._idstorage.get_entries(self._page_cache.i_pages): if not page_addr: continue - page = self.vmlinux.object("page", offset=page_addr, absolute=True) - if page: - yield page + if not layer.is_valid(page_addr): + continue + + yield self.vmlinux.object("page", offset=page_addr, absolute=True) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..187b1e280 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2489,7 +2489,12 @@ class inode(objects.StructType): """ if not self.i_size: return - elif not (self.i_mapping and self.i_mapping.nrpages > 0): + + if not ( + self.i_mapping + and self.i_mapping.is_readable() + and self.i_mapping.nrpages > 0 + ): return page_cache = linux.PageCache( @@ -2497,19 +2502,21 @@ class inode(objects.StructType): kernel_module_name="kernel", page_cache=self.i_mapping.dereference(), ) + yield from page_cache.get_cached_pages() - def get_contents(self): + def get_contents(self) -> Iterable[Tuple[int, bytes]]: """Get the inode cached pages from the page cache Yields: page_index (int): The page index in the Tree. File offset is page_index * PAGE_SIZE. - page_content (str): The page content + page_content (bytes): The page content """ for page_obj in self.get_pages(): page_index = int(page_obj.index) page_content = page_obj.get_content() - yield page_index, page_content + if page_content: + yield page_index, page_content class address_space(objects.StructType): @@ -2625,7 +2632,7 @@ class page(objects.StructType): return page_paddr - def get_content(self) -> Union[str, None]: + def get_content(self) -> Union[bytes, None]: """Returns the page content Returns: @@ -2641,8 +2648,13 @@ class page(objects.StructType): if not page_paddr: return None - page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) - return page_data + if not physical_layer.is_valid(page_paddr, length=vmlinux_layer.page_size): + vollog.debug( + "Unable to read page 0x%x content at 0x%x", self.vol.offset, page_paddr + ) + return None + + return physical_layer.read(page_paddr, vmlinux_layer.page_size) def get_flags_list(self) -> List[str]: """Returns a list of page flags @@ -2755,17 +2767,17 @@ class IDR(objects.StructType): class rb_root(objects.StructType): - def _walk_nodes(self, root_node) -> Iterator[int]: + def _walk_nodes(self, root_node: int) -> Iterator[int]: """Traverses the Red-Black tree from the root node and yields a pointer to each node in this tree. Args: - root_node: A Red-Black tree node from which to start descending + root_node: A Red-Black tree node pointer from which to start descending Yields: A pointer to every node descending from the specified root node """ - if not root_node: + if not (root_node and root_node.is_readable()): return yield root_node From 529b67fc96d7e975bdff83edd7096c4c0cd8db80 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Jan 2025 12:31:25 +1100 Subject: [PATCH 862/979] Linux: pagecache: Fix issue reported in #1527 --- volatility3/framework/plugins/linux/pagecache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 430190970..9aff0e4f9 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -469,6 +469,8 @@ class InodePages(plugins.PluginInterface): inode_size, page_idx, ) + continue + f.seek(current_fp) f.write(page_bytes) From 1a84f96c70060bc09aab3c1b3348d980f8b9bc0e Mon Sep 17 00:00:00 2001 From: Kerry Goodwine Date: Thu, 9 Jan 2025 15:49:24 -0500 Subject: [PATCH 863/979] Actions: Add new workflow for generating windows EXEs with pyinstaller --- .github/workflows/build-pyinstaller.yml | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/build-pyinstaller.yml diff --git a/.github/workflows/build-pyinstaller.yml b/.github/workflows/build-pyinstaller.yml new file mode 100644 index 000000000..bcba95403 --- /dev/null +++ b/.github/workflows/build-pyinstaller.yml @@ -0,0 +1,50 @@ +name: build-pyinstaller +on: + push: + branches: + - stable + - develop + - 'release/**' + pull_request: + branches: + - stable + - 'release/**' + +jobs: + + exe: + runs-on: windows-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pyinstaller + + - name: Pyinstall executable + run: | + pyinstaller --clean -y vol.spec + pyinstaller --clean -y volshell.spec + + - name: Move files + run: | + mv dist/vol.exe vol.exe + mv dist/volshell.exe volshell.exe + + - name: Archive + uses: actions/upload-artifact@v4 + with: + name: volatility3-pyinstaller + path: | + vol.exe + volshell.exe + README.md + LICENSE.txt From 9f08af47b161579bf31f9d45c8b248c23861388a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 12:51:52 +1100 Subject: [PATCH 864/979] Linux: Add support for Intel 32bit with PAE --- volatility3/framework/automagic/linux.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f22cae012..542d26a8d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -71,6 +71,12 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): elif "init_level4_pgt" in table.symbols: layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" + elif ( + "pkmap_count" in table.symbols + and table.get_symbol("pkmap_count").type.count == 512 + ): + layer_class = intel.LinuxIntelPAE + dtb_symbol_name = "swapper_pg_dir" else: layer_class = intel.LinuxIntel dtb_symbol_name = "swapper_pg_dir" From 28c74f8c1b853df3680de14f6fdc22958516a9c2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 13:23:39 +1100 Subject: [PATCH 865/979] Linux: Add support for Intel 32bit with PAE in early kernels, including versions 2.3.27 and 2.3.28. --- volatility3/framework/automagic/linux.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 542d26a8d..cb4f3cc64 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -71,10 +71,9 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): elif "init_level4_pgt" in table.symbols: layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" - elif ( - "pkmap_count" in table.symbols - and table.get_symbol("pkmap_count").type.count == 512 - ): + elif "pkmap_count" in table.symbols and table.get_symbol( + "pkmap_count" + ).type.count in (512, 2048): layer_class = intel.LinuxIntelPAE dtb_symbol_name = "swapper_pg_dir" else: From b27f98fed258b597e043a5c80304d8489da27b32 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 14:37:35 +1100 Subject: [PATCH 866/979] linux: pslist: fix task credentials rendering --- volatility3/framework/plugins/linux/pslist.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 37cf000fc..77b57e000 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -179,6 +179,10 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_output = "VMA start matching task start_code not found" return file_output + @staticmethod + def _format_cred(cred): + return renderers.NotAvailableValue() if cred is None else cred + def _generator( self, pid_filter: Callable[[Any], bool], @@ -212,16 +216,21 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): task_fields = self.get_task_fields(task, decorate_comm) + task_uid = self._format_cred(task_fields.uid) + task_gid = self._format_cred(task_fields.gid) + task_euid = self._format_cred(task_fields.euid) + task_egid = self._format_cred(task_fields.egid) + yield 0, ( format_hints.Hex(task_fields.offset), task_fields.user_pid, task_fields.user_tid, task_fields.user_ppid, task_fields.name, - task_fields.uid or renderers.NotAvailableValue(), - task_fields.gid or renderers.NotAvailableValue(), - task_fields.euid or renderers.NotAvailableValue(), - task_fields.egid or renderers.NotAvailableValue(), + task_uid, + task_gid, + task_euid, + task_egid, task_fields.creation_time or renderers.NotAvailableValue(), file_output, ) From 66084878627f636d0bebabbecc79842ac954d209 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 15:08:57 +1100 Subject: [PATCH 867/979] Linux: pagecache: Fix issue with incosistent inode page caches --- volatility3/framework/plugins/linux/pagecache.py | 6 +++++- volatility3/framework/symbols/linux/__init__.py | 14 ++++++++++++-- .../framework/symbols/linux/extensions/__init__.py | 9 +++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 9aff0e4f9..b2766be8d 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -520,7 +520,11 @@ class InodePages(plugins.PluginInterface): page_mapping_addr = page_obj.mapping page_index = int(page_obj.index) page_file_offset = page_index * vmlinux_layer.page_size - dump_safe = page_file_offset < inode_size + dump_safe = ( + page_file_offset < inode_size + and page_mapping_addr + and page_mapping_addr.is_readable() + ) page_flags_list = page_obj.get_flags_list() page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) fields = ( diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3265dfd37..08f69c326 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -3,6 +3,7 @@ # import math import contextlib +import logging from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union @@ -12,6 +13,8 @@ from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions +vollog = logging.getLogger(__name__) + class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): provides = {"type": "interface"} @@ -612,7 +615,7 @@ class IDStorage(ABC): raise NotImplementedError def nodep_to_node(self, nodep) -> interfaces.objects.ObjectInterface: - """Instanciates a tree node from its pointer + """Instantiates a tree node from its pointer Args: nodep: Pointer to the XArray/RadixTree node @@ -846,4 +849,11 @@ class PageCache: if not layer.is_valid(page_addr): continue - yield self.vmlinux.object("page", offset=page_addr, absolute=True) + page = self.vmlinux.object("page", offset=page_addr, absolute=True) + if not page.is_valid(): + vollog.error( + f"Invalid cached page at {page.vol.offset:#x}, aborting", + ) + break + + yield page diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 187b1e280..6ae022923 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2534,6 +2534,15 @@ class address_space(objects.StructType): class page(objects.StructType): + def is_valid(self) -> bool: + if self.mapping and not self.mapping.is_readable(): + return False + + if self.to_paddr() < 0: + return False + + return True + @functools.cached_property def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values From ce40659d8728f57f9b6e63709af4b7a17588a309 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:19:47 +1100 Subject: [PATCH 868/979] linux: radix_tree: Fix various issues, enhance early inconsistency detection, and improve compatibility with older kernel versions --- volatility3/framework/exceptions.py | 4 ++ .../framework/symbols/linux/__init__.py | 62 ++++++++++++------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index c44fb4f2e..41c67b88d 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -130,3 +130,7 @@ class OfflineException(VolatilityException): class RenderException(VolatilityException): """Thrown if there is an error during rendering""" + + +class LinuxPageCacheException(VolatilityException): + """Thrown if there is an error during Linux Page Cache processing""" diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 08f69c326..4c9934439 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -3,6 +3,7 @@ # import math import contextlib +import functools import logging from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union @@ -662,7 +663,7 @@ class IDStorage(ABC): height = self.get_tree_height(root.vol.offset) nodep = self.get_head_node(root) - if not nodep: + if not (nodep and nodep.is_readable()): return # Keep the internal flag before untagging it @@ -697,7 +698,7 @@ class XArray(IDStorage): def get_node_height(self, nodep) -> int: node = self.nodep_to_node(nodep) - return (node.shift / self.CHUNK_SHIFT) + 1 + return (node.shift // self.CHUNK_SHIFT) + 1 def get_head_node(self, tree) -> int: return tree.xa_head @@ -720,6 +721,7 @@ class RadixTree(IDStorage): RADIX_TREE_INTERNAL_NODE = 1 RADIX_TREE_EXCEPTIONAL_ENTRY = 2 RADIX_TREE_ENTRY_MASK = 3 + RADIX_TREE_MAP_SHIFT = 6 # CONFIG_BASE_FULL # Dynamic values. These will be initialized later RADIX_TREE_INDEX_BITS = None @@ -756,43 +758,57 @@ class RadixTree(IDStorage): def get_tree_height(self, treep) -> int: with contextlib.suppress(exceptions.SymbolError): if self.vmlinux.get_type("radix_tree_root").has_member("height"): - # kernels < 4.7.10 + # kernels < 4.7 d0891265bbc988dc91ed8580b38eb3dac128581b radix_tree_root = self.vmlinux.object( "radix_tree_root", offset=treep, absolute=True ) return radix_tree_root.height - # kernels >= 4.7.10 + # kernels >= 4.7 return 0 + @functools.cached_property + def _max_height_array(self): + if self.vmlinux.has_symbol("height_to_maxindex"): + # 2.6.24 26fb1589cb0aaec3a0b4418c54f30c1a2b1781f6 <= Kernels < 4.7 d0891265bbc988dc91ed8580b38eb3dac128581b + return self.vmlinux.object_from_symbol("height_to_maxindex") + elif self.vmlinux.has_symbol("height_to_maxnodes"): + # 4.8 c78c66d1ddfdbd2353f3fcfeba0268524537b096 <= kernels < 4.20 8cf2f98411e3a0865026a1061af637161b16d32b + return self.vmlinux.object_from_symbol("height_to_maxnodes") + + return None + def _radix_tree_maxindex(self, node, height) -> int: """Return the maximum key which can be store into a radix tree with this height.""" - if not self.vmlinux.has_symbol("height_to_maxindex"): - # Kernels >= 4.7 - return (self.CHUNK_SIZE << node.shift) - 1 + if self._max_height_array: + # 2.6.24 <= kernels <= 4.20 See _max_height_array() + return self._max_height_array[height] else: - # Kernels < 4.7 - height_to_maxindex_array = self.vmlinux.object_from_symbol( - "height_to_maxindex" - ) - maxindex = height_to_maxindex_array[height] - return maxindex + # Kernels >= 4.20 + return (self.CHUNK_SIZE << node.shift) - 1 def get_node_height(self, nodep) -> int: node = self.nodep_to_node(nodep) if hasattr(node, "shift"): # 4.7 <= Kernels < 4.20 - return (node.shift / self.CHUNK_SHIFT) + 1 + height = (node.shift // self.CHUNK_SHIFT) + 1 elif hasattr(node, "path"): # 3.15 <= Kernels < 4.7 - return node.path & self.RADIX_TREE_HEIGHT_MASK + height = node.path & self.RADIX_TREE_HEIGHT_MASK elif hasattr(node, "height"): # Kernels < 3.15 - return node.height + height = node.height else: raise exceptions.VolatilityException("Cannot find radix-tree node height") + if self._max_height_array and not (0 <= height < self._max_height_array.count): + error_msg = f"Radix Tree node {node.vol.offset:#x} height {height} exceeds max height of {self._max_height_array.count}" + vollog.error(error_msg) + raise exceptions.LinuxPageCacheException(error_msg) + + return height + def get_head_node(self, tree) -> int: return tree.rnode @@ -805,14 +821,16 @@ class RadixTree(IDStorage): def untag_node(self, nodep) -> int: return nodep & (~self.RADIX_TREE_ENTRY_MASK) - def is_valid_node(self, nodep) -> bool: + def _is_exceptional_node(self, nodep) -> bool: # In kernels 4.20, exceptional nodes were removed and internal entries took their bitmask - if self.vmlinux.has_type("radix_tree_root"): - return ( - nodep & self.RADIX_TREE_ENTRY_MASK - ) != self.RADIX_TREE_EXCEPTIONAL_ENTRY + return ( + self.vmlinux.has_type("radix_tree_root") + and (nodep & self.RADIX_TREE_ENTRY_MASK) + == self.RADIX_TREE_EXCEPTIONAL_ENTRY + ) - return True + def is_valid_node(self, nodep) -> bool: + return not self._is_exceptional_node(nodep) class PageCache: From 3c92d7f7b7d3d7d9548275e960571d4ac37a6a21 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:22:46 +1100 Subject: [PATCH 869/979] linux: page_cache: enhance early inconsistency detection --- volatility3/framework/symbols/linux/__init__.py | 14 ++++++-------- .../framework/symbols/linux/extensions/__init__.py | 7 ++++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 4c9934439..a7e6ef405 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -861,17 +861,15 @@ class PageCache: """ layer = self.vmlinux.context.layers[self.vmlinux.layer_name] for page_addr in self._idstorage.get_entries(self._page_cache.i_pages): - if not page_addr: - continue - if not layer.is_valid(page_addr): - continue + error_msg = f"Invalid cached page address at {page_addr:#x}, aborting" + vollog.error(error_msg) + raise exceptions.LinuxPageCacheException(error_msg) page = self.vmlinux.object("page", offset=page_addr, absolute=True) if not page.is_valid(): - vollog.error( - f"Invalid cached page at {page.vol.offset:#x}, aborting", - ) - break + error_msg = f"Invalid cached page at {page_addr:#x}, aborting" + vollog.error(error_msg) + raise exceptions.LinuxPageCacheException(error_msg) yield page diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 6ae022923..997370bbb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2513,6 +2513,11 @@ class inode(objects.StructType): page_content (bytes): The page content """ for page_obj in self.get_pages(): + if page_obj.mapping != self.i_mapping: + vollog.warning( + f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" + ) + continue page_index = int(page_obj.index) page_content = page_obj.get_content() if page_content: @@ -2524,7 +2529,7 @@ class address_space(objects.StructType): def i_pages(self): """Returns the appropriate member containing the page cache tree""" if self.has_member("i_pages"): - # Kernel >= 4.17 + # Kernel >= 4.17 b93b016313b3ba8003c3b8bb71f569af91f19fc7 return self.member("i_pages") elif self.has_member("page_tree"): # Kernel < 4.17 From c1497410b4721703ebbb9f30f2ab3db497a4438b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:28:25 +1100 Subject: [PATCH 870/979] linux: page_cache plugin: lazy file initialization and avoid redundant inode page cache walk during dumps not showing output with dumping to file. --- .../framework/plugins/linux/pagecache.py | 89 ++++++++++++------- 1 file changed, 57 insertions(+), 32 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index b2766be8d..7ad0f5a8f 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -8,7 +8,7 @@ import datetime from dataclasses import dataclass, astuple from typing import List, Set, Type, Iterable -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins from volatility3.framework.configuration import requirements @@ -453,16 +453,18 @@ class InodePages(plugins.PluginInterface): # created, saving both disk space and I/O time. # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. + inode_size = inode.i_size try: - with open_method(filename) as f: - inode_size = inode.i_size - f.truncate(inode_size) - + file_initialized = False + with open_method(filename) as file_obj: for page_idx, page_content in inode.get_contents(): current_fp = page_idx * vmlinux_layer.page_size max_length = inode_size - current_fp - page_bytes = page_content[:max_length] - if current_fp + len(page_bytes) > inode_size: + page_bytes_len = min(max_length, len(page_content)) + if ( + current_fp >= inode_size + or current_fp + page_bytes_len > inode_size + ): vollog.error( "Page out of file bounds: inode 0x%x, inode size %d, page index %d", inode.vol.offset, @@ -470,10 +472,20 @@ class InodePages(plugins.PluginInterface): page_idx, ) continue + page_bytes = page_content[:page_bytes_len] - f.seek(current_fp) - f.write(page_bytes) + if not file_initialized: + # Lazy initialization to avoid truncating the file until we are + # certain there is something to write + file_obj.truncate(inode_size) + file_initialized = True + file_obj.seek(current_fp) + file_obj.write(page_bytes) + except exceptions.LinuxPageCacheException: + vollog.error( + f"Error dumping cached pages for inode at {inode.vol.offset:#x}" + ) except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) @@ -514,31 +526,44 @@ class InodePages(plugins.PluginInterface): return None inode_size = inode.i_size - for page_obj in inode.get_pages(): - page_vaddr = page_obj.vol.offset - page_paddr = page_obj.to_paddr() - page_mapping_addr = page_obj.mapping - page_index = int(page_obj.index) - page_file_offset = page_index * vmlinux_layer.page_size - dump_safe = ( - page_file_offset < inode_size - and page_mapping_addr - and page_mapping_addr.is_readable() - ) - page_flags_list = page_obj.get_flags_list() - page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) - fields = ( - page_vaddr, - page_paddr, - page_mapping_addr, - page_index, - dump_safe, - page_flags, - ) + if not self.config["dump"]: + try: + for page_obj in inode.get_pages(): + if page_obj.mapping != inode.i_mapping: + vollog.warning( + f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" + ) + continue + page_vaddr = page_obj.vol.offset + page_paddr = page_obj.to_paddr() + page_mapping_addr = page_obj.mapping + page_index = int(page_obj.index) + page_file_offset = page_index * vmlinux_layer.page_size + dump_safe = ( + page_file_offset < inode_size + and page_mapping_addr + and page_mapping_addr.is_readable() + ) + page_flags_list = page_obj.get_flags_list() + page_flags = ",".join( + [x.replace("PG_", "") for x in page_flags_list] + ) + fields = ( + page_vaddr, + page_paddr, + page_mapping_addr, + page_index, + dump_safe, + page_flags, + ) - yield 0, fields + yield 0, fields + except exceptions.LinuxPageCacheException: + vollog.warning( + f"Page cache for inode at {inode.vol.offset:#x} is corrupt" + ) - if self.config["dump"]: + else: open_method = self.open inode_address = inode.vol.offset filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") From 5502a54198fc7617bb22e2e96519114c59704ab0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:38:41 +1100 Subject: [PATCH 871/979] linux: page_cache plugin: Refactor to make _generator more readable --- .../framework/plugins/linux/pagecache.py | 83 ++++++++++--------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 7ad0f5a8f..39ed60486 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type, Iterable +from typing import List, Set, Type, Iterable, Tuple from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.renderers import format_hints @@ -489,6 +489,44 @@ class InodePages(plugins.PluginInterface): except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) + def _generate_inode_fields( + self, + inode: interfaces.objects.ObjectInterface, + vmlinux_layer: interfaces.layers.TranslationLayerInterface, + ) -> Iterable[Tuple[int, int, int, int, bool, str]]: + inode_size = inode.i_size + try: + for page_obj in inode.get_pages(): + if page_obj.mapping != inode.i_mapping: + vollog.warning( + f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" + ) + continue + page_vaddr = page_obj.vol.offset + page_paddr = page_obj.to_paddr() + page_mapping_addr = page_obj.mapping + page_index = int(page_obj.index) + page_file_offset = page_index * vmlinux_layer.page_size + dump_safe = ( + page_file_offset < inode_size + and page_mapping_addr + and page_mapping_addr.is_readable() + ) + page_flags_list = page_obj.get_flags_list() + page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) + fields = ( + page_vaddr, + page_paddr, + page_mapping_addr, + page_index, + dump_safe, + page_flags, + ) + + yield 0, fields + except exceptions.LinuxPageCacheException: + vollog.warning(f"Page cache for inode at {inode.vol.offset:#x} is corrupt") + def _generator(self): vmlinux_module_name = self.config["kernel"] vmlinux = self.context.modules[vmlinux_module_name] @@ -510,7 +548,6 @@ class InodePages(plugins.PluginInterface): else: vollog.error("Unable to find inode with path %s", self.config["find"]) return None - elif self.config["inode"]: inode = vmlinux.object("inode", self.config["inode"], absolute=True) else: @@ -525,45 +562,7 @@ class InodePages(plugins.PluginInterface): vollog.error("The inode is not a regular file") return None - inode_size = inode.i_size - if not self.config["dump"]: - try: - for page_obj in inode.get_pages(): - if page_obj.mapping != inode.i_mapping: - vollog.warning( - f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" - ) - continue - page_vaddr = page_obj.vol.offset - page_paddr = page_obj.to_paddr() - page_mapping_addr = page_obj.mapping - page_index = int(page_obj.index) - page_file_offset = page_index * vmlinux_layer.page_size - dump_safe = ( - page_file_offset < inode_size - and page_mapping_addr - and page_mapping_addr.is_readable() - ) - page_flags_list = page_obj.get_flags_list() - page_flags = ",".join( - [x.replace("PG_", "") for x in page_flags_list] - ) - fields = ( - page_vaddr, - page_paddr, - page_mapping_addr, - page_index, - dump_safe, - page_flags, - ) - - yield 0, fields - except exceptions.LinuxPageCacheException: - vollog.warning( - f"Page cache for inode at {inode.vol.offset:#x} is corrupt" - ) - - else: + if self.config["dump"]: open_method = self.open inode_address = inode.vol.offset filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") @@ -571,6 +570,8 @@ class InodePages(plugins.PluginInterface): self.write_inode_content_to_file( inode, filename, open_method, vmlinux_layer ) + else: + yield from self._generate_inode_fields(inode, vmlinux_layer) def run(self): headers = [ From 9944fcc61f179a0d949266a90d3e0e290d5017c6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 21:15:00 +1100 Subject: [PATCH 872/979] linux: page_cache test case: Since the --dump no longer generate output, we test both modes, listing and dumping. --- test/test_volatility.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index bb7c9a851..e25e8278b 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -714,7 +714,7 @@ def test_linux_page_cache_inodepages(image, volatility, python): image, volatility, python, - pluginargs=["--inode", inode_address, "--dump"], + pluginargs=["--inode", inode_address], ) assert rc == 0 @@ -725,6 +725,14 @@ def test_linux_page_cache_inodepages(image, volatility, python): rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", out, ) + + rc, out, _err = runvol_plugin( + "linux.pagecache.InodePages", + image, + volatility, + python, + pluginargs=["--inode", inode_address, "--dump"], + ) assert os.path.exists(inode_dump_filename) with open(inode_dump_filename, "rb") as fp: inode_contents = fp.read() From d8254b63735388b9ef6be27009474ccea5726650 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 21:39:37 +1100 Subject: [PATCH 873/979] linux: page_cache test case: Improve test --- test/test_volatility.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index e25e8278b..8676d1f3e 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -708,24 +708,25 @@ def test_linux_page_cache_inodepages(image, volatility, python): inode_address = hex(0x88001AB5C270) inode_dump_filename = f"inode_{inode_address}.dmp" + + rc, out, _err = runvol_plugin( + "linux.pagecache.InodePages", + image, + volatility, + python, + pluginargs=["--inode", inode_address], + ) + + assert rc == 0 + assert out.count(b"\n") > 4 + + # PageVAddr PagePAddr MappingAddr .. DumpSafe + assert re.search( + rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", + out, + ) + try: - rc, out, _err = runvol_plugin( - "linux.pagecache.InodePages", - image, - volatility, - python, - pluginargs=["--inode", inode_address], - ) - - assert rc == 0 - assert out.count(b"\n") > 4 - - # PageVAddr PagePAddr MappingAddr .. DumpSafe - assert re.search( - rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", - out, - ) - rc, out, _err = runvol_plugin( "linux.pagecache.InodePages", image, @@ -733,6 +734,10 @@ def test_linux_page_cache_inodepages(image, volatility, python): python, pluginargs=["--inode", inode_address, "--dump"], ) + + assert rc == 0 + assert out.count(b"\n") >= 4 + assert os.path.exists(inode_dump_filename) with open(inode_dump_filename, "rb") as fp: inode_contents = fp.read() From 749a0f9c656d8da0f5087faf25f225a6097ac8b4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 16 Jan 2025 12:06:22 +0000 Subject: [PATCH 874/979] Core: Fix up ISFinfo looking for fastjsonschema --- volatility3/framework/plugins/isfinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 34b0a5653..1c2ac52e9 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -97,7 +97,7 @@ class IsfInfo(plugins.PluginInterface): if filter_item in isf_file: filtered_list.append(isf_file) - if find_spec("fastjsonschema") and self.config["validate"]: + if find_spec("jsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False" From b447bfa81c36e91c3cf30bdc432e6eba48afbc53 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:24:38 +0100 Subject: [PATCH 875/979] remove module tainting proxies --- .../framework/plugins/linux/modxview.py | 16 ++++++++++-- .../symbols/linux/extensions/__init__.py | 25 ------------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 3655200e8..69c6ac8bb 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -140,9 +140,21 @@ class Modxview(interfaces.plugins.PluginInterface): seen_addresses.add(module.vol.offset) if self.config.get("plain_taints"): - taints = module.get_taints_as_plain_string() + taints = tainting.Tainting.get_taints_as_plain_string( + self.context, + kernel_name, + module.taints, + True, + ) else: - taints = ",".join(module.get_taints_parsed()) + taints = ",".join( + tainting.Tainting.get_taints_parsed( + self.context, + kernel_name, + module.taints, + True, + ) + ) yield ( 0, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index ac2f87df0..289d6c0a4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -19,7 +19,6 @@ from volatility3.framework.layers import linear, intel from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf -from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -279,30 +278,6 @@ class module(generic.GenericIntelProcess): return None - def get_taints_as_plain_string(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Convenient wrapper around framework's Tainting capabilities. - - Returns: - The raw taints string. - """ - return tainting.Tainting( - self._context, - linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, - ).get_taints_as_plain_string(self.taints, True) - - def get_taints_parsed(self) -> List[str]: - """Convert the module's taints string to a 1-1 descriptor mapping. - Convenient wrapper around framework's Tainting capabilities. - - Returns: - A comprehensive (user-friendly) taint descriptor list. - """ - return tainting.Tainting( - self._context, - linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, - ).get_taints_parsed(self.taints, True) - @property def section_symtab(self): if self.has_member("kallsyms"): From 94704c6674d7f5fb9d57698faa0d9ed943c6158c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:26:57 +0100 Subject: [PATCH 876/979] 2.16.0 -> 2.17.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 832b2a5ba..3d68ab810 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 18 # Number of changes that only add to the interface +VERSION_MINOR = 17 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From cd8690a8059836f7e216c211c4397924ae311c84 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:27:22 +0100 Subject: [PATCH 877/979] require framework version 2.17.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 69c6ac8bb..3c2c5f05e 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 18, 0) + _required_framework_version = (2, 17, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 72056a8d0006471c2f2ed58ce36714bd0ac5fb96 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 07:28:13 +1100 Subject: [PATCH 878/979] linux: mount api: fix vfsmount pointer check --- volatility3/framework/symbols/linux/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 931c461b9..f9a3ddde5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -139,7 +139,9 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): if not (rdentry and rdentry.is_readable() and rmnt and rmnt.is_readable()): return "" - if isinstance(vfsmnt, objects.Pointer) and not (rmnt and rmnt.is_readable()): + if isinstance(vfsmnt, objects.Pointer) and not ( + vfsmnt and vfsmnt.is_readable() + ): # vfsmnt can be the vfsmount object itself (>=3.3) or a vfsmount * (<3.3) return "" From 4d05e8a76c4b91ed9c998bce316d230614af031e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 08:49:20 +1100 Subject: [PATCH 879/979] linux: mount API: escape '*' in docstrings to ensure correct documentation rendering --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 6d341653b..9ba3cabab 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1520,17 +1520,17 @@ class vfsmount(objects.StructType): """Helper to make sure it is comparing two pointers to 'vfsmount'. Depending on the kernel version, see 3376f34fff5be9954fd9a9c4fd68f4a0a36d480e, - the calling object (self) could be a 'vfsmount *' (<3.3) or a 'vfsmount' (>=3.3). + the calling object (self) could be a 'vfsmount \\*' (<3.3) or a 'vfsmount' (>=3.3). This way we trust in the framework "auto" dereferencing ability to assure that when we reach this point 'self' will be a 'vfsmount' already and self.vol.offset - a 'vfsmount *' and not a 'vfsmount **'. The argument must be a 'vfsmount *'. + a 'vfsmount \\*' and not a 'vfsmount \\*\\*'. The argument must be a 'vfsmount \\*'. Typically, it's called from do_get_path(). Args: vfsmount_ptr: A pointer to a 'vfsmount' Raises: - exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount *' + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \\*' Returns: 'True' if the given argument points to the same 'vfsmount' as 'self'. From ad0c48e8c9871538e8e31e7047553c68ddcfc69f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 08:50:57 +1100 Subject: [PATCH 880/979] linux: mount info plugin: revert minor version increment in favor of a patch-level update --- volatility3/framework/plugins/linux/mountinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 5a0d39f31..47d8705c8 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -36,7 +36,7 @@ class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 2, 0) - _version = (1, 3, 0) + _version = (1, 2, 4) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 50c690e280dedfeb8f64a46f908c9bddc9e06f70 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:27:07 +1100 Subject: [PATCH 881/979] linux: proc.Maps plugin: improve pointers validation --- volatility3/framework/plugins/linux/proc.py | 59 ++++++++++++--------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 441c6bc93..23d6605b7 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -21,7 +21,7 @@ class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @@ -83,18 +83,24 @@ class Maps(plugins.PluginInterface): Returns: Yields vmas based on the task and filtered based on the filter function """ - if task.mm: - for vma in task.mm.get_vma_iter(): - if filter_func(vma): - yield vma - else: - vollog.debug( - f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.pid} due to filter_func" - ) - else: + mm_pointer = task.mm + if not mm_pointer: vollog.debug( - f"Excluded pid {task.pid} as there is no mm member. It is likely a kernel thread." + f"Excluded pid {task.pid} as there is no mm member. It is likely a kernel thread" ) + return + + if not mm_pointer.is_readable(): + vollog.error(f"Task {task.pid} has an invalid mm member") + return + + for vma in mm_pointer.get_vma_iter(): + if filter_func(vma): + yield vma + else: + vollog.debug( + f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.pid} due to filter_func" + ) @classmethod def vma_dump( @@ -174,31 +180,32 @@ class Maps(plugins.PluginInterface): ] # if any of the user supplied addresses would fall within this vma return true - if addrs_in_vma: - return True - else: - return False + return bool(addrs_in_vma) vma_filter_func = vma_filter_function + for task in tasks: - if not task.mm: + if not (task.mm and task.mm.is_readable()): continue name = utility.array_to_string(task.comm) for vma in self.list_vmas(task, filter_func=vma_filter_func): flags = vma.get_protection() page_offset = vma.get_page_offset() - major = 0 - minor = 0 - inode = 0 - if vma.vm_file != 0: + inode_num = None + try: dentry = vma.vm_file.get_dentry() - if dentry != 0: - inode_object = dentry.d_inode - major = inode_object.i_sb.major - minor = inode_object.i_sb.minor - inode = inode_object.i_ino + inode_ptr = dentry.d_inode + inode_num = inode_ptr.i_ino + major = inode_ptr.i_sb.major + minor = inode_ptr.i_sb.minor + except exceptions.InvalidAddressException: + if not inode_num: + inode_num = 0 + major = 0 + minor = 0 + path = vma.get_name(self.context, task) file_output = "Disabled" @@ -238,7 +245,7 @@ class Maps(plugins.PluginInterface): format_hints.Hex(page_offset), major, minor, - inode, + inode_num, path, file_output, ), From 5dc4dfcee8aca6b3e1b0a314e53e8476aa499339 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:30:59 +1100 Subject: [PATCH 882/979] linux: bpf_prog extension object: improve pointers validation --- volatility3/framework/symbols/linux/extensions/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..a3cd66238 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2023,8 +2023,11 @@ class bpf_prog(objects.StructType): prog_tag_addr = self.tag.vol.offset prog_tag_size = self.tag.count - prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) + if not vmlinux_layer.is_valid(prog_tag_addr, prog_tag_size): + vollog.debug("Unable to read bpf tag string from 0x%x", prog_tag_addr) + return None + prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) prog_tag = binascii.hexlify(prog_tag_bytes).decode() return prog_tag From c28944cdd3c65c72e033d1fc344fdfa858a2545a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:33:01 +1100 Subject: [PATCH 883/979] linux:sockstat plugin and list_sockets API: improve pointers validation --- volatility3/framework/plugins/linux/sockstat.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 7376bcbee..764c04563 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -438,7 +438,7 @@ class Sockstat(plugins.PluginInterface): """Lists all network connections for all processes.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 2) + _version = (3, 0, 3) @classmethod def get_requirements(cls): @@ -514,25 +514,28 @@ class Sockstat(plugins.PluginInterface): fd_num, filp, _full_path = fd_internal.fd_fields task = fd_internal.task + if not (filp.f_op and filp.f_op.is_readable()): + continue + if filp.f_op not in (sfop_addr, dfop_addr): continue dentry = filp.get_dentry() - if not dentry: + if not (dentry and dentry.is_readable()): continue d_inode = dentry.d_inode - if not d_inode: + if not (d_inode and d_inode.is_readable()): continue socket_alloc = linux.LinuxUtilities.container_of( d_inode, "socket_alloc", "vfs_inode", vmlinux ) - socket = socket_alloc.socket - - if not (socket and socket.sk): + if not socket_alloc: + continue + socket = socket_alloc.socket + if not (socket.sk and socket.sk.is_readable()): continue - sock = socket.sk.dereference() sock_type = sock.get_type() From 77b8058913910badde4f6a48c0af568d45ceae6f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:41:25 +1100 Subject: [PATCH 884/979] linux:kmsg plugin: improve pointers validation --- volatility3/framework/plugins/linux/kmsg.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 638c2ccf2..248e37dd8 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -136,7 +136,12 @@ class ABCKmsg(ABC): """ def get_string(self, addr: int, length: int) -> str: - txt = self._context.layers[self.layer_name].read(addr, length) # type: ignore + layer = self._context.layers[self.layer_name] + if not layer.is_valid(addr, length): + return "" + + txt = layer.read(addr, length) + return txt.decode(encoding="utf8", errors="replace") def nsec_to_sec_str(self, nsec: int) -> str: @@ -281,9 +286,13 @@ class Kmsg_3_5_to_3_11(ABCKmsg): log_struct_name = self._get_log_struct_name() log_struct_size = self.vmlinux.get_type(log_struct_name).size dict_offset = msg.vol.offset + log_struct_size + msg.text_len - dict_data = self._context.layers[self.layer_name].read( - dict_offset, msg.dict_len - ) + layer = self._context.layers[self.layer_name] + try: + dict_data = layer.read(dict_offset, msg.dict_len) + except exceptions.InvalidAddressException: + vollog.debug("Unable to read kmsg dict from 0x%x", dict_offset) + return None + for chunk in dict_data.split(b"\x00"): yield " " + chunk.decode() From 583a06630f9cef5659fb5667c74911701e0e095c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:49:53 +1100 Subject: [PATCH 885/979] linux:check_syscall plugin: improve pointers validation --- volatility3/framework/plugins/linux/check_syscall.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 13d312f2f..9ffd4c497 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -82,7 +82,7 @@ class Check_syscall(plugins.PluginInterface): return table_size - def _get_table_info_disassembly(self, ptr_sz, vmlinux): + def _get_table_info_disassembly(self, ptr_sz, vmlinux) -> int: """Find the size of the system call table by disassembling functions that immediately reference it in their first instruction This is in the form 'cmp reg,NR_syscalls'.""" @@ -107,9 +107,13 @@ class Check_syscall(plugins.PluginInterface): return 0 vmlinux = self.context.modules[self.config["kernel"]] - data = self.context.layers.read(vmlinux.layer_name, func_addr, 6) + vmlinux_layer = self.context.layers[vmlinux.layer_name] + try: + data = vmlinux_layer.read(func_addr, 6) + except exceptions.InvalidAddressException: + return 0 - for address, size, mnemonic, op_str in md.disasm_lite(data, func_addr): + for _address, _size, mnemonic, op_str in md.disasm_lite(data, func_addr): if mnemonic == "CMP": table_size = int(op_str.split(",")[1].strip()) & 0xFFFF break From 6dac7195dc88518fa8b40a70d57e855d40b30b01 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:50:53 +1100 Subject: [PATCH 886/979] linux: elf_linkmap object extension: Use a more generic invalid address exception --- volatility3/framework/symbols/linux/extensions/elf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index eadcbbae0..fb5f89f60 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -437,7 +437,7 @@ class elf_linkmap(objects.StructType): def get_name(self): try: buf = self._context.layers.read(self.vol.layer_name, self.l_name, 256) - except exceptions.PagedInvalidAddressException: + except exceptions.InvalidAddressException: # Protection against memory smear vollog.log( constants.LOGLEVEL_VVVV, From adf81bc74a388d6ff5bffabe588bc5ca72147506 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 16 Jan 2025 19:59:31 +0000 Subject: [PATCH 887/979] Update copyright dates --- README.md | 2 +- doc/source/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cc33d3cc4..b74bdab0b 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ The latest generated copy of the documentation can be found at: Date: Fri, 17 Jan 2025 16:03:47 +0000 Subject: [PATCH 888/979] Core: Correct version dependencies to avoid conflicts Fixes #1546 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86e3921d2..542a1480a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ cloud = [ dev = [ "volatility3[full,cloud]", "jsonschema>=4.23.0,<5", - "pyinstaller>=6.11.0,<7", + "pyinstaller>=6.5.0,<7", "pyinstaller-hooks-contrib>=2024.9", "types-jsonschema>=4.23.0,<5", ] @@ -48,7 +48,7 @@ test = [ docs = [ "volatility3[dev]", - "sphinx>=8.0.0,<7", + "sphinx>=8.0.0,<9", "sphinx-autodoc-typehints>=2.5.0,<3", "sphinx-rtd-theme>=3.0.1,<4", ] From c4430cda8d6b13b0d69a787fe32e8158ae471c3a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 17 Jan 2025 16:09:35 +0000 Subject: [PATCH 889/979] Core: Try to maintain python-3.8 support for the documentation --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 542a1480a..8944bb058 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ test = [ docs = [ "volatility3[dev]", "sphinx>=8.0.0,<9", - "sphinx-autodoc-typehints>=2.5.0,<3", + "sphinx-autodoc-typehints>=2.0.0,<3", "sphinx-rtd-theme>=3.0.1,<4", ] From 13a8c53f7b64bc7180b665e363b2a3f0348e8b04 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 17 Jan 2025 16:13:35 +0000 Subject: [PATCH 890/979] Core: There was no clear reason to stop supporting older versions of sphinx --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8944bb058..3f16eeece 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ test = [ docs = [ "volatility3[dev]", - "sphinx>=8.0.0,<9", + "sphinx>=4.0.0,<9", "sphinx-autodoc-typehints>=2.0.0,<3", "sphinx-rtd-theme>=3.0.1,<4", ] From cc9486cf03f6f8b8035069f02702ce30a589cb7e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 02:19:11 +0100 Subject: [PATCH 891/979] pre-process module triaging to improve readability --- .../framework/plugins/linux/modxview.py | 71 ++++++++++--------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 3c2c5f05e..125c1cc33 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -82,7 +82,8 @@ class Modxview(interfaces.plugins.PluginInterface): kernel_name: str, run_hidden_modules: bool = True, ) -> Dict[str, List[extensions.module]]: - """Run module scanning plugins and aggregate the results. + """Run module scanning plugins and aggregate the results. It is designed + to not operate any inter-plugin results triage. Args: run_hidden_modules: specify if the hidden_modules plugin should be run @@ -128,46 +129,50 @@ class Modxview(interfaces.plugins.PluginInterface): def _generator(self): kernel_name = self.config["kernel"] run_results = self.run_modules_scanners(self.context, kernel_name) - modules_offsets = {} - for key in ["lsmod", "check_modules", "hidden_modules"]: - modules_offsets[key] = set(module.vol.offset for module in run_results[key]) + aggregated_modules = {} + # We want to be explicit on the plugins results we are interested in + for plugin_name in ["lsmod", "check_modules", "hidden_modules"]: + # Iterate over each recovered module + for module in run_results[plugin_name]: + # Use offsets as unique keys, whether a module + # appears in many plugin runs or not + if aggregated_modules.get(module.vol.offset): + # Append the plugin to the list of originating plugins + aggregated_modules[module.vol.offset][1].append(plugin_name) + else: + aggregated_modules[module.vol.offset] = (module, [plugin_name]) - seen_addresses = set() - for modules_list in run_results.values(): - for module in modules_list: - if module.vol.offset in seen_addresses: - continue - seen_addresses.add(module.vol.offset) - - if self.config.get("plain_taints"): - taints = tainting.Tainting.get_taints_as_plain_string( + for module_offset, (module, originating_plugins) in aggregated_modules.items(): + # Tainting parsing capabilities applied to the module + if self.config.get("plain_taints"): + taints = tainting.Tainting.get_taints_as_plain_string( + self.context, + kernel_name, + module.taints, + True, + ) + else: + taints = ",".join( + tainting.Tainting.get_taints_parsed( self.context, kernel_name, module.taints, True, ) - else: - taints = ",".join( - tainting.Tainting.get_taints_parsed( - self.context, - kernel_name, - module.taints, - True, - ) - ) - - yield ( - 0, - ( - module.get_name() or NotAvailableValue(), - format_hints.Hex(module.vol.offset), - module.vol.offset in modules_offsets["lsmod"], - module.vol.offset in modules_offsets["check_modules"], - module.vol.offset in modules_offsets["hidden_modules"], - taints or NotAvailableValue(), - ), ) + yield ( + 0, + ( + module.get_name() or NotAvailableValue(), + format_hints.Hex(module_offset), + "lsmod" in originating_plugins, + "check_modules" in originating_plugins, + "hidden_modules" in originating_plugins, + taints or NotAvailableValue(), + ), + ) + def run(self): columns = [ ("Name", str), From 3b679cbafbb50a2c986a63efd223cf9088bbc330 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:26:43 +0100 Subject: [PATCH 892/979] explicit None check --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 125c1cc33..c74bf28e8 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -136,7 +136,7 @@ class Modxview(interfaces.plugins.PluginInterface): for module in run_results[plugin_name]: # Use offsets as unique keys, whether a module # appears in many plugin runs or not - if aggregated_modules.get(module.vol.offset): + if aggregated_modules.get(module.vol.offset, None) is not None: # Append the plugin to the list of originating plugins aggregated_modules[module.vol.offset][1].append(plugin_name) else: From bb6556dbc0145682866d56bb2608b5b841e381e8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:37:52 +0100 Subject: [PATCH 893/979] correct arguments for pre_4_10_rc1 --- volatility3/framework/symbols/linux/utilities/tainting.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 552f51b98..14b69d3d6 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -117,9 +117,7 @@ class Tainting(interfaces.configuration.VersionableInterface): return cls._module_flags_taint_post_4_10_rc1( context, kernel_module_name, taints, is_module ) - return cls._module_flags_taint_pre_4_10_rc1( - context, kernel_module_name, taints, is_module - ) + return cls._module_flags_taint_pre_4_10_rc1(taints, is_module) @classmethod def get_taints_parsed( From 0b82f731375583076abdfabd332ce067612d69f5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:45:30 +0100 Subject: [PATCH 894/979] functools caching and doc. --- .../framework/symbols/linux/utilities/tainting.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 14b69d3d6..c1136436e 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -1,3 +1,5 @@ +import functools + from volatility3 import framework from volatility3.framework import interfaces from volatility3.framework.constants import linux as linux_constants @@ -18,11 +20,18 @@ class Tainting(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) @classmethod + @functools.lru_cache def _get_kernel_taint_flags_list( cls, context: interfaces.context.ContextInterface, kernel_module_name: str, ) -> Optional[List[interfaces.objects.ObjectInterface]]: + """Determine whether the kernel embeds taint flags definition + in-memory or not. + + Returns: + A list of "taint_flag" kernel objects if taint_flags symbok exists + """ kernel = context.modules[kernel_module_name] if kernel.has_symbol("taint_flags"): return list(kernel.object_from_symbol("taint_flags")) From 8095924e8a926990f6002f16d2c7259c5c750980 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:46:13 +0100 Subject: [PATCH 895/979] typo --- volatility3/framework/symbols/linux/utilities/tainting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index c1136436e..2360401d5 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -30,7 +30,7 @@ class Tainting(interfaces.configuration.VersionableInterface): in-memory or not. Returns: - A list of "taint_flag" kernel objects if taint_flags symbok exists + A list of "taint_flag" kernel objects if taint_flags symbol exists """ kernel = context.modules[kernel_module_name] if kernel.has_symbol("taint_flags"): From 2fe8ee5983bd5faf8a89db5712512aa411329dbc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 18 Jan 2025 13:18:53 +0000 Subject: [PATCH 896/979] Layers: Update LeechCore RawIO with better error handling for readlines Fixes #1419 --- volatility3/framework/layers/leechcore.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index eeede1673..06c359203 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -129,6 +129,8 @@ if HAS_LEECHCORE: def readline(self, __size: Optional[int] = ...) -> bytes: data = b"" + if not __size: + __size = 0 while __size > self._chunk_size or __size < 0: data += self.read(self._chunk_size) index = data.find(b"\n") From 0849c163a1c517fa8595f9cc7610a737d1904fc2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:48:37 +0100 Subject: [PATCH 897/979] appropriate symbols type hinting --- volatility3/framework/contexts/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index f527544c0..17a91e827 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -337,7 +337,7 @@ class Module(interfaces.context.ModuleInterface): ) @property - def symbols(self): + def symbols(self) -> Iterable[str]: return self.context.symbol_space[self.symbol_table_name].symbols get_symbol = get_module_wrapper("get_symbol") From d46cb3328d07ae2216045ba3fc33679c2ab13fbc Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:48:46 +0100 Subject: [PATCH 898/979] appropriate symbols type hinting --- volatility3/framework/interfaces/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index a87e0f1e8..2b95a18ad 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -303,8 +303,8 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): """Determines whether an enumeration is present in the module's symbol table.""" @abstractmethod - def symbols(self) -> List: - """Lists the symbols contained in the symbol table for this module""" + def symbols(self) -> Iterable[str]: + """Returns an iterable of the symbols contained in the symbol table for this module""" @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: From fb93d2333b8d3854d348548decc76ac67f358699 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:50:14 +0100 Subject: [PATCH 899/979] improve comments --- volatility3/framework/interfaces/symbols.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index b8712e38d..c0bebe1e2 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -122,7 +122,7 @@ class BaseSymbolTableInterface: @property def symbols(self) -> Iterable[str]: - """Returns an iterator of the Symbol names.""" + """Returns an iterable of the available symbol names.""" raise NotImplementedError( "Abstract property symbols not implemented by subclass." ) @@ -131,7 +131,7 @@ class BaseSymbolTableInterface: @property def types(self) -> Iterable[str]: - """Returns an iterator of the Symbol type names.""" + """Returns an iterable of the available symbol type names.""" raise NotImplementedError( "Abstract property types not implemented by subclass." ) @@ -149,7 +149,7 @@ class BaseSymbolTableInterface: @property def enumerations(self) -> Iterable[Any]: - """Returns an iterator of the Enumeration names.""" + """Returns an iterable of the available enumerations names.""" raise NotImplementedError( "Abstract property enumerations not implemented by subclass." ) @@ -366,6 +366,7 @@ class NativeTableInterface(BaseSymbolTableInterface): @property def symbols(self) -> Iterable[str]: + """Returns an iterable of the available symbol names.""" return [] def get_enumeration(self, name: str) -> objects.Template: From 1e9551b0530be824ab8d9a40db57cbd813d48136 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:50:28 +0100 Subject: [PATCH 900/979] types base class and comments improvements --- volatility3/framework/interfaces/symbols.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index c0bebe1e2..752d288f7 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -375,7 +375,13 @@ class NativeTableInterface(BaseSymbolTableInterface): ) @property - def enumerations(self) -> Iterable[str]: + def enumerations(self) -> Iterable[Any]: + """Returns an iterable of the available enumerations.""" + return [] + + @property + def types(self) -> Iterable[str]: + """Returns an iterable of the available symbol type names.""" return [] From aa99410dd2c50ee556293db959c469739c882684 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:51:31 +0100 Subject: [PATCH 901/979] prefer KeysView iterable to lists --- volatility3/framework/symbols/intermed.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 6802af7d6..0a30148aa 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -411,18 +411,23 @@ class Version1Format(ISFormatTable): @property def symbols(self) -> Iterable[str]: - """Returns an iterator of the symbol names.""" - return list(self._json_object.get("symbols", {})) + """Returns an iterable (KeysView) of the available symbol names.""" + return self._json_object.get("symbols", {}).keys() @property - def enumerations(self) -> Iterable[str]: - """Returns an iterator of the available enumerations.""" - return list(self._json_object.get("enums", {})) + def enumerations(self) -> Iterable[Any]: + """Returns an iterable (KeysView) of the available enumerations.""" + return self._json_object.get("enums", {}).keys() @property - def types(self) -> Iterable[str]: - """Returns an iterator of the symbol type names.""" - return list(self._json_object.get("user_types", {})) + list(self.natives.types) + def types(self): + """Returns an iterable (KeysView) of the available symbol type names.""" + # self.natives.types (set) is generally very small compared to user_types, + # so the dict conversion overhead can be neglected + return { + **self._json_object.get("user_types", {}), + **dict.fromkeys(self.natives.types), + }.keys() def get_type_class(self, name: str) -> Type[interfaces.objects.ObjectInterface]: return self._overrides.get(name, objects.AggregateType) From 3a2933155b6f92a8585666f611cd2069424ce5a9 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:51:52 +0100 Subject: [PATCH 902/979] improve comments --- volatility3/framework/symbols/native.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/native.py b/volatility3/framework/symbols/native.py index 7c3e1b312..61417532e 100644 --- a/volatility3/framework/symbols/native.py +++ b/volatility3/framework/symbols/native.py @@ -30,7 +30,7 @@ class NativeTable(interfaces.symbols.NativeTableInterface): @property def types(self) -> Iterable[str]: - """Returns an iterator of the symbol type names.""" + """Returns an iterable (set) of the available symbol type names.""" return self._types def get_type(self, type_name: str) -> interfaces.objects.Template: From ba09db6952d37590f62a647265c2bb5bb903ec3c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:54:04 +0100 Subject: [PATCH 903/979] improve comments --- volatility3/framework/interfaces/symbols.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index 752d288f7..2d142de9a 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -149,7 +149,7 @@ class BaseSymbolTableInterface: @property def enumerations(self) -> Iterable[Any]: - """Returns an iterable of the available enumerations names.""" + """Returns an iterable of the available enumerations.""" raise NotImplementedError( "Abstract property enumerations not implemented by subclass." ) From 4b3d93b0f0637c7d41acc545f397e3522a913978 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:00:03 +0100 Subject: [PATCH 904/979] 2.17.0 -> 2.18.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 3d68ab810..832b2a5ba 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 17 # Number of changes that only add to the interface +VERSION_MINOR = 18 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From c5628c5d79496ae051942598bab08c19d3632a18 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:48:44 +0100 Subject: [PATCH 905/979] revert the mistakenly removed types type hinting --- volatility3/framework/symbols/intermed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 0a30148aa..9ece69d8b 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -420,7 +420,7 @@ class Version1Format(ISFormatTable): return self._json_object.get("enums", {}).keys() @property - def types(self): + def types(self) -> Iterable[str]: """Returns an iterable (KeysView) of the available symbol type names.""" # self.natives.types (set) is generally very small compared to user_types, # so the dict conversion overhead can be neglected From 0ee016e65554539318705a5b7c292864fcc2f436 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:50:28 +0100 Subject: [PATCH 906/979] 2.17.0 -> 2.17.1 bump --- volatility3/framework/constants/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 832b2a5ba..041439909 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 18 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_MINOR = 17 # Number of changes that only add to the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From c2ef3c2fe575f2c3ea49541b7e1207e7d93884f1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:59:51 +0100 Subject: [PATCH 907/979] add fixme about merge operator --- volatility3/framework/symbols/intermed.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 9ece69d8b..cb0b67969 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -422,8 +422,12 @@ class Version1Format(ISFormatTable): @property def types(self) -> Iterable[str]: """Returns an iterable (KeysView) of the available symbol type names.""" - # self.natives.types (set) is generally very small compared to user_types, - # so the dict conversion overhead can be neglected + # We use ** instead of + # `set(self._json_object.get("user_types", {}).keys()).union(self.natives.types)` + # because converting user_types dict to a set is costly. + # It is more efficient to convert the (very small) self.natives.types set to a dict. + # FIXME: On Python3.8 support drop, merge the two dicts using the merge operator: + # (self._json_object.get("user_types", {}) | dict.fromkeys(self.natives.types)).keys() return { **self._json_object.get("user_types", {}), **dict.fromkeys(self.natives.types), From 726fbe6ccec8480c1baa74fadcc7fc4e87377bcd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 13:01:14 +0100 Subject: [PATCH 908/979] 2.17.1 -> 2.18.0 bump --- volatility3/framework/constants/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 041439909..832b2a5ba 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 17 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 18 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From 08830fee05c30adfc5ea4e997ce317f6f6927033 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:06:42 +0100 Subject: [PATCH 909/979] use architectures.LINUX_ARCHS --- volatility3/framework/plugins/linux/pagecache.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 382268515..408a9b98a 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -8,6 +8,7 @@ import datetime from dataclasses import dataclass, astuple from typing import List, Set, Type, Iterable +from volatility3.framework.constants import architectures from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins @@ -112,7 +113,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=["Intel32", "Intel64"], + architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 2, 0) @@ -397,7 +398,7 @@ class InodePages(plugins.PluginInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=["Intel32", "Intel64"], + architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( name="files", plugin=Files, version=(1, 0, 0) From d325e1ca176b55f68de03f462ca855631736ef6c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:08:27 +0100 Subject: [PATCH 910/979] add inode_size and format_symlink to Inode* dataclasses --- volatility3/framework/plugins/linux/pagecache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 408a9b98a..c86664f3d 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -38,6 +38,11 @@ class InodeUser: modification_time: str change_time: str path: str + inode_size: int + + @staticmethod + def format_symlink(symlink_source: str, symlink_dest: str): + return f"{symlink_source} -> {symlink_dest}" @dataclass @@ -81,6 +86,7 @@ class InodeInternal: access_time_dt = self.inode.get_access_time() modification_time_dt = self.inode.get_modification_time() change_time_dt = self.inode.get_change_time() + inode_size = int(self.inode.i_size) inode_user = InodeUser( superblock_addr=superblock_addr, @@ -96,6 +102,7 @@ class InodeInternal: modification_time=modification_time_dt, change_time=change_time_dt, path=self.path, + inode_size=inode_size, ) return inode_user From 1d0159325fbf04bb5029a9ed4ae2ea1acc770cee Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:09:20 +0100 Subject: [PATCH 911/979] switch to InodeUser.format_symlink --- volatility3/framework/plugins/linux/pagecache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index c86664f3d..a0dd8efe4 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -156,10 +156,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): """ # i_link (fast symlinks) were introduced in 4.2 if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: - i_link_str = inode.i_link.dereference().cast( + symlink_dest = inode.i_link.dereference().cast( "string", max_length=255, encoding="utf-8", errors="replace" ) - symlink_path = f"{symlink_path} -> {i_link_str}" + symlink_path = InodeUser.format_symlink(symlink_path, symlink_dest) return symlink_path From 48a8f3929bbe992ff687a494e3ea0e6a7446f42c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:10:32 +0100 Subject: [PATCH 912/979] add and leverage follow_symlinks parameter --- volatility3/framework/plugins/linux/pagecache.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index a0dd8efe4..4871d0d0f 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -220,12 +220,14 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, + follow_symlinks: bool = True, ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: context: The context that the plugin will operate within vmlinux_module_name: The name of the kernel module on which to operate + follow_symlinks: Whether to follow symlinks or not Yields: An InodeInternal object @@ -297,7 +299,9 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): continue seen_inodes.add(file_inode_ptr) - file_path = cls._follow_symlink(file_inode_ptr, file_path) + if follow_symlinks: + file_path = cls._follow_symlink(file_inode_ptr, file_path) + inode_in = InodeInternal( superblock=superblock, mountpoint=mountpoint, From 6f2ff4f7c6f702c11bf7d75b5ab78cb8f5881a20 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:11:03 +0100 Subject: [PATCH 913/979] add InodeSize column to Files --- volatility3/framework/plugins/linux/pagecache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 4871d0d0f..770c6391a 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -389,6 +389,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): ("ModificationTime", datetime.datetime), ("ChangeTime", datetime.datetime), ("FilePath", str), + ("InodeSize", int), ] return renderers.TreeGrid( From 85941060051b530350ad8e770de4c1f2d3edefee Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:12:10 +0100 Subject: [PATCH 914/979] 1.0.1 -> 1.2.0 Files bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 770c6391a..d57bd77f8 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -112,7 +112,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 2, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From e8b44efc17dbfd6e412436d324702ed3e4cc7c7f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:13:26 +0100 Subject: [PATCH 915/979] add and leverage write_inode_content_to_stream --- .../framework/plugins/linux/pagecache.py | 60 ++++++++++++------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index d57bd77f8..42fa7538f 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type, Iterable +from typing import List, Set, Type, Iterable, IO from volatility3.framework.constants import architectures from volatility3.framework import renderers, interfaces @@ -452,31 +452,47 @@ class InodePages(plugins.PluginInterface): vollog.error("The inode is not a regular file") return None - # By using truncate/seek, provided the filesystem supports it, a sparse file will be + try: + with open_method(filename) as f: + InodePages.write_inode_content_to_stream(inode, f, vmlinux_layer) + except OSError as e: + vollog.error("Unable to write to file (%s): %s", filename, e) + + @staticmethod + def write_inode_content_to_stream( + inode: interfaces.objects.ObjectInterface, + stream: IO, + vmlinux_layer: interfaces.layers.TranslationLayerInterface, + ) -> None: + """Extracts the inode's contents from the page cache and saves them to a stream + + Args: + inode: The inode to dump + stream: A IO steam to write to, typically FileHandlerInterface or BytesIO + vmlinux_layer: The kernel layer to obtain the page size + """ + + # By using truncate/seek, provided the filesystem supports it, and the + # stream is a File interface, a sparse file will be # created, saving both disk space and I/O time. # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. - try: - with open_method(filename) as f: - inode_size = inode.i_size - f.truncate(inode_size) + inode_size = inode.i_size + stream.truncate(inode_size) - for page_idx, page_content in inode.get_contents(): - current_fp = page_idx * vmlinux_layer.page_size - max_length = inode_size - current_fp - page_bytes = page_content[:max_length] - if current_fp + len(page_bytes) > inode_size: - vollog.error( - "Page out of file bounds: inode 0x%x, inode size %d, page index %d", - inode.vol.offset, - inode_size, - page_idx, - ) - f.seek(current_fp) - f.write(page_bytes) - - except OSError as e: - vollog.error("Unable to write to file (%s): %s", filename, e) + for page_idx, page_content in inode.get_contents(): + current_fp = page_idx * vmlinux_layer.page_size + max_length = inode_size - current_fp + page_bytes = page_content[:max_length] + if current_fp + len(page_bytes) > inode_size: + vollog.error( + "Page out of file bounds: inode 0x%x, inode size %d, page index %d", + inode.vol.offset, + inode_size, + page_idx, + ) + stream.seek(current_fp) + stream.write(page_bytes) def _generator(self): vmlinux_module_name = self.config["kernel"] From 818ddb746bb5bcbf45fe22fb54e8022d75d5fedd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:15:55 +0100 Subject: [PATCH 916/979] 2.0.0 -> 2.1.0 InodePages bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 42fa7538f..feac31bb7 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -402,7 +402,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From da068676d13eeefae033e46f8a916d91881bc2a6 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:16:09 +0100 Subject: [PATCH 917/979] typo --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index feac31bb7..e87bc2c9d 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -468,7 +468,7 @@ class InodePages(plugins.PluginInterface): Args: inode: The inode to dump - stream: A IO steam to write to, typically FileHandlerInterface or BytesIO + stream: An IO steam to write to, typically FileHandlerInterface or BytesIO vmlinux_layer: The kernel layer to obtain the page size """ From 452d6e705b973213238becd22d736f6ee1cb45e0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 16:26:25 +0100 Subject: [PATCH 918/979] 1.0.1 -> 1.1.0 Files bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index e87bc2c9d..89d30a9eb 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -112,7 +112,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 2, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From ecae545f5897e7c9a59d231d0b3f04b195cbf169 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:05:52 +0100 Subject: [PATCH 919/979] typo --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7d211150a..850045244 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2948,7 +2948,7 @@ class scatterlist(objects.StructType): Returns: An iterator of bytes """ - # Either "physical" is layer-1 because this is a module layer, either "physical" is the current layer + # Either "physical" is layer-1 because this is a module layer, or "physical" is the current layer physical_layer_name = self._context.layers[self.vol.layer_name].config.get( "memory_layer", self.vol.layer_name ) From eddba98ef7c8c72162233dd19332e770d2a916d7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:09:59 +0100 Subject: [PATCH 920/979] type hint format_symlink --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 89d30a9eb..d98103368 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -41,7 +41,7 @@ class InodeUser: inode_size: int @staticmethod - def format_symlink(symlink_source: str, symlink_dest: str): + def format_symlink(symlink_source: str, symlink_dest: str) -> str: return f"{symlink_source} -> {symlink_dest}" From 59703045c78941a12087a295b18cc8bc98414a06 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:26:42 +0100 Subject: [PATCH 921/979] switch calling convention to context and layer name --- .../framework/plugins/linux/pagecache.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index d98103368..d5297ab10 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -435,18 +435,20 @@ class InodePages(plugins.PluginInterface): @staticmethod def write_inode_content_to_file( + context: interfaces.context.ContextInterface, + layer_name: str, inode: interfaces.objects.ObjectInterface, filename: str, open_method: Type[interfaces.plugins.FileHandlerInterface], - vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a file Args: + context: The context on which to operate + layer_name: The name of the layer on which to operate inode: The inode to dump filename: Filename for writing the inode content open_method: class for constructing output files - vmlinux_layer: The kernel layer to obtain the page size """ if not inode.is_reg: vollog.error("The inode is not a regular file") @@ -454,24 +456,26 @@ class InodePages(plugins.PluginInterface): try: with open_method(filename) as f: - InodePages.write_inode_content_to_stream(inode, f, vmlinux_layer) + InodePages.write_inode_content_to_stream(context, layer_name, inode, f) except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) @staticmethod def write_inode_content_to_stream( + context: interfaces.context.ContextInterface, + layer_name: str, inode: interfaces.objects.ObjectInterface, stream: IO, - vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a stream Args: + context: The context on which to operate + layer_name: The name of the layer on which to operate inode: The inode to dump stream: An IO steam to write to, typically FileHandlerInterface or BytesIO - vmlinux_layer: The kernel layer to obtain the page size """ - + layer = context.layers[layer_name] # By using truncate/seek, provided the filesystem supports it, and the # stream is a File interface, a sparse file will be # created, saving both disk space and I/O time. @@ -481,7 +485,7 @@ class InodePages(plugins.PluginInterface): stream.truncate(inode_size) for page_idx, page_content in inode.get_contents(): - current_fp = page_idx * vmlinux_layer.page_size + current_fp = page_idx * layer.page_size max_length = inode_size - current_fp page_bytes = page_content[:max_length] if current_fp + len(page_bytes) > inode_size: @@ -557,7 +561,7 @@ class InodePages(plugins.PluginInterface): filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") vollog.info("[*] Writing inode at 0x%x to '%s'", inode_address, filename) self.write_inode_content_to_file( - inode, filename, open_method, vmlinux_layer + self.context, vmlinux_layer.name, inode, filename, open_method ) def run(self): From a02243bb3c4a14076cda7a516c7499e1734f19d0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:27:27 +0100 Subject: [PATCH 922/979] 2.1.0 -> 3.0.0 InodePages bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index d5297ab10..a86c1b936 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -402,7 +402,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 1, 0) + _version = (3, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From d42ffc01e22672602a346b21f90b3733cd02db3e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:33:47 +0100 Subject: [PATCH 923/979] typo --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index a86c1b936..77aa42338 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -473,7 +473,7 @@ class InodePages(plugins.PluginInterface): context: The context on which to operate layer_name: The name of the layer on which to operate inode: The inode to dump - stream: An IO steam to write to, typically FileHandlerInterface or BytesIO + stream: An IO stream to write to, typically FileHandlerInterface or BytesIO """ layer = context.layers[layer_name] # By using truncate/seek, provided the filesystem supports it, and the From 74b98e62c7aa4ca721f48651243122fa222548a5 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 19 Jan 2025 23:42:41 +0000 Subject: [PATCH 924/979] Revert "Add missing exception handling in env var recovery. Prevent backtraces" --- volatility3/framework/plugins/linux/envars.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 04b75c8a8..8cdbfe493 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -5,7 +5,7 @@ import logging from typing import Iterable, Tuple -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -58,16 +58,10 @@ class Envars(plugins.PluginInterface): Tuples of (key, value) representing each environment variable. """ - # This ensures the `task` is valid as well as its - # memory mapping structures - try: - task_name = utility.array_to_string(task.comm) - env_start = task.mm.env_start - env_end = task.mm.env_end - except exceptions.InvalidAddressException: - return None - + task_name = utility.array_to_string(task.comm) task_pid = task.pid + env_start = task.mm.env_start + env_end = task.mm.env_end env_area_size = env_end - env_start if not (0 < env_area_size <= env_area_max_size): vollog.debug( From e18bffdde95e4adb0ee89aeb28b3845c53b37687 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 19 Jan 2025 23:51:28 +0000 Subject: [PATCH 925/979] Revert "Pre linux.pagecache.recoverfs support" --- .../framework/plugins/linux/pagecache.py | 97 ++++++------------- 1 file changed, 32 insertions(+), 65 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 77aa42338..382268515 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,9 +6,8 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type, Iterable, IO +from typing import List, Set, Type, Iterable -from volatility3.framework.constants import architectures from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins @@ -38,11 +37,6 @@ class InodeUser: modification_time: str change_time: str path: str - inode_size: int - - @staticmethod - def format_symlink(symlink_source: str, symlink_dest: str) -> str: - return f"{symlink_source} -> {symlink_dest}" @dataclass @@ -86,7 +80,6 @@ class InodeInternal: access_time_dt = self.inode.get_access_time() modification_time_dt = self.inode.get_modification_time() change_time_dt = self.inode.get_change_time() - inode_size = int(self.inode.i_size) inode_user = InodeUser( superblock_addr=superblock_addr, @@ -102,7 +95,6 @@ class InodeInternal: modification_time=modification_time_dt, change_time=change_time_dt, path=self.path, - inode_size=inode_size, ) return inode_user @@ -112,7 +104,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -120,7 +112,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=architectures.LINUX_ARCHS, + architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 2, 0) @@ -156,10 +148,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): """ # i_link (fast symlinks) were introduced in 4.2 if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: - symlink_dest = inode.i_link.dereference().cast( + i_link_str = inode.i_link.dereference().cast( "string", max_length=255, encoding="utf-8", errors="replace" ) - symlink_path = InodeUser.format_symlink(symlink_path, symlink_dest) + symlink_path = f"{symlink_path} -> {i_link_str}" return symlink_path @@ -220,14 +212,12 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, - follow_symlinks: bool = True, ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: context: The context that the plugin will operate within vmlinux_module_name: The name of the kernel module on which to operate - follow_symlinks: Whether to follow symlinks or not Yields: An InodeInternal object @@ -299,9 +289,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): continue seen_inodes.add(file_inode_ptr) - if follow_symlinks: - file_path = cls._follow_symlink(file_inode_ptr, file_path) - + file_path = cls._follow_symlink(file_inode_ptr, file_path) inode_in = InodeInternal( superblock=superblock, mountpoint=mountpoint, @@ -389,7 +377,6 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): ("ModificationTime", datetime.datetime), ("ChangeTime", datetime.datetime), ("FilePath", str), - ("InodeSize", int), ] return renderers.TreeGrid( @@ -402,7 +389,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -410,7 +397,7 @@ class InodePages(plugins.PluginInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=architectures.LINUX_ARCHS, + architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( name="files", plugin=Files, version=(1, 0, 0) @@ -435,68 +422,48 @@ class InodePages(plugins.PluginInterface): @staticmethod def write_inode_content_to_file( - context: interfaces.context.ContextInterface, - layer_name: str, inode: interfaces.objects.ObjectInterface, filename: str, open_method: Type[interfaces.plugins.FileHandlerInterface], + vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a file Args: - context: The context on which to operate - layer_name: The name of the layer on which to operate inode: The inode to dump filename: Filename for writing the inode content open_method: class for constructing output files + vmlinux_layer: The kernel layer to obtain the page size """ if not inode.is_reg: vollog.error("The inode is not a regular file") return None - try: - with open_method(filename) as f: - InodePages.write_inode_content_to_stream(context, layer_name, inode, f) - except OSError as e: - vollog.error("Unable to write to file (%s): %s", filename, e) - - @staticmethod - def write_inode_content_to_stream( - context: interfaces.context.ContextInterface, - layer_name: str, - inode: interfaces.objects.ObjectInterface, - stream: IO, - ) -> None: - """Extracts the inode's contents from the page cache and saves them to a stream - - Args: - context: The context on which to operate - layer_name: The name of the layer on which to operate - inode: The inode to dump - stream: An IO stream to write to, typically FileHandlerInterface or BytesIO - """ - layer = context.layers[layer_name] - # By using truncate/seek, provided the filesystem supports it, and the - # stream is a File interface, a sparse file will be + # By using truncate/seek, provided the filesystem supports it, a sparse file will be # created, saving both disk space and I/O time. # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. - inode_size = inode.i_size - stream.truncate(inode_size) + try: + with open_method(filename) as f: + inode_size = inode.i_size + f.truncate(inode_size) - for page_idx, page_content in inode.get_contents(): - current_fp = page_idx * layer.page_size - max_length = inode_size - current_fp - page_bytes = page_content[:max_length] - if current_fp + len(page_bytes) > inode_size: - vollog.error( - "Page out of file bounds: inode 0x%x, inode size %d, page index %d", - inode.vol.offset, - inode_size, - page_idx, - ) - stream.seek(current_fp) - stream.write(page_bytes) + for page_idx, page_content in inode.get_contents(): + current_fp = page_idx * vmlinux_layer.page_size + max_length = inode_size - current_fp + page_bytes = page_content[:max_length] + if current_fp + len(page_bytes) > inode_size: + vollog.error( + "Page out of file bounds: inode 0x%x, inode size %d, page index %d", + inode.vol.offset, + inode_size, + page_idx, + ) + f.seek(current_fp) + f.write(page_bytes) + + except OSError as e: + vollog.error("Unable to write to file (%s): %s", filename, e) def _generator(self): vmlinux_module_name = self.config["kernel"] @@ -561,7 +528,7 @@ class InodePages(plugins.PluginInterface): filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") vollog.info("[*] Writing inode at 0x%x to '%s'", inode_address, filename) self.write_inode_content_to_file( - self.context, vmlinux_layer.name, inode, filename, open_method + inode, filename, open_method, vmlinux_layer ) def run(self): From bf76aad1e2367e2db4d561d9cc1cd76a42162420 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 20 Jan 2025 10:28:46 +1100 Subject: [PATCH 926/979] linux: page_cache.Files plugin: Ensure the inode's i_link pointer is readable --- volatility3/framework/plugins/linux/pagecache.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 39ed60486..f265241b6 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -147,7 +147,13 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): Otherwise, it returns the same symlink_path """ # i_link (fast symlinks) were introduced in 4.2 - if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: + if ( + inode + and inode.is_link + and inode.has_member("i_link") + and inode.i_link + and inode.i_link.is_readable() + ): i_link_str = inode.i_link.dereference().cast( "string", max_length=255, encoding="utf-8", errors="replace" ) From ec7a101eb92499741cafeb4dea1ab1ef1683a27c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 20 Jan 2025 11:05:09 +1100 Subject: [PATCH 927/979] linux: page_cache.InodePages plugin: Remove unnecesary casting --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index f265241b6..32b176b72 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -511,7 +511,7 @@ class InodePages(plugins.PluginInterface): page_vaddr = page_obj.vol.offset page_paddr = page_obj.to_paddr() page_mapping_addr = page_obj.mapping - page_index = int(page_obj.index) + page_index = page_obj.index page_file_offset = page_index * vmlinux_layer.page_size dump_safe = ( page_file_offset < inode_size From dfe3d255c064b9c78edf4f5f58eff6c15cc56486 Mon Sep 17 00:00:00 2001 From: Odysseas Stavrou Date: Mon, 20 Jan 2025 22:25:01 +0200 Subject: [PATCH 928/979] Volshell: Update Process retrieval methods with virtual/physical offsets --- volatility3/cli/volshell/linux.py | 58 +++++++++++++++++++++++++++++ volatility3/cli/volshell/windows.py | 47 +++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index cc58fa1c2..9ea3ea1f5 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -3,6 +3,7 @@ # from typing import Any, List, Optional, Tuple, Union +from enum import Enum from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -10,6 +11,16 @@ from volatility3.framework.configuration import requirements from volatility3.plugins.linux import pslist +# Could import the enum from psscan.py to avoid code duplication +class DescExitStateEnum(Enum): + """Enum for linux task exit_state as defined in include/linux/sched.h""" + + TASK_RUNNING = 0x00000000 + EXIT_DEAD = 0x00000010 + EXIT_ZOMBIE = 0x00000020 + EXIT_TRACE = EXIT_ZOMBIE | EXIT_DEAD + + class Volshell(generic.Volshell): """Shell environment to directly interact with a linux memory image.""" @@ -40,6 +51,52 @@ class Volshell(generic.Volshell): return None print(f"No task with task ID {pid} found") + def get_process(self, pid=None, offset=None): + """Get Task based on a process ID. Does not retrieve the layer, to change layer use the .pid attribute. The offset argument can be used both for physical or virtual offsets""" + + if pid is not None and offset is not None: + print("Only one parameter is accepted") + return None + + if offset is not None: + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] + + kernel_layer_name = vmlinux.layer_name + kernel_layer = self.context.layers[kernel_layer_name] + + memory_layer_name = kernel_layer.dependencies[0] + + ptask = self.context.object( + vmlinux.symbol_table_name + constants.BANG + "task_struct", + layer_name=memory_layer_name, + offset=offset, + native_layer_name=kernel_layer_name, + ) + + try: + DescExitStateEnum(ptask.exit_state) + except ValueError: + print( + f"task_struct @ {hex(ptask.vol.offset)} as exit_state {ptask.exit_state} is likely not valid" + ) + + if not (0 < ptask.pid < 65535): + print( + f"task_struct @ {hex(ptask.vol.offset)} as pid {ptask.pid} is likely not valid" + ) + + return ptask + + if pid is not None: + tasks = self.list_tasks() + for task in tasks: + if task.pid == pid: + return task + print(f"No task with task ID {pid} found") + + return None + def list_tasks(self): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols @@ -50,6 +107,7 @@ class Volshell(generic.Volshell): result += [ (["ct", "change_task", "cp"], self.change_task), (["lt", "list_tasks", "ps"], self.list_tasks), + (["gp", "get_process"], self.get_process), (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get("pid", None) is not None: diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 303d4d5c3..a77392561 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -44,11 +44,58 @@ class Volshell(generic.Volshell): ) ) + def get_process(self, pid=None, v_offset=None, p_offset=None): + """Returns the EPROCESS object that matches the pid. If v_offset/p_offset is provided, construct the EPROCESS object at the provided address. Only one parameter is allowed.""" + + if sum(1 if x is not None else 0 for x in [pid, v_offset, p_offset]) != 1: + print("Only one parameter is accepted") + return None + + kernel_name = self.config["kernel"] + kernel = self.context.modules[kernel_name] + + kernel_layer_name = kernel.layer_name + + kernel_layer = self.context.layers[kernel_layer_name] + memory_layer_name = kernel_layer.dependencies[0] + + eprocess_symbol = kernel.symbol_table_name + constants.BANG + "_EPROCESS" + + if v_offset is not None: + eproc = self.context.object( + eprocess_symbol, + layer_name=kernel_layer_name, + offset=v_offset, + ) + + return eproc + + if p_offset is not None: + eproc = self.context.object( + eprocess_symbol, + layer_name=memory_layer_name, + offset=p_offset, + native_layer_name=kernel_layer_name, + ) + + return eproc + + if pid is not None: + processes = self.list_processes() + for process in processes: + if process.UniqueProcessId == pid: + return process + print(f"No process with process ID {pid} found") + return None + + return None + def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (["cp", "change_process"], self.change_process), (["lp", "list_processes", "ps"], self.list_processes), + (["gp", "get_process"], self.get_process), (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get("pid", None) is not None: From b59f051353cd58b7d6e4bfda2f07746820f4f32a Mon Sep 17 00:00:00 2001 From: Odysseas Stavrou Date: Wed, 22 Jan 2025 02:39:35 +0200 Subject: [PATCH 929/979] Volshell: Updates to the get_process() methods --- volatility3/cli/volshell/linux.py | 55 +++++++++++++++++++---------- volatility3/cli/volshell/windows.py | 23 ++++++++---- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 9ea3ea1f5..b3689c3ae 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -51,42 +51,61 @@ class Volshell(generic.Volshell): return None print(f"No task with task ID {pid} found") - def get_process(self, pid=None, offset=None): - """Get Task based on a process ID. Does not retrieve the layer, to change layer use the .pid attribute. The offset argument can be used both for physical or virtual offsets""" + def get_process(self, pid=None, virtaddr=None, physaddr=None): + """Return the task_struct object that matches the pid. If a physical or a virtual address is provided, construct the task_struct object at said address. Only one parameter is allowed. - if pid is not None and offset is not None: + Args: + pid (int, optional): PID to search for + virtaddr (int, optional): Virtual address to construct object at + physaddr (int, optional): Physical address to construct object at + + Returns: + ObjectInterface: task_struct Object + """ + + if sum(1 if x is not None else 0 for x in [pid, virtaddr, physaddr]) != 1: print("Only one parameter is accepted") return None - if offset is not None: - vmlinux_module_name = self.config["kernel"] - vmlinux = self.context.modules[vmlinux_module_name] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] - kernel_layer_name = vmlinux.layer_name - kernel_layer = self.context.layers[kernel_layer_name] + kernel_layer_name = vmlinux.layer_name + kernel_layer = self.context.layers[kernel_layer_name] - memory_layer_name = kernel_layer.dependencies[0] + memory_layer_name = kernel_layer.dependencies[0] - ptask = self.context.object( - vmlinux.symbol_table_name + constants.BANG + "task_struct", + task_struct_symbol = vmlinux.symbol_table_name + constants.BANG + "task_struct" + + if virtaddr is not None: + task = self.context.object( + task_struct_symbol, + layer_name=kernel_layer_name, + offset=virtaddr, + ) + + if physaddr is not None: + task = self.context.object( + task_struct_symbol, layer_name=memory_layer_name, - offset=offset, + offset=physaddr, native_layer_name=kernel_layer_name, ) + if physaddr is not None or virtaddr is not None: try: - DescExitStateEnum(ptask.exit_state) + DescExitStateEnum(task.exit_state) except ValueError: print( - f"task_struct @ {hex(ptask.vol.offset)} as exit_state {ptask.exit_state} is likely not valid" + f"task_struct @ {hex(task.vol.offset)} as exit_state {task.exit_state} is likely not valid" ) - if not (0 < ptask.pid < 65535): + if not (0 < task.pid < 65535): print( - f"task_struct @ {hex(ptask.vol.offset)} as pid {ptask.pid} is likely not valid" + f"task_struct @ {hex(task.vol.offset)} as pid {task.pid} is likely not valid" ) - return ptask + return task if pid is not None: tasks = self.list_tasks() @@ -107,7 +126,7 @@ class Volshell(generic.Volshell): result += [ (["ct", "change_task", "cp"], self.change_task), (["lt", "list_tasks", "ps"], self.list_tasks), - (["gp", "get_process"], self.get_process), + (["gp", "get_process", "get_task"], self.get_process), (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get("pid", None) is not None: diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index a77392561..9b89a8b81 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -44,10 +44,19 @@ class Volshell(generic.Volshell): ) ) - def get_process(self, pid=None, v_offset=None, p_offset=None): - """Returns the EPROCESS object that matches the pid. If v_offset/p_offset is provided, construct the EPROCESS object at the provided address. Only one parameter is allowed.""" + def get_process(self, pid=None, virtaddr=None, physaddr=None): + """Returns the _EPROCESS object that matches the pid. If a physical or a virtual address is provided, construct the _EPROCESS object at said address. Only one parameter is allowed. - if sum(1 if x is not None else 0 for x in [pid, v_offset, p_offset]) != 1: + Args: + pid (int, optional): PID / UniqueProcessId to search for. + virtaddr (int, optional): Virtual address to construct object at + physaddr (int, optional): Physical address to construct object at + + Returns: + ObjectInterface: _EPROCESS Object + """ + + if sum(1 if x is not None else 0 for x in [pid, virtaddr, physaddr]) != 1: print("Only one parameter is accepted") return None @@ -61,20 +70,20 @@ class Volshell(generic.Volshell): eprocess_symbol = kernel.symbol_table_name + constants.BANG + "_EPROCESS" - if v_offset is not None: + if virtaddr is not None: eproc = self.context.object( eprocess_symbol, layer_name=kernel_layer_name, - offset=v_offset, + offset=virtaddr, ) return eproc - if p_offset is not None: + if physaddr is not None: eproc = self.context.object( eprocess_symbol, layer_name=memory_layer_name, - offset=p_offset, + offset=physaddr, native_layer_name=kernel_layer_name, ) From c10572905f3f3760594db07575534a5805a10fe3 Mon Sep 17 00:00:00 2001 From: Daniel Davidov <35842733+Danking555@users.noreply.github.com> Date: Wed, 22 Jan 2025 10:45:52 +0200 Subject: [PATCH 930/979] Add low stub offset kernel detection reference: Memprocfs and https://www.youtube.com/watch?v=_ShCSth6dWM --- volatility3/framework/automagic/pdbscan.py | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 729c48063..1ccecf97a 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -11,6 +11,7 @@ import contextlib import logging import math import os +import struct from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, layers @@ -376,8 +377,43 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): valid_kernel = (virtual_layer_name, address, res[0]) return valid_kernel + def method_low_stub_offset(self, + context: interfaces.context.ContextInterface, + vlayer: layers.intel.Intel, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[ValidKernelType]: + kernel_hint = 0 + kernel_base = 0 + physical_layer = context.layers.get('memory_layer') + + # try locating kernel base via x64 Low Stub in lower 1MB starting from second page (4KB) + # if "Discard Low Memory" setting is disabled in BIOS, the Low Stub may be at the third/fourth or further pages + for offset in range(0x1000,0x100000, 0x1000): + if 0xffffffffffff00ff & int.from_bytes(physical_layer.read(offset, 0x8), "little") != 0x00000001000600E9: + continue # not _PROCESSOR_START_BLOCK->Jmp + potential_kernel_hint = int.from_bytes(physical_layer.read(offset + 0x70, 0x8), "little") + if (0xfffff80000000003 & potential_kernel_hint) != 0xfffff80000000000: + continue # not _PROCESSOR_START_BLOCK->LmTarget + kernel_hint = potential_kernel_hint & 0xffffffffffff + kernel_base = kernel_hint & (~0x1fffff) & 0xffffffffffff + break + + if kernel_base: + # Scanning 32mb in 2mb chunks for the 'ntoskrnl' base address + while (kernel_base + 0x2000000) > kernel_hint: + for i in range(0, 0x200000, 0x1000): + valid_kernel = self.check_kernel_offset( + context, vlayer, kernel_base, progress_callback + ) + if valid_kernel: + return valid_kernel + kernel_base -= 0x200000 + + return None + # List of methods to be run, in order, to determine the valid kernels methods = [ + method_low_stub_offset, method_kdbg_offset, method_module_offset, method_fixed_mapping, From 34a7dfc72fb3dced313e9a52b73b96dff31b778a Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 23 Jan 2025 10:42:52 +0100 Subject: [PATCH 931/979] split linux modules utilities --- .../symbols/linux/utilities/modules.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 volatility3/framework/symbols/linux/utilities/modules.py diff --git a/volatility3/framework/symbols/linux/utilities/modules.py b/volatility3/framework/symbols/linux/utilities/modules.py new file mode 100644 index 000000000..ac9b2afaf --- /dev/null +++ b/volatility3/framework/symbols/linux/utilities/modules.py @@ -0,0 +1,68 @@ +from typing import Iterator, List, Tuple + +from volatility3 import framework +from volatility3.framework import constants, interfaces +from volatility3.framework.objects import utility + + +class Modules(interfaces.configuration.VersionableInterface): + """Kernel modules related utilities.""" + + _version = (1, 0, 0) + _required_framework_version = (2, 0, 0) + + framework.require_interface_version(*_required_framework_version) + + @staticmethod + def mask_mods_list( + context: interfaces.context.ContextInterface, + layer_name: str, + mods: Iterator[interfaces.objects.ObjectInterface], + ) -> List[Tuple[str, int, int]]: + """ + A helper function to mask the starting and end address of kernel modules + """ + mask = context.layers[layer_name].address_mask + + return [ + ( + utility.array_to_string(mod.name), + mod.get_module_base() & mask, + (mod.get_module_base() & mask) + mod.get_core_size(), + ) + for mod in mods + ] + + @staticmethod + def lookup_module_address( + context: interfaces.context.ContextInterface, + kernel_module_name: str, + handlers: List[Tuple[str, int, int]], + target_address: int, + ) -> Tuple[str, str]: + """ + Searches between the start and end address of the kernel module using target_address. + Returns the module and symbol name of the address provided. + """ + kernel_module = context.modules[kernel_module_name] + mod_name = "UNKNOWN" + symbol_name = "N/A" + + for name, start, end in handlers: + if start <= target_address <= end: + mod_name = name + if name == constants.linux.KERNEL_NAME: + symbols = list( + kernel_module.get_symbols_by_absolute_location(target_address) + ) + + if len(symbols): + symbol_name = ( + symbols[0].split(constants.BANG)[1] + if constants.BANG in symbols[0] + else symbols[0] + ) + + break + + return mod_name, symbol_name From 43ab0b0c314832742c00f7821cd6f3327529894e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 23 Jan 2025 10:43:14 +0100 Subject: [PATCH 932/979] add deprecation decorator --- .../framework/configuration/requirements.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 3e3608000..3af5601dc 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -11,6 +11,7 @@ expect to be in the context (such as particular layers or symboltables). import abc import logging import os +import functools from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, Type from urllib import parse, request @@ -723,3 +724,25 @@ class ModuleRequirement( """Builds the appropriate configuration for the specified requirement.""" return context.modules[value].build_configuration() + + +def deprecated_method(replacement: str, additional_information: str = ""): + """A decorator for marking functions as deprecated. + + Args: + replacement: The replacement function overriding the deprecated API (full path preferred, starting from "volatility3."). String was preferred, for convenience and to prevent import conflicts on caller side. + additional_information: Information appended at the end of the deprecation message + """ + + def decorator(deprecated_func): + @functools.wraps(deprecated_func) + def wrapper(*args, **kwargs): + nonlocal replacement, additional_information + deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__name__}\" is deprecated, use \"{replacement}\" instead. {additional_information}" + vollog.warning(deprecation_msg) + # Return the wrapped function with its original arguments + return deprecated_func(*args, **kwargs) + + return wrapper + + return decorator From 7575aa5d6354419b51d1ac563c1de4db62da0ca1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 23 Jan 2025 10:44:20 +0100 Subject: [PATCH 933/979] deprecate lookup_module_address and mask_mods_list --- .../framework/symbols/linux/__init__.py | 101 ++++++++---------- 1 file changed, 44 insertions(+), 57 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 423284b03..3dc744f78 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -8,11 +8,13 @@ import logging from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions +from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -81,7 +83,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 2, 0) + _version = (2, 2, 1) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -338,27 +340,6 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): yield fd_num, filp, full_path - @classmethod - def mask_mods_list( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - mods: Iterator[interfaces.objects.ObjectInterface], - ) -> List[Tuple[str, int, int]]: - """ - A helper function to mask the starting and end address of kernel modules - """ - mask = context.layers[layer_name].address_mask - - return [ - ( - utility.array_to_string(mod.name), - mod.get_module_base() & mask, - (mod.get_module_base() & mask) + mod.get_core_size(), - ) - for mod in mods - ] - @classmethod def generate_kernel_handler_info( cls, @@ -382,41 +363,9 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return [ (constants.linux.KERNEL_NAME, start_addr, end_addr) - ] + LinuxUtilities.mask_mods_list(context, kernel.layer_name, mods_list) - - @classmethod - def lookup_module_address( - cls, - kernel_module: interfaces.context.ModuleInterface, - handlers: List[Tuple[str, int, int]], - target_address: int, - ): - """ - Searches between the start and end address of the kernel module using target_address. - Returns the module and symbol name of the address provided. - """ - - mod_name = "UNKNOWN" - symbol_name = "N/A" - - for name, start, end in handlers: - if start <= target_address <= end: - mod_name = name - if name == constants.linux.KERNEL_NAME: - symbols = list( - kernel_module.get_symbols_by_absolute_location(target_address) - ) - - if len(symbols): - symbol_name = ( - symbols[0].split(constants.BANG)[1] - if constants.BANG in symbols[0] - else symbols[0] - ) - - break - - return mod_name, symbol_name + ] + linux_utilities_modules.Modules.mask_mods_list( + context, kernel.layer_name, mods_list + ) @classmethod def walk_internal_list(cls, vmlinux, struct_name, list_member, list_start): @@ -504,6 +453,44 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): [chr((code >> (i * 8)) & 0xFF) for i in range(code_bytes_length)] ) + ## Deprecated APIs ## + @classmethod + @requirements.deprecated_method( + replacement="volatility3.framework.symbols.linux.utilities.modules.Modules.mask_mods_list" + ) + def mask_mods_list( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + mods: Iterator[interfaces.objects.ObjectInterface], + ) -> List[Tuple[str, int, int]]: + """ + DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.mask_mods_list" instead. + + A helper function to mask the starting and end address of kernel modules + """ + return linux_utilities_modules.Modules.mask_mods_list(context, layer_name, mods) + + @classmethod + @requirements.deprecated_method( + replacement="volatility3.framework.symbols.linux.utilities.modules.Modules.lookup_module_address" + ) + def lookup_module_address( + cls, + kernel_module: interfaces.context.ModuleInterface, + handlers: List[Tuple[str, int, int]], + target_address: int, + ) -> Tuple[str, str]: + """ + DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.lookup_module_address" instead. + + Searches between the start and end address of the kernel module using target_address. + Returns the module and symbol name of the address provided. + """ + return linux_utilities_modules.Modules.lookup_module_address( + kernel_module.context, kernel_module.name, handlers, target_address + ) + class IDStorage(ABC): """Abstraction to support both XArray and RadixTree""" From fd77c041537b40f38db7428ce05a876ad5b1e08b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 23 Jan 2025 10:46:42 +0100 Subject: [PATCH 934/979] move to linux_utilities_modules APIs --- volatility3/framework/plugins/linux/check_idt.py | 7 +++++-- .../framework/plugins/linux/keyboard_notifiers.py | 7 +++++-- volatility3/framework/plugins/linux/kthreads.py | 9 ++++++--- volatility3/framework/plugins/linux/netfilter.py | 7 +++++-- volatility3/framework/plugins/linux/tty_check.py | 7 +++++-- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_idt.py b/volatility3/framework/plugins/linux/check_idt.py index 07582e2c1..5859e73d6 100644 --- a/volatility3/framework/plugins/linux/check_idt.py +++ b/volatility3/framework/plugins/linux/check_idt.py @@ -5,6 +5,7 @@ import logging from typing import List +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3.framework import interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -99,8 +100,10 @@ class Check_idt(interfaces.plugins.PluginInterface): idt_addr = idt_addr & address_mask - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( - vmlinux, handlers, idt_addr + module_name, symbol_name = ( + linux_utilities_modules.Modules.lookup_module_address( + self.context, vmlinux.name, handlers, idt_addr + ) ) yield ( diff --git a/volatility3/framework/plugins/linux/keyboard_notifiers.py b/volatility3/framework/plugins/linux/keyboard_notifiers.py index 72273a77b..c1b7572c6 100644 --- a/volatility3/framework/plugins/linux/keyboard_notifiers.py +++ b/volatility3/framework/plugins/linux/keyboard_notifiers.py @@ -4,6 +4,7 @@ import logging +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -66,8 +67,10 @@ class Keyboard_notifiers(interfaces.plugins.PluginInterface): ): call_addr = call_back.notifier_call - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( - vmlinux, handlers, call_addr + module_name, symbol_name = ( + linux_utilities_modules.Modules.lookup_module_address( + self.context, vmlinux.name, handlers, call_addr + ) ) yield (0, [format_hints.Hex(call_addr), module_name, symbol_name]) diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index 40e992069..2e1bbed47 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -4,6 +4,7 @@ import logging from typing import List +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins @@ -20,7 +21,7 @@ class Kthreads(plugins.PluginInterface): """Enumerates kthread functions""" _required_framework_version = (2, 11, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -88,8 +89,10 @@ class Kthreads(plugins.PluginInterface): if kthread.has_member("full_name") else task_name ) - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( - vmlinux, handlers, threadfn + module_name, symbol_name = ( + linux_utilities_modules.Modules.lookup_module_address( + self.context, vmlinux.name, handlers, threadfn + ) ) fields = [ diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 73496dfd9..ccb831b61 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field from abc import ABC, abstractmethod import logging +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from typing import Iterator, List, Tuple from volatility3 import framework from volatility3.framework import ( @@ -263,8 +264,10 @@ class AbstractNetfilter(ABC): """Helper to obtain the module and symbol name in the format needed for the output of this plugin. """ - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( - self.vmlinux, self.handlers, addr + module_name, symbol_name = ( + linux_utilities_modules.Modules.lookup_module_address( + self._context, self.vmlinux.name, self.handlers, addr + ) ) if module_name == "UNKNOWN": diff --git a/volatility3/framework/plugins/linux/tty_check.py b/volatility3/framework/plugins/linux/tty_check.py index 45238ef8c..f375968a4 100644 --- a/volatility3/framework/plugins/linux/tty_check.py +++ b/volatility3/framework/plugins/linux/tty_check.py @@ -5,6 +5,7 @@ import logging from typing import List +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins @@ -79,8 +80,10 @@ class tty_check(plugins.PluginInterface): recv_buf = tty_dev.ldisc.ops.receive_buf - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( - vmlinux, handlers, recv_buf + module_name, symbol_name = ( + linux_utilities_modules.Modules.lookup_module_address( + self.context, vmlinux.name, handlers, recv_buf + ) ) yield (0, (name, format_hints.Hex(recv_buf), module_name, symbol_name)) From b8d9c7b88311016cea97b8b60afeec4d47558af0 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 23 Jan 2025 11:57:43 -0600 Subject: [PATCH 935/979] #1473 - add missing exception handling for get_key --- volatility3/framework/plugins/windows/envars.py | 12 ++++++------ .../framework/plugins/windows/getservicesids.py | 5 +++-- .../framework/plugins/windows/getsids.py | 2 +- .../plugins/windows/registry/userassist.py | 17 ++++++++++++----- .../framework/plugins/windows/svcscan.py | 6 +++--- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index cac4ecf40..48e1ef671 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -76,14 +76,14 @@ class Envars(interfaces.plugins.PluginInterface): "CurrentControlSet\\Control\\Session Manager\\Environment" ) sys = True - except KeyError: - with contextlib.suppress(KeyError): + except (KeyError, registry.RegistryFormatException): + with contextlib.suppress(KeyError, registry.RegistryFormatException): key = hive.get_key( "ControlSet001\\Control\\Session Manager\\Environment" ) sys = True if sys: - with contextlib.suppress(KeyError): + with contextlib.suppress(KeyError, registry.RegistryFormatException): for node in key.get_values(): try: value_node_name = node.get_name() @@ -100,11 +100,11 @@ class Envars(interfaces.plugins.PluginInterface): continue ## The user-specific variables - with contextlib.suppress(KeyError): + with contextlib.suppress(KeyError, registry.RegistryFormatException): key = hive.get_key("Environment") ntuser = True if ntuser: - with contextlib.suppress(KeyError): + with contextlib.suppress(KeyError, registry.RegistryFormatException): for node in key.get_values(): try: value_node_name = node.get_name() @@ -123,7 +123,7 @@ class Envars(interfaces.plugins.PluginInterface): ## The volatile user variables try: key = hive.get_key("Volatile Environment") - except KeyError: + except (KeyError, registry.RegistryFormatException): continue try: for node in key.get_values(): diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index eece7fb6c..b97d2bb46 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -10,6 +10,7 @@ from typing import List from volatility3.framework import renderers, interfaces, constants, exceptions from volatility3.framework.configuration import requirements +from volatility3.framework.layers import registry from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) @@ -86,10 +87,10 @@ class GetServiceSIDs(interfaces.plugins.PluginInterface): # Get ControlSet\Services. try: services = hive.get_key(r"CurrentControlSet\Services") - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): try: services = hive.get_key(r"ControlSet001\Services") - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): continue if services: diff --git a/volatility3/framework/plugins/windows/getsids.py b/volatility3/framework/plugins/windows/getsids.py index df0c7a835..00c78e1cf 100644 --- a/volatility3/framework/plugins/windows/getsids.py +++ b/volatility3/framework/plugins/windows/getsids.py @@ -158,7 +158,7 @@ class GetSIDs(interfaces.plugins.PluginInterface): layers.registry.RegistryFormatException, ): continue - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, layers.registry.RegistryFormatException): continue return sids diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 932ee9d6f..646fb1d7f 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -13,7 +13,7 @@ from typing import Any, Generator, List, Tuple from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers.physical import BufferDataLayer -from volatility3.framework.layers.registry import RegistryHive +from volatility3.framework.layers.registry import RegistryHive, RegistryFormatException from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.plugins.windows.registry import hivelist @@ -167,10 +167,17 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac self._determine_userassist_type() - userassist_node_path = hive.get_key( - "software\\microsoft\\windows\\currentversion\\explorer\\userassist", - return_list=True, - ) + try: + userassist_node_path = hive.get_key( + "software\\microsoft\\windows\\currentversion\\explorer\\userassist", + return_list=True, + ) + except RegistryFormatException as e: + vollog.warning(f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}") + return None + except KeyError: + vollog.warning(f"UserAssist key not found in {hive_name} at {hive.hive_offset:#x}") + return None if not userassist_node_path: vollog.warning("list_userassist did not find a valid node_path (or None)") diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index bd477ba27..93087f352 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -15,7 +15,7 @@ from volatility3.framework import ( symbols, ) from volatility3.framework.configuration import requirements -from volatility3.framework.layers import scanners +from volatility3.framework.layers import scanners, registry from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions @@ -159,12 +159,12 @@ class SvcScan(interfaces.plugins.PluginInterface): return cast( objects.StructType, hive.get_key(r"CurrentControlSet\Services") ) - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): try: return cast( objects.StructType, hive.get_key(r"ControlSet001\Services") ) - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): vollog.log( constants.LOGLEVEL_VVVV, "Could not retrieve any control set from SYSTEM hive", From 6c4cafa64f68e6b001cd1ed32e8e5fb3d9993f30 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 23 Jan 2025 11:59:37 -0600 Subject: [PATCH 936/979] #1473 - black fixes --- .../framework/plugins/windows/getservicesids.py | 12 ++++++++++-- volatility3/framework/plugins/windows/getsids.py | 6 +++++- .../framework/plugins/windows/registry/userassist.py | 8 ++++++-- volatility3/framework/plugins/windows/svcscan.py | 12 ++++++++++-- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index b97d2bb46..207d0e2ad 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -87,10 +87,18 @@ class GetServiceSIDs(interfaces.plugins.PluginInterface): # Get ControlSet\Services. try: services = hive.get_key(r"CurrentControlSet\Services") - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): try: services = hive.get_key(r"ControlSet001\Services") - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): continue if services: diff --git a/volatility3/framework/plugins/windows/getsids.py b/volatility3/framework/plugins/windows/getsids.py index 00c78e1cf..a75bbe7ea 100644 --- a/volatility3/framework/plugins/windows/getsids.py +++ b/volatility3/framework/plugins/windows/getsids.py @@ -158,7 +158,11 @@ class GetSIDs(interfaces.plugins.PluginInterface): layers.registry.RegistryFormatException, ): continue - except (KeyError, exceptions.InvalidAddressException, layers.registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + layers.registry.RegistryFormatException, + ): continue return sids diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 646fb1d7f..d50b5216e 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -173,10 +173,14 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac return_list=True, ) except RegistryFormatException as e: - vollog.warning(f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}") + vollog.warning( + f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}" + ) return None except KeyError: - vollog.warning(f"UserAssist key not found in {hive_name} at {hive.hive_offset:#x}") + vollog.warning( + f"UserAssist key not found in {hive_name} at {hive.hive_offset:#x}" + ) return None if not userassist_node_path: diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 93087f352..17baac5b0 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -159,12 +159,20 @@ class SvcScan(interfaces.plugins.PluginInterface): return cast( objects.StructType, hive.get_key(r"CurrentControlSet\Services") ) - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): try: return cast( objects.StructType, hive.get_key(r"ControlSet001\Services") ) - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): vollog.log( constants.LOGLEVEL_VVVV, "Could not retrieve any control set from SYSTEM hive", From 81ba89eef663727608886e66595dfd7b3dcd9831 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 23 Jan 2025 12:20:14 -0600 Subject: [PATCH 937/979] #1473 - update exception message --- volatility3/framework/plugins/windows/registry/userassist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index d50b5216e..87016553a 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -174,7 +174,7 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac ) except RegistryFormatException as e: vollog.warning( - f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}" + f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}: {e}" ) return None except KeyError: From a68be50798254cbadc490393721e74180b4117cc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 23 Jan 2025 20:30:21 +0000 Subject: [PATCH 938/979] Revert "Typing fix" This reverts commit c82d432b10258136ff0777dfec1fbf5844316132. --- volatility3/framework/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index a1925faef..754939460 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,6 +5,7 @@ # Check the python version to ensure it's suitable import glob import sys +from volatility3.framework import check_python_version as check_python_version import zipfile import importlib import inspect @@ -57,7 +58,7 @@ class NonInheritable: self.default_value = value self.cls = cls - def __get__(self, obj: Any, get_type: Optional[Type] = None) -> Any: + def __get__(self, obj: Any, get_type: Type = Optional[None]) -> Any: if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) From 61f60ac464ba01885216704faf37ed6b6fc4beb6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 23 Jan 2025 20:52:08 +0000 Subject: [PATCH 939/979] Core: Move the python check somewhere it can't accidentally be removed --- volatility3/framework/__init__.py | 12 +++++++++++- volatility3/framework/check_python_version.py | 14 -------------- volatility3/framework/constants/__init__.py | 2 ++ 3 files changed, 13 insertions(+), 15 deletions(-) delete mode 100644 volatility3/framework/check_python_version.py diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 754939460..466e697bb 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,7 +5,6 @@ # Check the python version to ensure it's suitable import glob import sys -from volatility3.framework import check_python_version as check_python_version import zipfile import importlib import inspect @@ -16,6 +15,17 @@ from typing import Any, Dict, Generator, List, Optional, Tuple, Type, TypeVar from volatility3.framework import constants, interfaces +if ( + sys.version_info.major != constants.REQUIRED_PYTHON_VERSION[0] + or sys.version_info.minor < constants.REQUIRED_PYTHON_VERSION[1] + or ( + sys.version_info.minor == constants.REQUIRED_PYTHON_VERSION[1] + and sys.version_info.micro < constants.REQUIRED_PYTHON_VERSION[2] + ) +): + raise RuntimeError( + f"Volatility framework requires python version {".".join(str(x) for x in constants.REQUIRED_PYTHON_VERSION)} or greater" + ) # ## # diff --git a/volatility3/framework/check_python_version.py b/volatility3/framework/check_python_version.py deleted file mode 100644 index f2d284f2a..000000000 --- a/volatility3/framework/check_python_version.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys - -required_python_version = (3, 8, 0) -if ( - sys.version_info.major != required_python_version[0] - or sys.version_info.minor < required_python_version[1] - or ( - sys.version_info.minor == required_python_version[1] - and sys.version_info.micro < required_python_version[2] - ) -): - raise RuntimeError( - f"Volatility framework requires python version {required_python_version[0]}.{required_python_version[1]}.{required_python_version[2]} or greater" - ) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 23cc2dde5..2e6ae0261 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -23,6 +23,8 @@ from volatility3.framework.constants._version import ( VERSION_SUFFIX as VERSION_SUFFIX, ) +REQUIRED_PYTHON_VERSION = (3, 8, 0) + PLUGINS_PATH = [ os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "plugins")), os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "plugins")), From 128e1be154cc5a9853da3413565685689bf702e5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 23 Jan 2025 20:57:44 +0000 Subject: [PATCH 940/979] Core: Fix f-string quotes --- volatility3/framework/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 466e697bb..0bbdefa43 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -24,7 +24,7 @@ if ( ) ): raise RuntimeError( - f"Volatility framework requires python version {".".join(str(x) for x in constants.REQUIRED_PYTHON_VERSION)} or greater" + f"Volatility framework requires python version {'.'.join(str(x) for x in constants.REQUIRED_PYTHON_VERSION)} or greater" ) # ## From 17d52c27bfc31ad9ecad7fb0b8604586039ce2b0 Mon Sep 17 00:00:00 2001 From: Daniel Davidov <35842733+Danking555@users.noreply.github.com> Date: Fri, 24 Jan 2025 21:09:28 +0200 Subject: [PATCH 941/979] Update method_low_stub_offset & run ruff & black * Eliminate unnecessary scanning for 32 bit processors where the structure PROCESSOR_START_BLOCK doesn't exist * Put offsets as values of constants in a class - LowStubLayout. * Add documentation in the class and in the function method_low_stub_offset * Run "ruff check --fix" and "black ." * Checked the method works on 3 physical machines --- volatility3/framework/automagic/pdbscan.py | 74 ++++++++++++++++++---- 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 1ccecf97a..7d289bcb6 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -11,7 +11,6 @@ import contextlib import logging import math import os -import struct from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, layers @@ -377,25 +376,73 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): valid_kernel = (virtual_layer_name, address, res[0]) return valid_kernel - def method_low_stub_offset(self, + class LowStubLayout: + """ + Represents the layout of the Low Stub which exists only on x64 machines with no virtualization/emulation, + responsible for transitioning from Real Mode(16 bit) to Protected Mode(32 bit) and Long Mode(64 bit) on boot/return from sleep. + Contains offsets to fields and structures within the undocumented structure _PROCESSOR_START_BLOCK. + Here's a reference: https://github.com/mic101/windows/blob/master/WRK-v1.2/base/ntos/inc/amd64.h#L3334 + """ + + # Expected signature for validation, constructed from: + # PROCESSOR_START_BLOCK->Jmp->OpCode | PROCESSOR_START_BLOCK->Jmp->Offset | PROCESSOR_START_BLOCK->CompletionFlag + JMP_AND_COMPLETION_SIGNATURE = 0x00000001000600E9 + + # Address of LmTarget (Long Mode target) + PROCESSOR_START_BLOCK_LM_TARGET_OFFSET = ( + 0x70 # PROCESSOR_START_BLOCK->LmTarget, PVOID 8 bytes + ) + + # CR3 register within structures describing initial processor state to be started + PROCESSOR_START_BLOCK_CR3_OFFSET = 0xA0 # PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3, ULONG64 8 bytes + + def method_low_stub_offset( + self, context: interfaces.context.ContextInterface, vlayer: layers.intel.Intel, progress_callback: constants.ProgressCallback = None, ) -> Optional[ValidKernelType]: + # This method is only valid for x64 systems + if not isinstance(vlayer, intel.Intel32e): + return None kernel_hint = 0 kernel_base = 0 - physical_layer = context.layers.get('memory_layer') + physical_layer = context.layers.get("memory_layer") - # try locating kernel base via x64 Low Stub in lower 1MB starting from second page (4KB) - # if "Discard Low Memory" setting is disabled in BIOS, the Low Stub may be at the third/fourth or further pages - for offset in range(0x1000,0x100000, 0x1000): - if 0xffffffffffff00ff & int.from_bytes(physical_layer.read(offset, 0x8), "little") != 0x00000001000600E9: - continue # not _PROCESSOR_START_BLOCK->Jmp - potential_kernel_hint = int.from_bytes(physical_layer.read(offset + 0x70, 0x8), "little") - if (0xfffff80000000003 & potential_kernel_hint) != 0xfffff80000000000: - continue # not _PROCESSOR_START_BLOCK->LmTarget - kernel_hint = potential_kernel_hint & 0xffffffffffff - kernel_base = kernel_hint & (~0x1fffff) & 0xffffffffffff + # Try locating kernel base via x64 Low Stub in lower 1MB starting from second page (4KB) + # If "Discard Low Memory" setting is disabled in BIOS, the Low Stub may be at the third/fourth or further pages + for offset in range(0x1000, 0x100000, 0x1000): + jmp_and_completion_values = int.from_bytes( + physical_layer.read(offset, 0x8), "little" + ) + if ( + 0xFFFFFFFFFFFF00FF & jmp_and_completion_values + != self.LowStubLayout.JMP_AND_COMPLETION_SIGNATURE + ): + continue + cr3_value = int.from_bytes( + physical_layer.read( + offset + self.LowStubLayout.PROCESSOR_START_BLOCK_CR3_OFFSET, 0x8 + ), + "little", + ) + + # Compare previously observed valid page table address that's stored in vlayer._initial_entry + # with PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3 + # which was observed to be an invalid page address, so add 1 (to make it valid too) + if (cr3_value + 1) != vlayer._initial_entry: + continue + potential_kernel_hint = int.from_bytes( + physical_layer.read( + offset + self.LowStubLayout.PROCESSOR_START_BLOCK_LM_TARGET_OFFSET, + 0x8, + ), + "little", + ) + if 0x3 & potential_kernel_hint: + continue + kernel_hint = potential_kernel_hint & 0xFFFFFFFFFFFF + kernel_base = kernel_hint & (~0x1FFFFF) & 0xFFFFFFFFFFFF break if kernel_base: @@ -408,7 +455,6 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): if valid_kernel: return valid_kernel kernel_base -= 0x200000 - return None # List of methods to be run, in order, to determine the valid kernels From e9088be0d86fa2f68774d47371ebd2736287f1c5 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 28 Dec 2024 22:46:39 +0000 Subject: [PATCH 942/979] Prevent infinite looping and out of memory errors #1482 --- .../framework/symbols/windows/extensions/registry.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index c9544a8ba..b282b13cf 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -133,8 +133,17 @@ class CM_KEY_BODY(objects.StructType): def get_full_key_name(self) -> str: output = [] + seen = set() + kcb = self.KeyControlBlock while kcb.ParentKcb: + if kcb.ParentKcb.vol.offset in seen: + return "" + seen.add(kcb.ParentKcb.vol.offset) + + if len(output) > 128: + return "" + if kcb.NameBlock.Name is None: break From 506a61d8846e6a3399ab1f964d41b09a197592a6 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 24 Jan 2025 22:09:24 +0000 Subject: [PATCH 943/979] Address feedback --- volatility3/framework/plugins/windows/handles.py | 4 ++-- volatility3/framework/symbols/windows/extensions/registry.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 38ccfbfbc..6a391fe35 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -341,7 +341,7 @@ class Handles(interfaces.plugins.PluginInterface): try: obj_name = entry.NameInfo.Name.String except (ValueError, exceptions.InvalidAddressException): - obj_name = "" + obj_name = None except exceptions.InvalidAddressException: vollog.log( @@ -359,7 +359,7 @@ class Handles(interfaces.plugins.PluginInterface): format_hints.Hex(entry.HandleValue), obj_type, format_hints.Hex(entry.GrantedAccess), - obj_name, + obj_name or renderers.NotAvailableValue(), ), ) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index b282b13cf..a8cc7703c 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -138,11 +138,11 @@ class CM_KEY_BODY(objects.StructType): kcb = self.KeyControlBlock while kcb.ParentKcb: if kcb.ParentKcb.vol.offset in seen: - return "" + return None seen.add(kcb.ParentKcb.vol.offset) if len(output) > 128: - return "" + return None if kcb.NameBlock.Name is None: break From 8b0165b6f6ea7a6ebb70ddb01b2de29b627ddcfd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 14:45:21 +0100 Subject: [PATCH 944/979] add linux_utilities_modules requirement --- volatility3/framework/plugins/linux/check_idt.py | 5 +++++ volatility3/framework/plugins/linux/keyboard_notifiers.py | 5 +++++ volatility3/framework/plugins/linux/kthreads.py | 5 +++++ volatility3/framework/plugins/linux/netfilter.py | 5 +++++ volatility3/framework/plugins/linux/tty_check.py | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/volatility3/framework/plugins/linux/check_idt.py b/volatility3/framework/plugins/linux/check_idt.py index 5859e73d6..ffb707af5 100644 --- a/volatility3/framework/plugins/linux/check_idt.py +++ b/volatility3/framework/plugins/linux/check_idt.py @@ -28,6 +28,11 @@ class Check_idt(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.VersionRequirement( + name="linux_utilities_modules", + component=linux_utilities_modules.Modules, + version=(1, 0, 0), + ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) ), diff --git a/volatility3/framework/plugins/linux/keyboard_notifiers.py b/volatility3/framework/plugins/linux/keyboard_notifiers.py index c1b7572c6..8577de848 100644 --- a/volatility3/framework/plugins/linux/keyboard_notifiers.py +++ b/volatility3/framework/plugins/linux/keyboard_notifiers.py @@ -27,6 +27,11 @@ class Keyboard_notifiers(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.VersionRequirement( + name="linux_utilities_modules", + component=linux_utilities_modules.Modules, + version=(1, 0, 0), + ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index 2e1bbed47..bd0e895a4 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -31,6 +31,11 @@ class Kthreads(plugins.PluginInterface): description="Linux kernel", architectures=architectures.LINUX_ARCHS, ), + requirements.VersionRequirement( + name="linux_utilities_modules", + component=linux_utilities_modules.Modules, + version=(1, 0, 0), + ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index ccb831b61..33a8ca7cc 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -691,6 +691,11 @@ class Netfilter(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.VersionRequirement( + name="linux_utilities_modules", + component=linux_utilities_modules.Modules, + version=(1, 0, 0), + ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=cls._required_lsmod_version ), diff --git a/volatility3/framework/plugins/linux/tty_check.py b/volatility3/framework/plugins/linux/tty_check.py index f375968a4..9bbca246c 100644 --- a/volatility3/framework/plugins/linux/tty_check.py +++ b/volatility3/framework/plugins/linux/tty_check.py @@ -30,6 +30,11 @@ class tty_check(plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.VersionRequirement( + name="linux_utilities_modules", + component=linux_utilities_modules.Modules, + version=(1, 0, 0), + ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), From ce671bb2fa3a7d03042a993814b4c11f19650cf4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 15:02:25 +0100 Subject: [PATCH 945/979] transfer deprecated_method into framework module --- volatility3/framework/__init__.py | 29 ++++++++++++++++++- .../framework/configuration/requirements.py | 23 --------------- .../framework/symbols/linux/__init__.py | 17 +++++++---- 3 files changed, 39 insertions(+), 30 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index a1925faef..12254ca77 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -11,7 +11,8 @@ import inspect import logging import os import traceback -from typing import Any, Dict, Generator, List, Optional, Tuple, Type, TypeVar +import functools +from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Type, TypeVar from volatility3.framework import constants, interfaces @@ -52,6 +53,32 @@ def require_interface_version(*args) -> None: ) +class Deprecation: + """Deprecation related methods.""" + + @staticmethod + def deprecated_method(replacement: Callable, additional_information: str = ""): + """A decorator for marking functions as deprecated. + + Args: + replacement: The replacement function overriding the deprecated API, in the form of a Callable (typically a method) + additional_information: Information appended at the end of the deprecation message + """ + + def decorator(deprecated_func): + @functools.wraps(deprecated_func) + def wrapper(*args, **kwargs): + nonlocal replacement, additional_information + deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__name__}\" is deprecated, use \"{replacement.__module__ + '.' + replacement.__name__}\" instead. {additional_information}" + vollog.warning(deprecation_msg) + # Return the wrapped function with its original arguments + return deprecated_func(*args, **kwargs) + + return wrapper + + return decorator + + class NonInheritable: def __init__(self, value: Any, cls: Type) -> None: self.default_value = value diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 3af5601dc..3e3608000 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -11,7 +11,6 @@ expect to be in the context (such as particular layers or symboltables). import abc import logging import os -import functools from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, Type from urllib import parse, request @@ -724,25 +723,3 @@ class ModuleRequirement( """Builds the appropriate configuration for the specified requirement.""" return context.modules[value].build_configuration() - - -def deprecated_method(replacement: str, additional_information: str = ""): - """A decorator for marking functions as deprecated. - - Args: - replacement: The replacement function overriding the deprecated API (full path preferred, starting from "volatility3."). String was preferred, for convenience and to prevent import conflicts on caller side. - additional_information: Information appended at the end of the deprecation message - """ - - def decorator(deprecated_func): - @functools.wraps(deprecated_func) - def wrapper(*args, **kwargs): - nonlocal replacement, additional_information - deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__name__}\" is deprecated, use \"{replacement}\" instead. {additional_information}" - vollog.warning(deprecation_msg) - # Return the wrapped function with its original arguments - return deprecated_func(*args, **kwargs) - - return wrapper - - return decorator diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3dc744f78..0b7ef751c 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -10,11 +10,16 @@ from typing import Iterator, List, Tuple, Optional, Union import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3 import framework -from volatility3.framework import constants, exceptions, interfaces, objects +from volatility3.framework import ( + constants, + exceptions, + interfaces, + objects, + Deprecation, +) from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions -from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -455,8 +460,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ## Deprecated APIs ## @classmethod - @requirements.deprecated_method( - replacement="volatility3.framework.symbols.linux.utilities.modules.Modules.mask_mods_list" + @Deprecation.deprecated_method( + replacement=linux_utilities_modules.Modules.mask_mods_list ) def mask_mods_list( cls, @@ -472,8 +477,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return linux_utilities_modules.Modules.mask_mods_list(context, layer_name, mods) @classmethod - @requirements.deprecated_method( - replacement="volatility3.framework.symbols.linux.utilities.modules.Modules.lookup_module_address" + @Deprecation.deprecated_method( + replacement=linux_utilities_modules.Modules.lookup_module_address ) def lookup_module_address( cls, From 2e1b77f4b3186bae3dfbc7e51e2ba51e9fd850e3 Mon Sep 17 00:00:00 2001 From: Daniel Davidov <35842733+Danking555@users.noreply.github.com> Date: Sat, 25 Jan 2025 16:22:49 +0200 Subject: [PATCH 946/979] Move LowStubLayout constants to windows.constants * Moved constants out of the class and moved to constants.windows * Applied ruff and black --- volatility3/framework/automagic/pdbscan.py | 26 +++---------------- .../framework/constants/windows/__init__.py | 18 +++++++++++++ 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 7d289bcb6..f9c0d853d 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -376,26 +376,6 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): valid_kernel = (virtual_layer_name, address, res[0]) return valid_kernel - class LowStubLayout: - """ - Represents the layout of the Low Stub which exists only on x64 machines with no virtualization/emulation, - responsible for transitioning from Real Mode(16 bit) to Protected Mode(32 bit) and Long Mode(64 bit) on boot/return from sleep. - Contains offsets to fields and structures within the undocumented structure _PROCESSOR_START_BLOCK. - Here's a reference: https://github.com/mic101/windows/blob/master/WRK-v1.2/base/ntos/inc/amd64.h#L3334 - """ - - # Expected signature for validation, constructed from: - # PROCESSOR_START_BLOCK->Jmp->OpCode | PROCESSOR_START_BLOCK->Jmp->Offset | PROCESSOR_START_BLOCK->CompletionFlag - JMP_AND_COMPLETION_SIGNATURE = 0x00000001000600E9 - - # Address of LmTarget (Long Mode target) - PROCESSOR_START_BLOCK_LM_TARGET_OFFSET = ( - 0x70 # PROCESSOR_START_BLOCK->LmTarget, PVOID 8 bytes - ) - - # CR3 register within structures describing initial processor state to be started - PROCESSOR_START_BLOCK_CR3_OFFSET = 0xA0 # PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3, ULONG64 8 bytes - def method_low_stub_offset( self, context: interfaces.context.ContextInterface, @@ -417,12 +397,12 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): ) if ( 0xFFFFFFFFFFFF00FF & jmp_and_completion_values - != self.LowStubLayout.JMP_AND_COMPLETION_SIGNATURE + != constants.windows.JMP_AND_COMPLETION_SIGNATURE ): continue cr3_value = int.from_bytes( physical_layer.read( - offset + self.LowStubLayout.PROCESSOR_START_BLOCK_CR3_OFFSET, 0x8 + offset + constants.windows.PROCESSOR_START_BLOCK_CR3_OFFSET, 0x8 ), "little", ) @@ -434,7 +414,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): continue potential_kernel_hint = int.from_bytes( physical_layer.read( - offset + self.LowStubLayout.PROCESSOR_START_BLOCK_LM_TARGET_OFFSET, + offset + constants.windows.PROCESSOR_START_BLOCK_LM_TARGET_OFFSET, 0x8, ), "little", diff --git a/volatility3/framework/constants/windows/__init__.py b/volatility3/framework/constants/windows/__init__.py index 7face984a..6f37acd2d 100644 --- a/volatility3/framework/constants/windows/__init__.py +++ b/volatility3/framework/constants/windows/__init__.py @@ -10,3 +10,21 @@ KERNEL_MODULE_NAMES = ["ntkrnlmp", "ntkrnlpa", "ntkrpamp", "ntoskrnl"] """The list of names that kernel modules can have within the windows OS""" PE_MAX_EXTRACTION_SIZE = 1024 * 1024 * 256 + +""" +The following constants represent the layout of the Low Stub which exists only on x64 machines with no virtualization/emulation, +responsible for transitioning from Real Mode(16 bit) to Protected Mode(32 bit) and Long Mode(64 bit) on boot/return from sleep. +Contains offsets to fields and structures within the undocumented structure _PROCESSOR_START_BLOCK. +Here's a reference: https://github.com/mic101/windows/blob/master/WRK-v1.2/base/ntos/inc/amd64.h#L3334 +""" +# Expected signature for validation, constructed from: +# PROCESSOR_START_BLOCK->Jmp->OpCode | PROCESSOR_START_BLOCK->Jmp->Offset | PROCESSOR_START_BLOCK->CompletionFlag +JMP_AND_COMPLETION_SIGNATURE = 0x00000001000600E9 + +# Address of LmTarget (Long Mode target) +PROCESSOR_START_BLOCK_LM_TARGET_OFFSET = ( + 0x70 # PROCESSOR_START_BLOCK->LmTarget, PVOID 8 bytes +) + +# CR3 register within structures describing initial processor state to be started +PROCESSOR_START_BLOCK_CR3_OFFSET = 0xA0 # PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3, ULONG64 8 bytes From d8658f0729f7abbd8410f76571aad6369deabee6 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 17:08:27 +0100 Subject: [PATCH 947/979] put deprecated functions order back --- .../framework/symbols/linux/__init__.py | 75 +++++++++---------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0b7ef751c..bc2492f7a 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -345,6 +345,23 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): yield fd_num, filp, full_path + @classmethod + @Deprecation.deprecated_method( + replacement=linux_utilities_modules.Modules.mask_mods_list + ) + def mask_mods_list( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + mods: Iterator[interfaces.objects.ObjectInterface], + ) -> List[Tuple[str, int, int]]: + """ + DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.mask_mods_list" instead. + + A helper function to mask the starting and end address of kernel modules + """ + return linux_utilities_modules.Modules.mask_mods_list(context, layer_name, mods) + @classmethod def generate_kernel_handler_info( cls, @@ -372,6 +389,26 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): context, kernel.layer_name, mods_list ) + @classmethod + @Deprecation.deprecated_method( + replacement=linux_utilities_modules.Modules.lookup_module_address + ) + def lookup_module_address( + cls, + kernel_module: interfaces.context.ModuleInterface, + handlers: List[Tuple[str, int, int]], + target_address: int, + ) -> Tuple[str, str]: + """ + DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.lookup_module_address" instead. + + Searches between the start and end address of the kernel module using target_address. + Returns the module and symbol name of the address provided. + """ + return linux_utilities_modules.Modules.lookup_module_address( + kernel_module.context, kernel_module.name, handlers, target_address + ) + @classmethod def walk_internal_list(cls, vmlinux, struct_name, list_member, list_start): while list_start: @@ -458,44 +495,6 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): [chr((code >> (i * 8)) & 0xFF) for i in range(code_bytes_length)] ) - ## Deprecated APIs ## - @classmethod - @Deprecation.deprecated_method( - replacement=linux_utilities_modules.Modules.mask_mods_list - ) - def mask_mods_list( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - mods: Iterator[interfaces.objects.ObjectInterface], - ) -> List[Tuple[str, int, int]]: - """ - DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.mask_mods_list" instead. - - A helper function to mask the starting and end address of kernel modules - """ - return linux_utilities_modules.Modules.mask_mods_list(context, layer_name, mods) - - @classmethod - @Deprecation.deprecated_method( - replacement=linux_utilities_modules.Modules.lookup_module_address - ) - def lookup_module_address( - cls, - kernel_module: interfaces.context.ModuleInterface, - handlers: List[Tuple[str, int, int]], - target_address: int, - ) -> Tuple[str, str]: - """ - DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.lookup_module_address" instead. - - Searches between the start and end address of the kernel module using target_address. - Returns the module and symbol name of the address provided. - """ - return linux_utilities_modules.Modules.lookup_module_address( - kernel_module.context, kernel_module.name, handlers, target_address - ) - class IDStorage(ABC): """Abstraction to support both XArray and RadixTree""" From 825720ed5d8f290b244735c758149e5b9d208c12 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 19:48:31 +0100 Subject: [PATCH 948/979] catch UnsatisfiedException at plugin runtime --- volatility3/cli/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 82a2a4205..d3ce74847 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -500,6 +500,16 @@ class CommandLine: renderer.filter = text_filter.CLIFilter(grid, args.filters) renderer.column_hide_list = args.hide_columns renderer.render(grid) + except exceptions.UnsatisfiedException as excp: + output = sys.stderr + output.write( + "An unsatisfied framework exception was encountered post plugin construction:\n" + ) + self.process_unsatisfied_exceptions(excp) + output.write( + f"Unable to validate the requirements: {[x for x in excp.unsatisfied]}\n", + ) + sys.exit(1) except exceptions.VolatilityException as excp: self.process_exceptions(excp) From 2ed00cc91a6764d05162c01055fc17c18124c6aa Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 19:52:47 +0100 Subject: [PATCH 949/979] add optional version requirement to deprecated_method --- volatility3/framework/__init__.py | 56 ++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 12254ca77..bf4ec4447 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -12,9 +12,11 @@ import logging import os import traceback import functools +import warnings from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Type, TypeVar -from volatility3.framework import constants, interfaces +from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework.configuration import requirements # ## @@ -57,20 +59,64 @@ class Deprecation: """Deprecation related methods.""" @staticmethod - def deprecated_method(replacement: Callable, additional_information: str = ""): + def deprecated_method( + replacement: Callable, + replacement_base_class_required_version: Tuple[int, int, int] = None, + additional_information: str = "", + ): """A decorator for marking functions as deprecated. Args: replacement: The replacement function overriding the deprecated API, in the form of a Callable (typically a method) + replacement_base_class_required_version: The "replacement" base class version that the deprecated method expects before proxying to it. This implies that "replacement" is a method from a class that inherits from VersionableInterface. additional_information: Information appended at the end of the deprecation message """ def decorator(deprecated_func): @functools.wraps(deprecated_func) def wrapper(*args, **kwargs): - nonlocal replacement, additional_information - deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__name__}\" is deprecated, use \"{replacement.__module__ + '.' + replacement.__name__}\" instead. {additional_information}" - vollog.warning(deprecation_msg) + nonlocal replacement, replacement_base_class_required_version, additional_information + # Prevent version mismatches between deprecated (proxy) methods and the ones they proxy + if replacement_base_class_required_version is not None and callable( + replacement + ): + # example: replacement = volatility3.MyClass.my_dummy_function + # "MyClass.my_dummy_function" -> "MyClass" + replacement_base_class_name = replacement.__qualname__.split(".")[0] + # replacement.__globals__ example: {'MyClass': } + replacement_base_class = replacement.__globals__.get( + replacement_base_class_name + ) + + # Verify that the base class inherits from VersionableInterface + if inspect.isclass(replacement_base_class) and issubclass( + replacement_base_class, + interfaces.configuration.VersionableInterface, + ): + # Construct a requirement + req = requirements.VersionRequirement( + name=replacement_base_class.__name__, + component=replacement_base_class, + version=replacement_base_class_required_version, + ) + # Verify the requirement + if not req.matches_required( + req._version, req._component.version + ): + full_unsat_req_path = ( + deprecated_func.__module__ + + "." + + deprecated_func.__qualname__ + + "." + + req.name + ) + # Catched by the cli and redirected to process_unsatisfied_exceptions + raise exceptions.UnsatisfiedException( + {full_unsat_req_path: req} + ) + + deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__qualname__}\" is deprecated, use \"{replacement.__module__ + '.' + replacement.__qualname__}\" instead. {additional_information}" + warnings.warn(deprecation_msg, FutureWarning) # Return the wrapped function with its original arguments return deprecated_func(*args, **kwargs) From 3657c6fe5e8db92b1a6364a962b778a89e802624 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 19:54:20 +0100 Subject: [PATCH 950/979] require Modules >= 1.0.0 on deprecated methods --- volatility3/framework/symbols/linux/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index bc2492f7a..6a01efc3a 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -347,7 +347,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): @classmethod @Deprecation.deprecated_method( - replacement=linux_utilities_modules.Modules.mask_mods_list + replacement=linux_utilities_modules.Modules.mask_mods_list, + replacement_base_class_required_version=(1, 0, 0), ) def mask_mods_list( cls, @@ -391,7 +392,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): @classmethod @Deprecation.deprecated_method( - replacement=linux_utilities_modules.Modules.lookup_module_address + replacement=linux_utilities_modules.Modules.lookup_module_address, + replacement_base_class_required_version=(1, 0, 0), ) def lookup_module_address( cls, From 4262eff8898b3fbc017abe3c1d1e4f13fa6fb189 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 20:04:51 +0100 Subject: [PATCH 951/979] adhere to AbstractNetfilter requirement checking --- .../framework/plugins/linux/netfilter.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 33a8ca7cc..ccb7509aa 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -99,6 +99,20 @@ class AbstractNetfilter(ABC): f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" ) + linux_utilities_modules_required_version = ( + Netfilter._required_linux_utilities_modules_version + ) + linux_utilities_modules_current_version = ( + linux_utilities_modules.Modules._version + ) + if not requirements.VersionRequirement.matches_required( + linux_utilities_modules_required_version, + linux_utilities_modules_current_version, + ): + raise exceptions.PluginRequirementException( + f"linux_utilities_modules.Modules version not suitable: required {linux_utilities_modules_required_version} found {linux_utilities_modules_current_version}" + ) + modules = lsmod.Lsmod.list_modules(context, kernel_module_name) self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( context, kernel_module_name, modules @@ -680,6 +694,7 @@ class Netfilter(interfaces.plugins.PluginInterface): _version = (1, 1, 0) + _required_linux_utilities_modules_version = (1, 0, 0) _required_linuxutils_version = (2, 1, 0) _required_lsmod_version = (2, 0, 0) @@ -694,7 +709,7 @@ class Netfilter(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="linux_utilities_modules", component=linux_utilities_modules.Modules, - version=(1, 0, 0), + version=cls._required_linux_utilities_modules_version, ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=cls._required_lsmod_version From 6d43dcd3a842d308705f3ecd3c023c94be473846 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 10:31:46 +0100 Subject: [PATCH 952/979] add VersionMismatchException --- volatility3/framework/exceptions.py | 31 ++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index 41c67b88d..0409ae5f0 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -8,9 +8,10 @@ space or symbol tables, and by layers when an address is invalid. The :class:`PagedInvalidAddressException` contains information about the size of the invalid page. """ -from typing import Dict, Optional +from typing import Callable, Dict, Optional, Tuple from volatility3.framework import interfaces +from volatility3.framework.interfaces.configuration import VersionableInterface class VolatilityException(Exception): @@ -134,3 +135,31 @@ class RenderException(VolatilityException): class LinuxPageCacheException(VolatilityException): """Thrown if there is an error during Linux Page Cache processing""" + + +class VersionMismatchException(VolatilityException): + """Thrown if a version mismatch has been encountered between two components.""" + + def __init__( + self, + source_component: Callable, + target_component: VersionableInterface, + target_version: Tuple[int, int, int], + failure_reason: str = None, + *args, + ): + """ + Args: + source_component: The component that required the target component + target_component: The component that is required. Must inherit from VersionableInterface + target_version: The version of the target component that was required, and ultimately was not satisfied + failure_reason: A detailed failure reason to enhande debugging and bug tracking + """ + super().__init__(*args) + self.source_component = source_component + self.target_component = target_component + self.target_version = target_version + self.failure_reason = failure_reason + + def __str__(self): + return f"{self.source_component.__module__+ '.' + self.source_component.__qualname__}: Version {self.target_version} dependency on {self.target_component.__module__+ '.' + self.target_component.__qualname__} {self.target_component.version} unmet." From 4bd385cc9dadbfe8800fddc2bce9cef1ae4dadd5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 10:33:02 +0100 Subject: [PATCH 953/979] handle VersionMismatchException --- volatility3/cli/__init__.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index d3ce74847..b57d9a3f3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -500,16 +500,6 @@ class CommandLine: renderer.filter = text_filter.CLIFilter(grid, args.filters) renderer.column_hide_list = args.hide_columns renderer.render(grid) - except exceptions.UnsatisfiedException as excp: - output = sys.stderr - output.write( - "An unsatisfied framework exception was encountered post plugin construction:\n" - ) - self.process_unsatisfied_exceptions(excp) - output.write( - f"Unable to validate the requirements: {[x for x in excp.unsatisfied]}\n", - ) - sys.exit(1) except exceptions.VolatilityException as excp: self.process_exceptions(excp) @@ -583,6 +573,8 @@ class CommandLine: fulltrace = traceback.TracebackException.from_exception(excp).format(chain=True) vollog.debug("".join(fulltrace)) + file_a_bug_msg = f"Please re-run with -vvv and file a bug with the output at {constants.BUG_URL}" + if isinstance(excp, exceptions.InvalidAddressException): general = "Volatility was unable to read a requested page:" if isinstance(excp, exceptions.SwappedInvalidAddressException): @@ -627,9 +619,7 @@ class CommandLine: elif isinstance(excp, exceptions.LayerException): general = f"Volatility experienced a layer-related issue: {excp.layer_name}" detail = f"{excp}" - caused_by = [ - "A faulty layer implementation (re-run with -vvv and file a bug)" - ] + caused_by = [f"A faulty layer implementation. {file_a_bug_msg}"] elif isinstance(excp, exceptions.MissingModuleException): general = f"Volatility could not import a necessary module: {excp.module}" detail = f"{excp}" @@ -640,13 +630,17 @@ class CommandLine: general = "Volatility experienced an issue when rendering the output:" detail = f"{excp}" caused_by = ["An invalid renderer option, such as no visible columns"] + elif isinstance(excp, exceptions.VersionMismatchException): + general = "A version mismatch was detected between two components:" + detail = f"{excp}" + caused_by = [ + excp.failure_reason or "An outdated API caller, such as a method.", + file_a_bug_msg, + ] else: general = "Volatility encountered an unexpected situation." detail = "" - caused_by = [ - "Please re-run using with -vvv and file a bug with the output", - f"at {constants.BUG_URL}", - ] + caused_by = [file_a_bug_msg] # Code that actually renders the exception output = sys.stderr From ba6b709aba9496f75a61a806f06719189ac97491 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 10:34:18 +0100 Subject: [PATCH 954/979] use VersionMismatchException --- volatility3/framework/__init__.py | 37 ++++++++++--------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index bf4ec4447..5d4e0f3c4 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -61,25 +61,23 @@ class Deprecation: @staticmethod def deprecated_method( replacement: Callable, - replacement_base_class_required_version: Tuple[int, int, int] = None, + replacement_version: Tuple[int, int, int] = None, additional_information: str = "", ): """A decorator for marking functions as deprecated. Args: replacement: The replacement function overriding the deprecated API, in the form of a Callable (typically a method) - replacement_base_class_required_version: The "replacement" base class version that the deprecated method expects before proxying to it. This implies that "replacement" is a method from a class that inherits from VersionableInterface. + replacement_version: The "replacement" base class version that the deprecated method expects before proxying to it. This implies that "replacement" is a method from a class that inherits from VersionableInterface. additional_information: Information appended at the end of the deprecation message """ def decorator(deprecated_func): @functools.wraps(deprecated_func) def wrapper(*args, **kwargs): - nonlocal replacement, replacement_base_class_required_version, additional_information + nonlocal replacement, replacement_version, additional_information # Prevent version mismatches between deprecated (proxy) methods and the ones they proxy - if replacement_base_class_required_version is not None and callable( - replacement - ): + if replacement_version is not None and callable(replacement): # example: replacement = volatility3.MyClass.my_dummy_function # "MyClass.my_dummy_function" -> "MyClass" replacement_base_class_name = replacement.__qualname__.split(".")[0] @@ -93,26 +91,15 @@ class Deprecation: replacement_base_class, interfaces.configuration.VersionableInterface, ): - # Construct a requirement - req = requirements.VersionRequirement( - name=replacement_base_class.__name__, - component=replacement_base_class, - version=replacement_base_class_required_version, - ) - # Verify the requirement - if not req.matches_required( - req._version, req._component.version + # SemVer check + if not requirements.VersionRequirement.matches_required( + replacement_version, replacement_base_class.version ): - full_unsat_req_path = ( - deprecated_func.__module__ - + "." - + deprecated_func.__qualname__ - + "." - + req.name - ) - # Catched by the cli and redirected to process_unsatisfied_exceptions - raise exceptions.UnsatisfiedException( - {full_unsat_req_path: req} + raise exceptions.VersionMismatchException( + deprecated_func, + replacement_base_class, + replacement_version, + "A deprecated method was unable to proxy the call to its replacement", ) deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__qualname__}\" is deprecated, use \"{replacement.__module__ + '.' + replacement.__qualname__}\" instead. {additional_information}" From 28081ed989599bc395b190cf503e8d83ef5b8fed Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 10:35:25 +0100 Subject: [PATCH 955/979] tidy up replacement_base_class_required_version --- volatility3/framework/symbols/linux/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 6a01efc3a..397c36c01 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -348,7 +348,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): @classmethod @Deprecation.deprecated_method( replacement=linux_utilities_modules.Modules.mask_mods_list, - replacement_base_class_required_version=(1, 0, 0), + replacement_version=(1, 0, 0), ) def mask_mods_list( cls, @@ -393,7 +393,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): @classmethod @Deprecation.deprecated_method( replacement=linux_utilities_modules.Modules.lookup_module_address, - replacement_base_class_required_version=(1, 0, 0), + replacement_version=(1, 0, 0), ) def lookup_module_address( cls, From 459483651b847241cb4189c4d1449fd18c0bdca7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 10:53:33 +0100 Subject: [PATCH 956/979] typo --- volatility3/framework/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index 0409ae5f0..99c5f155e 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -153,7 +153,7 @@ class VersionMismatchException(VolatilityException): source_component: The component that required the target component target_component: The component that is required. Must inherit from VersionableInterface target_version: The version of the target component that was required, and ultimately was not satisfied - failure_reason: A detailed failure reason to enhande debugging and bug tracking + failure_reason: A detailed failure reason to enhance debugging and bug tracking """ super().__init__(*args) self.source_component = source_component From 80eebd2f49bee665f194e4adba92cf12a192a997 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 13:18:30 +0100 Subject: [PATCH 957/979] use classmethod instead of staticmethod --- volatility3/framework/symbols/linux/utilities/modules.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/modules.py b/volatility3/framework/symbols/linux/utilities/modules.py index ac9b2afaf..82c63fc18 100644 --- a/volatility3/framework/symbols/linux/utilities/modules.py +++ b/volatility3/framework/symbols/linux/utilities/modules.py @@ -13,8 +13,9 @@ class Modules(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) - @staticmethod + @classmethod def mask_mods_list( + cls, context: interfaces.context.ContextInterface, layer_name: str, mods: Iterator[interfaces.objects.ObjectInterface], @@ -33,8 +34,9 @@ class Modules(interfaces.configuration.VersionableInterface): for mod in mods ] - @staticmethod + @classmethod def lookup_module_address( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str, handlers: List[Tuple[str, int, int]], From 5d58ba63dbd6a60fd36faca6f23a3a1b4a11e724 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 13:19:01 +0100 Subject: [PATCH 958/979] use __name__ instead of overkill __qualname__ --- volatility3/framework/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index 99c5f155e..a3d660444 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -162,4 +162,4 @@ class VersionMismatchException(VolatilityException): self.failure_reason = failure_reason def __str__(self): - return f"{self.source_component.__module__+ '.' + self.source_component.__qualname__}: Version {self.target_version} dependency on {self.target_component.__module__+ '.' + self.target_component.__qualname__} {self.target_component.version} unmet." + return f"{self.source_component.__module__+ '.' + self.source_component.__qualname__}: Version {self.target_version} dependency on {self.target_component.__module__+ '.' + self.target_component.__name__} {self.target_component.version} unmet." From 4cb386858ebe7c62f05b16c9f869fdf762084121 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 13:20:48 +0100 Subject: [PATCH 959/979] use __self__ and enhance exception msg --- volatility3/framework/__init__.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 5d4e0f3c4..aa93340cd 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -77,14 +77,12 @@ class Deprecation: def wrapper(*args, **kwargs): nonlocal replacement, replacement_version, additional_information # Prevent version mismatches between deprecated (proxy) methods and the ones they proxy - if replacement_version is not None and callable(replacement): - # example: replacement = volatility3.MyClass.my_dummy_function - # "MyClass.my_dummy_function" -> "MyClass" - replacement_base_class_name = replacement.__qualname__.split(".")[0] - # replacement.__globals__ example: {'MyClass': } - replacement_base_class = replacement.__globals__.get( - replacement_base_class_name - ) + if ( + replacement_version is not None + and callable(replacement) + and hasattr(replacement, "__self__") + ): + replacement_base_class = replacement.__self__ # Verify that the base class inherits from VersionableInterface if inspect.isclass(replacement_base_class) and issubclass( @@ -99,7 +97,7 @@ class Deprecation: deprecated_func, replacement_base_class, replacement_version, - "A deprecated method was unable to proxy the call to its replacement", + "This is a bug, the deprecated call needs to be removed and the caller needs to update their code to use the new method.", ) deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__qualname__}\" is deprecated, use \"{replacement.__module__ + '.' + replacement.__qualname__}\" instead. {additional_information}" From a25165d935c02c07510b8b4721b074d7bed21258 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 13:44:23 +0100 Subject: [PATCH 960/979] 2.18.1 -> 2.19.0 bump --- volatility3/framework/constants/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 689e39664..f2403cf4a 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 18 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 19 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From 1ac2dbc49c10552d1d26debe2eb54e0f6922c1c8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 26 Jan 2025 13:44:33 +0000 Subject: [PATCH 961/979] Shift exposed staticmethods to classmethods --- .../framework/plugins/linux/capabilities.py | 4 ++- volatility3/framework/plugins/linux/envars.py | 5 +-- .../framework/plugins/linux/hidden_modules.py | 6 ++-- .../framework/plugins/linux/pagecache.py | 11 +++--- .../framework/plugins/linux/vmayarascan.py | 5 +-- .../framework/plugins/windows/cachedump.py | 18 +++++----- .../plugins/windows/direct_system_calls.py | 13 +++---- .../framework/plugins/windows/mftscan.py | 21 ++++++----- .../framework/plugins/windows/netscan.py | 6 ++-- .../framework/plugins/windows/pe_symbols.py | 35 +++++++++++-------- .../framework/plugins/windows/poolscanner.py | 6 ++-- .../framework/plugins/windows/shimcachemem.py | 4 ++- .../framework/plugins/windows/svcscan.py | 5 +-- .../plugins/windows/unloadedmodules.py | 5 +-- .../framework/plugins/windows/vadyarascan.py | 5 +-- volatility3/framework/plugins/yarascan.py | 14 ++++---- 16 files changed, 93 insertions(+), 70 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 1d0c60c11..b758a04b4 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -35,7 +35,9 @@ class CapabilitiesData: cap_permitted: interfaces.objects.ObjectInterface cap_effective: interfaces.objects.ObjectInterface cap_bset: interfaces.objects.ObjectInterface - cap_ambient: interfaces.objects.ObjectInterface + cap_ambient: ( + interfaces.objects.ObjectInterface | interfaces.renderers.BaseAbsentValue + ) def astuple(self) -> Tuple: """Returns a shallow copy of the capability sets in a tuple. diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 8cdbfe493..cc43c4130 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -18,7 +18,7 @@ class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" _required_framework_version = (2, 13, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls): @@ -40,8 +40,9 @@ class Envars(plugins.PluginInterface): ), ] - @staticmethod + @classmethod def get_task_env_variables( + cls, context: interfaces.context.ContextInterface, task: interfaces.objects.ObjectInterface, env_area_max_size: int = 8192, diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index fd4b28943..e1ba40926 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -16,8 +16,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): """Carves memory to find hidden kernel modules""" _required_framework_version = (2, 10, 0) - - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -32,8 +31,9 @@ class Hidden_modules(interfaces.plugins.PluginInterface): ), ] - @staticmethod + @classmethod def get_modules_memory_boundaries( + cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, ) -> Tuple[int]: diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 32b176b72..4d1250255 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -104,7 +104,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -360,8 +360,8 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): yield description, timeliner.TimeLinerType.MODIFIED, inode_out.modification_time yield description, timeliner.TimeLinerType.CHANGED, inode_out.change_time - @staticmethod - def format_fields_with_headers(headers, generator): + @classmethod + def format_fields_with_headers(cls, headers, generator): """Uses the headers type to cast the fields obtained from the generator""" for level, fields in generator: formatted_fields = [] @@ -405,7 +405,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (2, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -436,8 +436,9 @@ class InodePages(plugins.PluginInterface): ), ] - @staticmethod + @classmethod def write_inode_content_to_file( + cls, inode: interfaces.objects.ObjectInterface, filename: str, open_method: Type[interfaces.plugins.FileHandlerInterface], diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 4db23e50b..e9e56dd0f 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -18,7 +18,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -105,8 +105,9 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): value, ) - @staticmethod + @classmethod def get_vma_maps( + cls, task: interfaces.objects.ObjectInterface, ) -> Iterable[Tuple[int, int]]: """Creates a map of start/end addresses for each virtual memory area in a task. diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index 6c730e6ae..f4f2e061e 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -22,7 +22,7 @@ class Cachedump(interfaces.plugins.PluginInterface): """Dumps lsa secrets from memory""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -43,16 +43,16 @@ class Cachedump(interfaces.plugins.PluginInterface): ), ] - @staticmethod + @classmethod def get_nlkm( - sechive: registry.RegistryHive, lsakey: bytes, is_vista_or_later: bool + cls, sechive: registry.RegistryHive, lsakey: bytes, is_vista_or_later: bool ): return lsadump.Lsadump.get_secret_by_name( sechive, "NL$KM", lsakey, is_vista_or_later ) - @staticmethod - def decrypt_hash(edata: bytes, nlkm: bytes, ch, xp: bool): + @classmethod + def decrypt_hash(cls, edata: bytes, nlkm: bytes, ch, xp: bool): if xp: hmac_md5 = HMAC.new(nlkm, ch) rc4key = hmac_md5.digest() @@ -69,8 +69,8 @@ class Cachedump(interfaces.plugins.PluginInterface): data += aes.decrypt(buf) return data - @staticmethod - def parse_cache_entry(cache_data: bytes) -> Tuple[int, int, int, bytes, bytes]: + @classmethod + def parse_cache_entry(cls, cache_data: bytes) -> Tuple[int, int, int, bytes, bytes]: (uname_len, domain_len) = unpack(" Tuple[str, str, str, bytes]: """Get the data from the cache and separate it into the username, domain name, and hash data""" uname_offset = 72 diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 183e4095c..af626f511 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -53,7 +53,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): """Detects the Direct System Call technique used to bypass EDRs""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) # DLLs that are expected to host system call invocations valid_syscall_handlers = ("ntdll.dll", "win32u.dll") @@ -200,8 +200,8 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return disasm_bytes, end_inst - @staticmethod - def get_disasm_function(architecture: str) -> Callable: + @classmethod + def get_disasm_function(cls, architecture: str) -> Callable: """ Returns the disassembly handler for the given architecture .detail is used to get full instruction information @@ -284,8 +284,9 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return None - @staticmethod + @classmethod def get_vad_maps( + cls, task: interfaces.objects.ObjectInterface, ) -> List[Tuple[int, int, str]]: """Creates a map of start/end addresses within a virtual address @@ -310,9 +311,9 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return vads - @staticmethod + @classmethod def get_range_path( - ranges: List[Tuple[int, int, str]], address: int + cls, ranges: List[Tuple[int, int, str]], address: int ) -> Optional[str]: """ Returns the path for the range holding `address`, if found diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index c4d05e634..2c5827a25 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -22,7 +22,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls): @@ -37,8 +37,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] - @staticmethod + @classmethod def enumerate_mft_records( + cls, context: interfaces.context.ContextInterface, config_path: str, primary_layer_name: str, @@ -128,8 +129,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): layer_name=layer.name, ) - @staticmethod + @classmethod def parse_mft_records( + cls, record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, @@ -191,8 +193,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_name, ) - @staticmethod + @classmethod def parse_data_record( + cls, mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, record_map: Dict[int, Tuple[str, int, int]], @@ -325,7 +328,7 @@ class ADS(interfaces.plugins.PluginInterface): _required_framework_version = (2, 7, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -343,8 +346,9 @@ class ADS(interfaces.plugins.PluginInterface): ), ] - @staticmethod + @classmethod def parse_ads_data_records( + cls, record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, @@ -394,7 +398,7 @@ class ResidentData(interfaces.plugins.PluginInterface): _required_framework_version = (2, 7, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -412,8 +416,9 @@ class ResidentData(interfaces.plugins.PluginInterface): ), ] - @staticmethod + @classmethod def parse_first_data_records( + cls, record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 162031104..c30792908 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -23,7 +23,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for network objects present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -50,9 +50,9 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] - @staticmethod + @classmethod def create_netscan_constraints( - context: interfaces.context.ContextInterface, symbol_table: str + cls, context: interfaces.context.ContextInterface, symbol_table: str ) -> List[poolscanner.PoolConstraint]: """Creates a list of Pool Tag Constraints for network objects. diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 21e657ab3..88ced7e06 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -244,7 +244,7 @@ class PESymbols(interfaces.plugins.PluginInterface): _required_framework_version = (2, 7, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) # used for special handling of the kernel PDB file. See later notes os_module_name = "ntoskrnl.exe" @@ -330,9 +330,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return pe_ret - @staticmethod + @classmethod def range_info_for_address( - ranges: ranges_type, address: int + cls, ranges: ranges_type, address: int ) -> Optional[range_type]: """ Helper for getting the range information for an address. @@ -351,8 +351,8 @@ class PESymbols(interfaces.plugins.PluginInterface): return None - @staticmethod - def filepath_for_address(ranges: ranges_type, address: int) -> Optional[str]: + @classmethod + def filepath_for_address(cls, ranges: ranges_type, address: int) -> Optional[str]: """ Helper to get the file path for an address @@ -369,8 +369,8 @@ class PESymbols(interfaces.plugins.PluginInterface): return None - @staticmethod - def filename_for_path(filepath: str) -> str: + @classmethod + def filename_for_path(cls, filepath: str) -> str: """ Consistent way to get the filename regardless of platform @@ -382,8 +382,9 @@ class PESymbols(interfaces.plugins.PluginInterface): """ return ntpath.basename(filepath).lower() - @staticmethod + @classmethod def addresses_for_process_symbols( + cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, @@ -416,8 +417,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_symbols - @staticmethod + @classmethod def path_and_symbol_for_address( + cls, context: interfaces.context.ContextInterface, config_path: str, collected_modules: collected_modules_type, @@ -733,8 +735,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return found, remaining - @staticmethod + @classmethod def find_symbols( + cls, context: interfaces.context.ContextInterface, config_path: str, wanted_modules: PESymbolFinder.cached_value_dict, @@ -775,8 +778,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_symbols, missing_symbols - @staticmethod + @classmethod def get_kernel_modules( + cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, @@ -837,8 +841,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_modules - @staticmethod + @classmethod def get_vads_for_process_cache( + cls, vads_cache: Dict[int, ranges_type], owner_proc: interfaces.objects.ObjectInterface, ) -> Optional[ranges_type]: @@ -865,8 +870,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return vads - @staticmethod + @classmethod def get_proc_vads_with_file_paths( + cls, proc: interfaces.objects.ObjectInterface, ) -> ranges_type: """ @@ -928,8 +934,9 @@ class PESymbols(interfaces.plugins.PluginInterface): yield proc, proc_layer_name, vads - @staticmethod + @classmethod def get_process_modules( + cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index efde09638..5be0e7fa8 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -127,8 +127,8 @@ class PoolHeaderScanner(interfaces.layers.ScannerInterface): class PoolScanner(plugins.PluginInterface): """A generic pool scanner plugin.""" - _version = (1, 0, 0) _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -181,9 +181,9 @@ class PoolScanner(plugins.PluginInterface): ), ) - @staticmethod + @classmethod def builtin_constraints( - symbol_table: str, tags_filter: Optional[List[bytes]] = None + cls, symbol_table: str, tags_filter: Optional[List[bytes]] = None ) -> List[PoolConstraint]: """Get built-in PoolConstraints given a list of pool tags. diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index 9d968c30a..1e1024656 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -24,6 +24,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf """Reads Shimcache entries from the ahcache.sys AVL tree""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) # These checks must be completed from newest -> oldest OS version. _win_version_file_map: List[Tuple[versions.OsDistinguisher, bool, str]] = [ @@ -74,8 +75,9 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf ), ] - @staticmethod + @classmethod def create_shimcache_table( + cls, context: interfaces.context.ContextInterface, symbol_table: str, config_path: str, diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 17baac5b0..6645fa6a3 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -35,7 +35,7 @@ class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 1) + _version = (3, 0, 2) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -61,8 +61,9 @@ class SvcScan(interfaces.plugins.PluginInterface): ), ] - @staticmethod + @classmethod def get_record_tuple( + cls, service_record: interfaces.objects.ObjectInterface, binary_info: ServiceBinaryInfo, ): diff --git a/volatility3/framework/plugins/windows/unloadedmodules.py b/volatility3/framework/plugins/windows/unloadedmodules.py index 077fe33cb..d9f104ae8 100644 --- a/volatility3/framework/plugins/windows/unloadedmodules.py +++ b/volatility3/framework/plugins/windows/unloadedmodules.py @@ -22,7 +22,7 @@ class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInt """Lists the unloaded kernel modules.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -34,8 +34,9 @@ class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInt ), ] - @staticmethod + @classmethod def create_unloadedmodules_table( + cls, context: interfaces.context.ContextInterface, symbol_table: str, config_path: str, diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 2e9cc44ea..11ddc3716 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -18,7 +18,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 1, 1) + _version = (1, 1, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -104,8 +104,9 @@ class VadYaraScan(interfaces.plugins.PluginInterface): value, ) - @staticmethod + @classmethod def get_vad_maps( + cls, task: interfaces.objects.ObjectInterface, ) -> Iterable[Tuple[int, int]]: """Creates a map of start/end addresses within a virtual address diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 310bbd072..38c8b6085 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -37,7 +37,7 @@ except ImportError: class YaraScanner(interfaces.layers.ScannerInterface): - _version = (2, 1, 0) + _version = (2, 1, 1) # yara.Rules isn't exposed, so we can't type this properly def __init__(self, rules) -> None: @@ -79,23 +79,23 @@ class YaraScanner(interfaces.layers.ScannerInterface): for offset, name, value in match.strings: yield (offset + data_offset, match.rule, name, value) - @staticmethod - def get_rule(rule): + @classmethod + def get_rule(cls, rule): if USE_YARA_X: return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") return yara.compile( sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} ) - @staticmethod - def from_compiled_file(filepath): + @classmethod + def from_compiled_file(cls, filepath): with resources.ResourceAccessor().open(filepath, "rb") as fp: if USE_YARA_X: return yara_x.Rules.deserialize_from(file=fp) return yara.load(file=fp) - @staticmethod - def from_file(filepath): + @classmethod + def from_file(cls, filepath): with resources.ResourceAccessor().open(filepath, "rb") as fp: if USE_YARA_X: return yara_x.compile(fp.read().decode()) From e4e54a5c58e24b053b0509de952ea1647e07877f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 26 Jan 2025 13:48:00 +0000 Subject: [PATCH 962/979] Don't fix the type error as part of the shift. --- volatility3/framework/plugins/linux/capabilities.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index b758a04b4..1d0c60c11 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -35,9 +35,7 @@ class CapabilitiesData: cap_permitted: interfaces.objects.ObjectInterface cap_effective: interfaces.objects.ObjectInterface cap_bset: interfaces.objects.ObjectInterface - cap_ambient: ( - interfaces.objects.ObjectInterface | interfaces.renderers.BaseAbsentValue - ) + cap_ambient: interfaces.objects.ObjectInterface def astuple(self) -> Tuple: """Returns a shallow copy of the capability sets in a tuple. From 8b31ae612f8458e349cf0a063a578b97ec7ef8e9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 27 Jan 2025 09:32:33 +0000 Subject: [PATCH 963/979] Layers: Make the low-stub method less brittle --- volatility3/framework/automagic/pdbscan.py | 64 ++++++++++++---------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index f9c0d853d..dd2ad0683 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -392,38 +392,42 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): # Try locating kernel base via x64 Low Stub in lower 1MB starting from second page (4KB) # If "Discard Low Memory" setting is disabled in BIOS, the Low Stub may be at the third/fourth or further pages for offset in range(0x1000, 0x100000, 0x1000): - jmp_and_completion_values = int.from_bytes( - physical_layer.read(offset, 0x8), "little" - ) - if ( - 0xFFFFFFFFFFFF00FF & jmp_and_completion_values - != constants.windows.JMP_AND_COMPLETION_SIGNATURE - ): - continue - cr3_value = int.from_bytes( - physical_layer.read( - offset + constants.windows.PROCESSOR_START_BLOCK_CR3_OFFSET, 0x8 - ), - "little", - ) + try: + jmp_and_completion_values = int.from_bytes( + physical_layer.read(offset, 0x8), "little" + ) + if ( + 0xFFFFFFFFFFFF00FF & jmp_and_completion_values + != constants.windows.JMP_AND_COMPLETION_SIGNATURE + ): + continue + cr3_value = int.from_bytes( + physical_layer.read( + offset + constants.windows.PROCESSOR_START_BLOCK_CR3_OFFSET, 0x8 + ), + "little", + ) - # Compare previously observed valid page table address that's stored in vlayer._initial_entry - # with PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3 - # which was observed to be an invalid page address, so add 1 (to make it valid too) - if (cr3_value + 1) != vlayer._initial_entry: + # Compare previously observed valid page table address that's stored in vlayer._initial_entry + # with PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3 + # which was observed to be an invalid page address, so add 1 (to make it valid too) + if (cr3_value + 1) != vlayer._initial_entry: + continue + potential_kernel_hint = int.from_bytes( + physical_layer.read( + offset + + constants.windows.PROCESSOR_START_BLOCK_LM_TARGET_OFFSET, + 0x8, + ), + "little", + ) + if 0x3 & potential_kernel_hint: + continue + kernel_hint = potential_kernel_hint & 0xFFFFFFFFFFFF + kernel_base = kernel_hint & (~0x1FFFFF) & 0xFFFFFFFFFFFF + break + except exceptions.InvalidAddressException: continue - potential_kernel_hint = int.from_bytes( - physical_layer.read( - offset + constants.windows.PROCESSOR_START_BLOCK_LM_TARGET_OFFSET, - 0x8, - ), - "little", - ) - if 0x3 & potential_kernel_hint: - continue - kernel_hint = potential_kernel_hint & 0xFFFFFFFFFFFF - kernel_base = kernel_hint & (~0x1FFFFF) & 0xFFFFFFFFFFFF - break if kernel_base: # Scanning 32mb in 2mb chunks for the 'ntoskrnl' base address From 7d14f4c778d1f624a35a6db103f99b9c00cfd8b0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 27 Jan 2025 14:57:39 +0100 Subject: [PATCH 964/979] remove additional_description in favor of the docstring implementation --- volatility3/framework/interfaces/plugins.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 7ad78d0ba..f763815a6 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -112,8 +112,6 @@ class PluginInterface( # Be careful with inheritance around this (We default to requiring a version which doesn't exist, so it must be set) _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" - additional_description: str = None - """Display additional description of the plugin after the description of the arguments. See: https://docs.python.org/3/library/argparse.html#epilog""" def __init__( self, From d2859f2390070111f42e470dc9e1c6d6ad38a802 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 27 Jan 2025 15:00:03 +0100 Subject: [PATCH 965/979] remove additional_description in favor of the docstring implementation --- volatility3/cli/__init__.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index b57d9a3f3..020dac2d2 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -363,11 +363,20 @@ class CommandLine: metavar="PLUGIN", ) for plugin in sorted(plugin_list): + # First line of a plugin docstring will be the short description for -h + # Following lines will be the additional description (argparse epilog) + short_help = additional_help = None + if plugin_list[plugin].__doc__ is not None: + doc_split = plugin_list[plugin].__doc__.strip().split("\n", 1) + short_help = doc_split[0] + if len(doc_split) > 1: + additional_help = doc_split[1].strip() + plugin_parser = subparser.add_parser( plugin, - help=plugin_list[plugin].__doc__, - description=plugin_list[plugin].__doc__, - epilog=plugin_list[plugin].additional_description, + help=short_help, + description=short_help, + epilog=additional_help, ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) From 08f03642807056c0f3857eaa78cd4053fc7f391b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 27 Jan 2025 15:02:02 +0100 Subject: [PATCH 966/979] adapt plugins docstring to fit short and additional help format --- volatility3/framework/plugins/configwriter.py | 4 ++-- volatility3/framework/plugins/linux/modxview.py | 4 ++-- volatility3/framework/plugins/linux/pstree.py | 3 +-- volatility3/framework/plugins/mac/mount.py | 4 ++-- volatility3/framework/plugins/mac/pstree.py | 3 +-- volatility3/framework/plugins/timeliner.py | 4 ++-- volatility3/framework/plugins/windows/pstree.py | 3 +-- volatility3/framework/plugins/windows/psxview.py | 7 ++++--- volatility3/framework/plugins/windows/registry/hivescan.py | 3 +-- volatility3/framework/plugins/windows/scheduled_tasks.py | 5 ++--- 10 files changed, 18 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/configwriter.py b/volatility3/framework/plugins/configwriter.py index eca01a84a..a567a6acd 100644 --- a/volatility3/framework/plugins/configwriter.py +++ b/volatility3/framework/plugins/configwriter.py @@ -14,8 +14,8 @@ vollog = logging.getLogger(__name__) class ConfigWriter(plugins.PluginInterface): - """Runs the automagics and both prints and outputs configuration in the - output directory.""" + """Runs the automagics and both prints and outputs configuration in the \ +output directory.""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index c74bf28e8..0dd503829 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -15,8 +15,8 @@ vollog = logging.getLogger(__name__) class Modxview(interfaces.plugins.PluginInterface): - """Centralize lsmod, check_modules and hidden_modules results to efficiently - spot modules presence and taints.""" + """Centralize lsmod, check_modules and hidden_modules results to efficiently \ +spot modules presence and taints.""" _version = (1, 0, 0) _required_framework_version = (2, 17, 0) diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index 74e172139..fd28fcbbd 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -9,8 +9,7 @@ from volatility3.plugins.linux import pslist class PsTree(interfaces.plugins.PluginInterface): - """Plugin for listing processes in a tree based on their parent process - ID.""" + """Plugin for listing processes in a tree based on their parent process ID.""" _required_framework_version = (2, 13, 0) _version = (1, 1, 1) diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index 1a1e33571..0f3aa745c 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -11,8 +11,8 @@ from volatility3.framework.symbols import mac class Mount(plugins.PluginInterface): - """A module containing a collection of plugins that produce data typically - found in Mac's mount command""" + """A module containing a collection of plugins that produce data typically \ +found in Mac's mount command""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/mac/pstree.py b/volatility3/framework/plugins/mac/pstree.py index e62d5eb72..ad5bb309b 100644 --- a/volatility3/framework/plugins/mac/pstree.py +++ b/volatility3/framework/plugins/mac/pstree.py @@ -10,8 +10,7 @@ from volatility3.plugins.mac import pslist class PsTree(plugins.PluginInterface): - """Plugin for listing processes in a tree based on their parent process - ID.""" + """Plugin for listing processes in a tree based on their parent process ID.""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 0f4064d79..6000704eb 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -41,8 +41,8 @@ class TimeLinerInterface(metaclass=abc.ABCMeta): class Timeliner(interfaces.plugins.PluginInterface): - """Runs all relevant plugins that provide time related information and - orders the results by time.""" + """Runs all relevant plugins that provide time related information and \ +orders the results by time.""" _required_framework_version = (2, 0, 0) _version = (1, 1, 0) diff --git a/volatility3/framework/plugins/windows/pstree.py b/volatility3/framework/plugins/windows/pstree.py index 2be96277c..4f3fe0455 100644 --- a/volatility3/framework/plugins/windows/pstree.py +++ b/volatility3/framework/plugins/windows/pstree.py @@ -14,8 +14,7 @@ vollog = logging.getLogger(__name__) class PsTree(interfaces.plugins.PluginInterface): - """Plugin for listing processes in a tree based on their parent process - ID.""" + """Plugin for listing processes in a tree based on their parent process ID.""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index b5ddd2ee5..aa379bdc5 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -21,9 +21,10 @@ vollog = logging.getLogger(__name__) class PsXView(plugins.PluginInterface): - """Lists all processes found via four of the methods described in \"The Art of Memory Forensics,\" which may help - identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this - plugin's output in a terminal.""" + """Lists all processes found via four of the methods described in \"The Art of Memory Forensics\" which may help \ +identify processes that are trying to hide themselves. + +We recommend using -r pretty if you are looking at this plugin's output in a terminal.""" # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the functionality # which the original plugin used to do it. diff --git a/volatility3/framework/plugins/windows/registry/hivescan.py b/volatility3/framework/plugins/windows/registry/hivescan.py index 7b3c0b622..6e0171a78 100644 --- a/volatility3/framework/plugins/windows/registry/hivescan.py +++ b/volatility3/framework/plugins/windows/registry/hivescan.py @@ -12,8 +12,7 @@ from volatility3.plugins.windows import poolscanner, bigpools class HiveScan(interfaces.plugins.PluginInterface): - """Scans for registry hives present in a particular windows memory - image.""" + """Scans for registry hives present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) _version = (1, 0, 0) diff --git a/volatility3/framework/plugins/windows/scheduled_tasks.py b/volatility3/framework/plugins/windows/scheduled_tasks.py index 6dd5613c4..31aaec4f0 100644 --- a/volatility3/framework/plugins/windows/scheduled_tasks.py +++ b/volatility3/framework/plugins/windows/scheduled_tasks.py @@ -1099,9 +1099,8 @@ class DynamicInfo: class ScheduledTasks(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): - """Decodes scheduled task information from the Windows registry, including - information about triggers, actions, run times, and creation times. - """ + """Decodes scheduled task information from the Windows registry, including \ +information about triggers, actions, run times, and creation times.""" _required_framework_version = (2, 11, 0) _version = (1, 0, 0) From f64291faec4991bbe13f0e5cda655ae463b1f3bb Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 27 Jan 2025 17:44:18 +0100 Subject: [PATCH 967/979] split help on two consecutive newlines --- volatility3/cli/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 020dac2d2..a41cf95a3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -363,12 +363,13 @@ class CommandLine: metavar="PLUGIN", ) for plugin in sorted(plugin_list): - # First line of a plugin docstring will be the short description for -h - # Following lines will be the additional description (argparse epilog) + # First line of a plugin docstring will be the short description for -h. + # Text after the first two consecutive new lines will be + # the additional description (argparse epilog). short_help = additional_help = None if plugin_list[plugin].__doc__ is not None: - doc_split = plugin_list[plugin].__doc__.strip().split("\n", 1) - short_help = doc_split[0] + doc_split = plugin_list[plugin].__doc__.split("\n\n", 1) + short_help = doc_split[0].strip() if len(doc_split) > 1: additional_help = doc_split[1].strip() From 105df4e140767045e51c69c9a24ad17d231564bc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 27 Jan 2025 22:29:59 +0000 Subject: [PATCH 968/979] Windows: Fix vadyarascan typo --- volatility3/framework/plugins/windows/vadyarascan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 2e9cc44ea..9758c5994 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -84,7 +84,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): if not vad_maps_to_scan: vollog.warning( - f"No VADs were found for task {task.UniqueProcessID}, not scanning" + f"No VADs were found for task {task.UniqueProcessId}, not scanning" ) continue From 74a834b6de089a0ba8cca62d9e86314996faa9fb Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 09:22:58 +1100 Subject: [PATCH 969/979] linux: vfsmount: improve kernel implementation detection --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f612dfc3b..ec79b0203 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1575,13 +1575,11 @@ class vfsmount(objects.StructType): # the 'mnt_parent' member was relocated from the 'vfsmount' struct to the newly # introduced 'mount' struct. - Alternatively, vmlinux.has_type('mount') can be used here but it is faster. - Returns: 'True' if the kernel lacks the 'mount' struct, typically indicating kernel < 3.3. """ - return self.has_member("mnt_parent") + return not self._context.symbol_space.has_type("mount") def is_equal(self, vfsmount_ptr) -> bool: """Helper to make sure it is comparing two pointers to 'vfsmount'. From 53364f7dab197b4d3b83de259cc7ff632016e8ad Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 10:41:00 +1100 Subject: [PATCH 970/979] linux: LinuxUtilities: Revert do_get_path() typing --- volatility3/framework/symbols/linux/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 4634a2bfb..af0697c10 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -6,7 +6,7 @@ import contextlib import functools import logging from abc import ABC, abstractmethod -from typing import Iterator, List, Tuple, Optional +from typing import Iterator, List, Tuple, Optional, Union import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3 import framework @@ -133,7 +133,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt) @classmethod - def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> str: + def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> Union[None, str]: """Returns a pathname of the mount point or file It mimics the Linux kernel prepend_path function. From 3bd1c70a9bc3732bf4ab4131c4ee5170cffaa945 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 11:31:06 +1100 Subject: [PATCH 971/979] linux: kmsg plugin: improve return types --- volatility3/framework/plugins/linux/kmsg.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 248e37dd8..a5ce84ae9 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -5,7 +5,7 @@ import re import logging from abc import ABC, abstractmethod from enum import Enum -from typing import Generator, Iterator, List, Tuple +from typing import Generator, Iterator, List, Tuple, Union from volatility3.framework import ( class_subclasses, @@ -135,10 +135,11 @@ class ABCKmsg(ABC): bool: True if the kernel being analyzed fulfill the class requirements. """ - def get_string(self, addr: int, length: int) -> str: + def get_string(self, addr: int, length: int) -> Union[str, None]: layer = self._context.layers[self.layer_name] if not layer.is_valid(addr, length): - return "" + vollog.error("Failed to read log record at address 0x%x", addr) + return None txt = layer.read(addr, length) @@ -268,7 +269,7 @@ class Kmsg_3_5_to_3_11(ABCKmsg): def _get_log_struct_name(self): return "log" - def get_text_from_log(self, msg) -> str: + def get_text_from_log(self, msg) -> Union[str, None]: log_struct_name = self._get_log_struct_name() log_struct_size = self.vmlinux.get_type(log_struct_name).size msg_offset = msg.vol.offset + log_struct_size @@ -277,7 +278,8 @@ class Kmsg_3_5_to_3_11(ABCKmsg): def get_log_lines(self, msg) -> Generator[str, None, None]: if msg.text_len > 0: text = self.get_text_from_log(msg) - yield from text.splitlines() + if text: + yield from text.splitlines() def get_dict_lines(self, msg) -> Generator[str, None, None]: if msg.dict_len == 0: @@ -412,7 +414,7 @@ class Kmsg_5_10_to_(ABCKmsg): def symtab_checks(cls, vmlinux) -> bool: return vmlinux.has_symbol("prb") - def get_text_from_data_ring(self, text_data_ring, desc, info) -> str: + def get_text_from_data_ring(self, text_data_ring, desc, info) -> Union[str, None]: text_data_sz = text_data_ring.size_bits text_data_mask = 1 << text_data_sz @@ -440,7 +442,8 @@ class Kmsg_5_10_to_(ABCKmsg): def get_log_lines(self, text_data_ring, desc, info) -> Generator[str, None, None]: text = self.get_text_from_data_ring(text_data_ring, desc, info) - yield from text.splitlines() + if text: + yield from text.splitlines() def get_dict_lines(self, info) -> Generator[str, None, None]: dict_text = utility.array_to_string(info.dev_info.subsystem) From 4466d9accda1fbb125a34292422b3ad046e5bd2f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 11:47:01 +1100 Subject: [PATCH 972/979] linux: kmsg plugin: log warning instead of error --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index a5ce84ae9..849060d3c 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -138,7 +138,7 @@ class ABCKmsg(ABC): def get_string(self, addr: int, length: int) -> Union[str, None]: layer = self._context.layers[self.layer_name] if not layer.is_valid(addr, length): - vollog.error("Failed to read log record at address 0x%x", addr) + vollog.warning("Failed to read log record at address 0x%x", addr) return None txt = layer.read(addr, length) From c2f2fc6b42899c98b2e25700c77fe1152a1ee393 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 18:25:58 +1100 Subject: [PATCH 973/979] linux: vmcoreinfo API: fix single letter variable --- volatility3/framework/symbols/linux/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0a6448528..846928c3a 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -903,7 +903,7 @@ class VMCoreInfo(interfaces.configuration.VersionableInterface): """Converts the input VMCoreInfo data buffer into a dictionary""" # Ensure the whole buffer is printable - if not all(c in string.printable.encode() for c in vmcoreinfo_data): + if not all(char in string.printable.encode() for char in vmcoreinfo_data): # Abort, we are in the wrong place return None From f6a22eed05a7ba3784df47df77c0d8f8643e9f8c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 18:44:18 +1100 Subject: [PATCH 974/979] linux: vmcoreinfo API: significantly improve the performance when validating each character, from O(n) to O(1).. where n is 100 chars long --- volatility3/framework/symbols/linux/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 846928c3a..afdfee39c 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -903,7 +903,8 @@ class VMCoreInfo(interfaces.configuration.VersionableInterface): """Converts the input VMCoreInfo data buffer into a dictionary""" # Ensure the whole buffer is printable - if not all(char in string.printable.encode() for char in vmcoreinfo_data): + printable_bytes_set = set(string.printable.encode()) + if not all(byte in printable_bytes_set for byte in vmcoreinfo_data): # Abort, we are in the wrong place return None From 8b6c00e64f016ac5a08a51c4fefe19a78c8c270a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 18:48:42 +1100 Subject: [PATCH 975/979] linux: vmcoreinfo plugin: simplify hex conversion --- volatility3/framework/plugins/linux/vmcoreinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/vmcoreinfo.py b/volatility3/framework/plugins/linux/vmcoreinfo.py index 0f07f8589..658626014 100644 --- a/volatility3/framework/plugins/linux/vmcoreinfo.py +++ b/volatility3/framework/plugins/linux/vmcoreinfo.py @@ -39,7 +39,7 @@ class VMCoreInfo(plugins.PluginInterface): ): for key, value in vmcoreinfo.items(): if key.startswith("SYMBOL(") or key == "KERNELOFFSET": - value = f"0x{value:x}" + value = hex(value) else: value = str(value) From 5d2819a98dccfe7e13dd3c05b93e34a630ff9a2a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 20:25:22 +1100 Subject: [PATCH 976/979] linux: elf: fix imports --- .../framework/symbols/linux/extensions/elf.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index c333021d6..969a08a47 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -2,16 +2,11 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Dict, Tuple, Optional import logging +from typing import Dict, Optional, Tuple -from volatility3.framework import constants -from volatility3.framework.constants.linux import ( - ELF_IDENT, - ELF_CLASS, - KSYM_NAME_LEN, -) -from volatility3.framework import objects, interfaces, exceptions +from volatility3.framework import constants, exceptions, interfaces, objects +from volatility3.framework.constants import linux as linux_constants vollog = logging.getLogger(__name__) @@ -64,13 +59,13 @@ class elf(objects.StructType): ei_class = self._context.object( symbol_table_name + constants.BANG + "unsigned char", layer_name=layer_name, - offset=object_info.offset + ELF_IDENT.EI_CLASS, + offset=object_info.offset + linux_constants.ELF_IDENT.EI_CLASS, ) - if ei_class == ELF_CLASS.ELFCLASS32: + if ei_class == linux_constants.ELF_CLASS.ELFCLASS32: self._type_prefix = "Elf32_" self._ei_class_size = 32 - elif ei_class == ELF_CLASS.ELFCLASS64: + elif ei_class == linux_constants.ELF_CLASS.ELFCLASS64: self._type_prefix = "Elf64_" self._ei_class_size = 64 else: From f3095513a14683d4ae8184503621514941fe17ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 20:26:51 +1100 Subject: [PATCH 977/979] linux: elf_sym: use a class attribute for the symbol max length --- .../framework/symbols/linux/extensions/elf.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 969a08a47..89a421402 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -312,6 +312,8 @@ class elf(objects.StructType): class elf_sym(objects.StructType): """An elf symbol entry""" + _MAX_NAME_LENGTH = linux_constants.KSYM_NAME_LEN + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._cached_strtab = None @@ -324,20 +326,13 @@ class elf_sym(objects.StructType): def cached_strtab(self, cached_strtab): self._cached_strtab = cached_strtab - def get_name(self, max_size=KSYM_NAME_LEN) -> Optional[str]: - """Returns the symbol name - - Args: - max_size: Maximum length for a symbol name string. Defaults to KSYM_NAME_LEN (512 bytes). - - Returns: - The symbol name - """ + def get_name(self) -> Optional[str]: + """Returns the symbol name""" addr = self._cached_strtab + self.st_name layer = self._context.layers[self.vol.layer_name] - name_bytes = layer.read(addr, max_size, pad=True) + name_bytes = layer.read(addr, self._MAX_NAME_LENGTH, pad=True) if not name_bytes: return None From 67a83871adac20171a862d55dcafe5ab383c024b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 20:54:38 +1100 Subject: [PATCH 978/979] linux: module object extension: Add a function to return the extended ELF symbol information (symbol object and its index) so that doesn't change the get_symbols() interface --- .../symbols/linux/extensions/__init__.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 5f4b7c170..2e06ee687 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -212,14 +212,10 @@ class module(generic.GenericIntelProcess): ) return elf_table_name - def get_symbols( + def get_symbols_ex( self, ) -> Iterable[Tuple[int, interfaces.objects.ObjectInterface]]: - """Get symbols of the module - - Yields: - A tuple containing the ELF symbol index and the corresponding ELF symbol object - """ + """Get ELF symbol objects and their index for this module""" if not self.section_strtab or self.num_symtab < 1: return None @@ -239,10 +235,18 @@ class module(generic.GenericIntelProcess): subtype=sym_type, count=self.num_symtab, ) - for elf_sym_num, elf_sym_obj in enumerate(elf_syms): + for elf_sym_index, elf_sym_obj in enumerate(elf_syms): # Prepare the symbol object for methods like get_name() elf_sym_obj.cached_strtab = self.section_strtab - yield elf_sym_num, elf_sym_obj + yield elf_sym_index, elf_sym_obj + + def get_symbols( + self, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Get ELF symbol objects for this module""" + + for _elf_sym_index, elf_sym_obj in self.get_symbols_ex(): + yield elf_sym_obj def get_symbols_names_and_addresses(self) -> Iterable[Tuple[str, int]]: """Get names and addresses for each symbol of the module @@ -251,13 +255,13 @@ class module(generic.GenericIntelProcess): A tuple for each symbol containing the symbol name and its corresponding value """ layer = self._context.layers[self.vol.layer_name] - for _sym_num, sym in self.get_symbols(): - sym_name = sym.get_name() + for elf_sym_obj in self.get_symbols(): + sym_name = elf_sym_obj.get_name() if not sym_name: continue # Normalize sym.st_value offset, which is an address pointing to the symbol value - sym_address = sym.st_value & layer.address_mask + sym_address = elf_sym_obj.st_value & layer.address_mask yield (sym_name, sym_address) def get_symbol(self, wanted_sym_name) -> Optional[int]: From 981e09756791f9fab0975f093da87be089a43fda Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 21:03:33 +1100 Subject: [PATCH 979/979] linux: module object extension: Revert the changes to get the ELF symbol index --- .../symbols/linux/extensions/__init__.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 2e06ee687..8893e6e52 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -212,10 +212,8 @@ class module(generic.GenericIntelProcess): ) return elf_table_name - def get_symbols_ex( - self, - ) -> Iterable[Tuple[int, interfaces.objects.ObjectInterface]]: - """Get ELF symbol objects and their index for this module""" + def get_symbols(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Get ELF symbol objects for this module""" if not self.section_strtab or self.num_symtab < 1: return None @@ -235,17 +233,9 @@ class module(generic.GenericIntelProcess): subtype=sym_type, count=self.num_symtab, ) - for elf_sym_index, elf_sym_obj in enumerate(elf_syms): + for elf_sym_obj in elf_syms: # Prepare the symbol object for methods like get_name() elf_sym_obj.cached_strtab = self.section_strtab - yield elf_sym_index, elf_sym_obj - - def get_symbols( - self, - ) -> Iterable[interfaces.objects.ObjectInterface]: - """Get ELF symbol objects for this module""" - - for _elf_sym_index, elf_sym_obj in self.get_symbols_ex(): yield elf_sym_obj def get_symbols_names_and_addresses(self) -> Iterable[Tuple[str, int]]: