From 48e29ae530ecfb65fa3311a7547c16ea77f7f5cd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 May 2022 23:44:29 +0100 Subject: [PATCH 001/130] CLI: Add support for a configuration options file --- volatility3/cli/__init__.py | 58 ++++++++++++++++++++++------ volatility3/cli/volshell/__init__.py | 51 +++++++++++++++++------- 2 files changed, 83 insertions(+), 26 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 8851e2b18..62be115d3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,7 +19,7 @@ import os import sys import tempfile import traceback -from typing import Any, Dict, Type, Union +from typing import Any, Dict, List, Tuple, Type, Union from urllib import parse, request import volatility3.plugins @@ -92,6 +92,9 @@ class CommandLine: renderers = dict([(x.name.lower(), x) for x in framework.class_subclasses(text_renderer.CLIRenderer)]) + # Load up system defaults + delayed_logs, default_config = self.load_system_defaults('vol.json') + parser = volargparse.HelpfulArgParser(add_help = False, prog = self.CLI_NAME, description = "An open-source memory forensics framework") @@ -174,6 +177,8 @@ class CommandLine: default = False, action = 'store_true') + parser.set_defaults(**default_config) + # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h'] @@ -184,17 +189,7 @@ class CommandLine: banner_output = sys.stderr banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") - if partial_args.plugin_dirs: - volatility3.plugins.__path__ = [os.path.abspath(p) - for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH - - if partial_args.symbol_dirs: - volatility3.symbols.__path__ = [os.path.abspath(p) - for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS - - if partial_args.cache_path: - constants.CACHE_PATH = partial_args.cache_path - + ### Start up logging if partial_args.log: file_logger = logging.FileHandler(partial_args.log) file_logger.setLevel(1) @@ -210,6 +205,21 @@ class CommandLine: else: console.setLevel(10 - (partial_args.verbosity - 2)) + for level, msg in delayed_logs: + vollog.log(level, msg) + + ### Alter constants if necessary + if partial_args.plugin_dirs: + volatility3.plugins.__path__ = [os.path.abspath(p) + for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH + + if partial_args.symbol_dirs: + volatility3.symbols.__path__ = [os.path.abspath(p) + for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS + + if partial_args.cache_path: + constants.CACHE_PATH = partial_args.cache_path + vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") @@ -366,6 +376,30 @@ class CommandLine: raise ValueError(f"File does not exist: {filename}") return parse.urlunparse(single_location) + def load_system_defaults(self, filename: str) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: + """Modify the main configuration based on the default configuration override""" + # Build the config path + default_config_path = os.path.join(os.path.expanduser("~"), ".config", "volatility3", filename) + if sys.platform == 'win32': + default_config_path = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3", + filename) + + delayed_logs = [] + + # Process it if the files exist + if os.path.exists(default_config_path): + result = json.load(open(default_config_path, 'rb')) + if not isinstance(result, dict): + delayed_logs.append((logging.INFO, + f'Default configuration file {default_config_path} does not contain a dictionary')) + else: + delayed_logs.append( + (logging.INFO, f"Loading default configuration options from {default_config_path}")) + delayed_logs.append((logging.DEBUG, + f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}")) + return delayed_logs, result + return delayed_logs, {} + def process_exceptions(self, excp): """Provide useful feedback if an exception occurs during a run of a plugin.""" # Ensure there's nothing in the cache diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 769e958fd..647be8a1a 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -45,6 +45,9 @@ class VolShell(cli.CommandLine): framework.require_interface_version(2, 0, 0) + # Load up system defaults + delayed_logs, default_config = self.load_system_defaults('volshell.json') + parser = argparse.ArgumentParser(prog = self.CLI_NAME, description = "A tool for interactivate forensic analysis of memory images") parser.add_argument("-c", @@ -68,13 +71,16 @@ class VolShell(cli.CommandLine): default = "", type = str) parser.add_argument("-v", "--verbosity", help = "Increase output verbosity", default = 0, action = "count") + parser.add_argument("--log", + help = "Log output to a file as well as the console", + default = None, + type = str) parser.add_argument("-o", "--output-dir", help = "Directory in which to output any generated files", default = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')), type = str) parser.add_argument("-q", "--quiet", help = "Remove progress feedback", default = False, action = 'store_true') - parser.add_argument("--log", help = "Log output to a file as well as the console", default = None, type = str) parser.add_argument("-f", "--file", metavar = 'FILE', @@ -97,6 +103,10 @@ class VolShell(cli.CommandLine): help = f"Change the default path ({constants.CACHE_PATH}) used to store the cache", default = constants.CACHE_PATH, type = str) + parser.add_argument("--offline", + help = "Do not search online for additional JSON files", + default = False, + action = 'store_true') # Volshell specific flags os_specific = parser.add_mutually_exclusive_group(required = False) @@ -108,10 +118,33 @@ class VolShell(cli.CommandLine): os_specific.add_argument("-l", "--linux", default = False, action = "store_true", help = "Run a Linux volshell") os_specific.add_argument("-m", "--mac", default = False, action = "store_true", help = "Run a Mac volshell") + parser.set_defaults(**default_config) + # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h'] partial_args, _ = parser.parse_known_args(known_args) + + ### Start up logging + if partial_args.log: + file_logger = logging.FileHandler(partial_args.log) + file_logger.setLevel(1) + file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S', + fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + file_logger.setFormatter(file_formatter) + rootlog.addHandler(file_logger) + vollog.info("Logging started") + if partial_args.verbosity < 3: + if partial_args.verbosity < 1: + sys.tracebacklimit = None + console.setLevel(30 - (partial_args.verbosity * 10)) + else: + console.setLevel(10 - (partial_args.verbosity - 2)) + + for level, msg in delayed_logs: + vollog.log(level, msg) + + ### Alter constants if necessary if partial_args.plugin_dirs: volatility3.plugins.__path__ = [os.path.abspath(p) for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH @@ -126,23 +159,13 @@ class VolShell(cli.CommandLine): vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") - if partial_args.log: - file_logger = logging.FileHandler(partial_args.log) - file_logger.setLevel(0) - file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S', - fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s') - file_logger.setFormatter(file_formatter) - vollog.addHandler(file_logger) - vollog.info("Logging started") - - if partial_args.verbosity < 3: - console.setLevel(30 - (partial_args.verbosity * 10)) - else: - console.setLevel(10 - (partial_args.verbosity - 2)) if partial_args.clear_cache: framework.clear_cache() + if partial_args.offline: + constants.OFFLINE = partial_args.offline + # Do the initialization ctx = contexts.Context() # Construct a blank context failures = framework.import_files(volatility3.plugins, From 9ea1a960cdece1c6897735adc58f592f73c44e86 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 30 May 2022 01:34:19 +0100 Subject: [PATCH 002/130] CLI: Synchonize volshell code a little better courtesy of @digitalisx --- volatility3/cli/volshell/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 647be8a1a..e2e7bc394 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -15,12 +15,14 @@ from volatility3.cli.volshell import generic, linux, mac, windows from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins # Make sure we log everything + +rootlog = logging.getLogger() vollog = logging.getLogger() vollog.setLevel(0) -# Trim the console down by default console = logging.StreamHandler() console.setLevel(logging.WARNING) formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') +# Trim the console down by default console.setFormatter(formatter) vollog.addHandler(console) From 2e4c257e15e83b33fb40cd9369c90dac2c1786e9 Mon Sep 17 00:00:00 2001 From: ikelos Date: Mon, 30 May 2022 01:53:07 +0100 Subject: [PATCH 003/130] Update volatility3/cli/__init__.py Co-authored-by: Donghyun Kim --- volatility3/cli/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 62be115d3..6881311bf 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -388,7 +388,8 @@ class CommandLine: # Process it if the files exist if os.path.exists(default_config_path): - result = json.load(open(default_config_path, 'rb')) + with open(default_config_path, 'rb') as config_json: + result = json.load(config_json) if not isinstance(result, dict): delayed_logs.append((logging.INFO, f'Default configuration file {default_config_path} does not contain a dictionary')) From 0b33a79dc6465884105a60cb3d2054c61885a2eb Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Wed, 2 Nov 2022 21:10:51 +0000 Subject: [PATCH 004/130] Report IRP entries that point inside a hidden module. This is a common rootkit technique. --- volatility3/framework/plugins/windows/driverirp.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index 7f9bc6b08..4013bdb8f 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -48,7 +48,10 @@ class DriverIrp(interfaces.plugins.PluginInterface): for i, address in enumerate(driver.MajorFunction): module_symbols = collection.get_module_symbols_by_absolute_location(address) + module_found = False + for module_name, symbol_generator in module_symbols: + module_found = True symbols_found = False for symbol in symbol_generator: @@ -60,6 +63,11 @@ class DriverIrp(interfaces.plugins.PluginInterface): yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], format_hints.Hex(address), module_name, renderers.NotAvailableValue())) + if not module_found: + yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], + format_hints.Hex(address), renderers.NotAvailableValue(), renderers.NotAvailableValue())) + + def run(self): return renderers.TreeGrid([ From 7469872c8bfcd8d7a84637ede54180008e624b0a Mon Sep 17 00:00:00 2001 From: RuBublik Date: Wed, 17 May 2023 21:41:37 +0300 Subject: [PATCH 005/130] added 'PoolConstraint' of Thread objects to 'PoolScanner.default_constraints' as part of adding support for thread pool tag scanning --- .../framework/plugins/windows/poolscanner.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index e131c5f78..028241bb8 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -224,6 +224,20 @@ class PoolScanner(plugins.PluginInterface): size=(600, None), page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), + # threads on windows before windows8 + PoolConstraint(b'Thr\xe5', # -> “protected” allocation, MSB is set. + type_name = symbol_table + constants.BANG + "_ETHREAD", + object_type="Thread", + size = (600, None), # -> 0x0258 - size of strcut in win5.1 + page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + ), + # threads on windows starting with windows8 + PoolConstraint(b'Thre', + type_name = symbol_table + constants.BANG + "_ETHREAD", + object_type="Thread", + size = (600, None), # -> 0x0258 - size of strcut in win5.1 + page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + ), # files on windows before windows 8 PoolConstraint( b"Fil\xe5", From 550d1096ebe2c2d1f22a6c6446495051d54e84ec Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sat, 20 May 2023 14:20:15 +0300 Subject: [PATCH 006/130] temporary fix to ETHREAD class, add 'is_valid' method --- .../framework/symbols/windows/extensions/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ba00a4053..cfaa3aced 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -492,9 +492,13 @@ class KMUTANT(objects.StructType, pool.ExecutiveObject): return header.NameInfo.Name.String # type: ignore -class ETHREAD(objects.StructType): +class ETHREAD(objects.StructType, pool.ExecutiveObject): """A class for executive thread objects.""" + def is_valid(self) -> bool: + """Determine if the object is valid.""" + return True # temporary, need to implement validation later. + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 0fd475e10bb9644c28abef36c8f4c7dabec318d6 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sat, 20 May 2023 15:49:09 +0300 Subject: [PATCH 007/130] added permanent implementation for 'ETHREAD.is_valid' --- .../symbols/windows/extensions/__init__.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index cfaa3aced..e17128897 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -497,7 +497,30 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): def is_valid(self) -> bool: """Determine if the object is valid.""" - return True # temporary, need to implement validation later. + + try: + + # validation by thread creation time: + ctime = self.get_create_time() + if not isinstance(ctime, datetime.datetime): + return False + + # validation by parent process: + own_proc = self.owning_process() + # return own_proc.is_valid() + if own_proc.UniqueProcessId % 4 != 0: # NT pids are divisible by 4 + return False + + # passed all valitations + return True + except: + return False + + def get_create_time(self): + return conversion.wintime_to_datetime(self.CreateTime.QuadPart) + + def get_exit_time(self): + return conversion.wintime_to_datetime(self.ExitTime.QuadPart) def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 601870829e33c32688fbecccede7423f1e3b7839 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:12:46 +0300 Subject: [PATCH 008/130] twicked thread constraint for edge cases --- volatility3/framework/plugins/windows/poolscanner.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 028241bb8..ce1015789 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -222,17 +222,21 @@ class PoolScanner(plugins.PluginInterface): type_name=symbol_table + constants.BANG + "_EPROCESS", object_type="Process", size=(600, None), + skip_type_test = True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows before windows8 - PoolConstraint(b'Thr\xe5', # -> “protected” allocation, MSB is set. + PoolConstraint( + b'Thr\xe5', # -> “protected” allocation, MSB is set. type_name = symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", size = (600, None), # -> 0x0258 - size of strcut in win5.1 + skip_type_test = True, page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE ), # threads on windows starting with windows8 - PoolConstraint(b'Thre', + PoolConstraint( + b'Thre', type_name = symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", size = (600, None), # -> 0x0258 - size of strcut in win5.1 From 046c8e4d1e7f3bf9d6857de5601ebd4ab089af6d Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:18:02 +0300 Subject: [PATCH 009/130] fix of is_valid - removed reliace on owning _eprocess and added exclusion for system process (does not have creation time) --- .../symbols/windows/extensions/__init__.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index e17128897..fe4884dbc 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -499,18 +499,21 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): """Determine if the object is valid.""" try: - - # validation by thread creation time: - ctime = self.get_create_time() - if not isinstance(ctime, datetime.datetime): - return False - - # validation by parent process: - own_proc = self.owning_process() - # return own_proc.is_valid() - if own_proc.UniqueProcessId % 4 != 0: # NT pids are divisible by 4 + + # validation by TID: + if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 return False + # validation by PID of parent process: + if self.Cid.UniqueProcess % 4 != 0: + return False + + # validation by thread creation time: + if self.Cid.UniqueProcess != 4: # The System process (PID 4) has no create time + ctime = self.get_create_time() + if not isinstance(ctime, datetime.datetime): + return False + # passed all valitations return True except: From fab818b48d950a1210c77e97a4da09cf24f2bd03 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 18:24:01 +0300 Subject: [PATCH 010/130] added thrdscan plugin to utilize added support for ethread pool tag scanning --- .../framework/plugins/windows/thrdscan.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 volatility3/framework/plugins/windows/thrdscan.py diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py new file mode 100644 index 000000000..b0b80fe46 --- /dev/null +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -0,0 +1,107 @@ +## +## plugin for testing addition of threads scan support to poolscanner.py +## +import logging +import datetime +from typing import Iterable + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import poolscanner + +vollog = logging.getLogger(__name__) + + +class ThrdScan(interfaces.plugins.PluginInterface): + """Scans for windows threads.""" + + # cuz installed Framework interface version 2 + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) + ), + ] + + @classmethod + def scan_threads( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + ) -> Iterable[interfaces.objects.ObjectInterface]: + """Scans for threads using the poolscanner module and constraints. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + symbol_table: The name of the table containing the kernel symbols + + Returns: + A list of _ETHREAD objects found by scanning memory for the "Thre" / "Thr\\xE5" pool signatures + """ + + constraints = poolscanner.PoolScanner.builtin_constraints( + symbol_table, [b"Thr\xe5", b"Thre"] + ) + + for result in poolscanner.PoolScanner.generate_pool_scan( + context, layer_name, symbol_table, constraints + ): + _constraint, mem_object, _header = result + yield mem_object + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + + for ethread in self.scan_threads( + self.context, kernel.layer_name, kernel.symbol_table_name + ): + try: + thread_offset = ethread.vol.offset + owner_proc_pid = ethread.Cid.UniqueProcess + thread_tid = ethread.Cid.UniqueThread + thread_start_addr = ethread.StartAddress + thread_create_time = ethread.get_create_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_exit_time = ethread.get_exit_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + except (ValueError, exceptions.InvalidAddressException): + vollog.debug( + "Thread :{}, invalid address {} in layer {}".format( + thread_tid, thread_start_addr, kernel.layer_name + ) + ) + continue + + yield ( + 0, + ( + hex(format_hints.Hex(thread_offset)), + owner_proc_pid, + thread_tid, + hex(thread_start_addr), + str(thread_create_time) if isinstance(thread_create_time, datetime.datetime) else "", + str(thread_exit_time) if isinstance(thread_exit_time, datetime.datetime) else "" + ) + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", str), + ("PID", int), + ("TID", int), + ("Start Address", str), + ("Create Time", str), + ("Exit Time", str), + ], + self._generator(), + ) \ No newline at end of file From 5749b3ef5283fc36d064a766e5228dc2eef5b402 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 21 May 2023 19:22:22 +0300 Subject: [PATCH 011/130] added test for thrdscan plugin --- test/test_volatility.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index aaad615bc..5e41899da 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -189,6 +189,16 @@ def test_windows_svcscan(image, volatility, python): assert rc == 0 +def test_windows_thrdscan(image, volatility, python): + rc, out, err = runvol_plugin("windows.thrdscan.ThrdScan", image, volatility, python) + # find pid 4 (of system process) which starts with lowest tids + assert out.find(b"\t4\t8") != -1 + assert out.find(b"\t4\t12") != -1 + assert out.find(b"\t4\t16") != -1 + #assert out.find(b"this raieses AssertionError") != -1 + assert rc == 0 + + def test_windows_privileges(image, volatility, python): rc, out, err = runvol_plugin( "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"] From 950a76d1e0dfb460215c3b6e5dc1108a0421863f Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 11:17:23 +0300 Subject: [PATCH 012/130] fixed typo --- .../framework/symbols/windows/extensions/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index fe4884dbc..4ad74f61a 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -514,10 +514,11 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False - # passed all valitations - return True - except: + except exceptions.InvalidAddressException: return False + + # passed all validations + return True def get_create_time(self): return conversion.wintime_to_datetime(self.CreateTime.QuadPart) From bef5149ba64e640c39e83774e0fa969893d5ee18 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 11:39:38 +0300 Subject: [PATCH 013/130] changed TreeGrid yielded types to specific simpletypes instead of str --- .../framework/plugins/windows/thrdscan.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index b0b80fe46..8b445991f 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -84,24 +84,24 @@ class ThrdScan(interfaces.plugins.PluginInterface): yield ( 0, ( - hex(format_hints.Hex(thread_offset)), + format_hints.Hex(thread_offset), owner_proc_pid, thread_tid, - hex(thread_start_addr), - str(thread_create_time) if isinstance(thread_create_time, datetime.datetime) else "", - str(thread_exit_time) if isinstance(thread_exit_time, datetime.datetime) else "" + format_hints.Hex(thread_start_addr), + thread_create_time, + thread_exit_time, ) ) def run(self): return renderers.TreeGrid( [ - ("Offset", str), + ("Offset", format_hints.Hex), ("PID", int), ("TID", int), - ("Start Address", str), - ("Create Time", str), - ("Exit Time", str), + ("Start Address", format_hints.Hex), + ("Create Time", datetime.datetime), + ("Exit Time", datetime.datetime), ], self._generator(), ) \ No newline at end of file From c6c501ecb57e1757a510d019a3752af78dda96cc Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:31:25 +0300 Subject: [PATCH 014/130] implemented generate_timeline method in ThrdScan --- .../framework/plugins/windows/thrdscan.py | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 8b445991f..ca3f4fc69 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -9,15 +9,16 @@ from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import poolscanner +from volatility3.plugins import timeliner vollog = logging.getLogger(__name__) -class ThrdScan(interfaces.plugins.PluginInterface): +class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -93,15 +94,40 @@ class ThrdScan(interfaces.plugins.PluginInterface): ) ) + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + row_dict = {} + ( + row_dict["Offset"], + row_dict["PID"], + row_dict["TID"], + row_dict["StartAddress"], + row_dict["CreateTime"], + row_dict["ExitTime"], + ) = row_data + + # Skip threads with no creation time + # - mainly system process threads + if not isinstance(row_dict["CreateTime"], datetime.datetime): + continue + description = (f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})") + + # yield created time, and if there is exit time, yield it too. + yield (description, timeliner.TimeLinerType.CREATED, row_dict["CreateTime"]) + if isinstance(row_dict["ExitTime"], datetime.datetime): + yield (description, timeliner.TimeLinerType.MODIFIED, row_dict["ExitTime"]) + + def run(self): return renderers.TreeGrid( [ ("Offset", format_hints.Hex), ("PID", int), ("TID", int), - ("Start Address", format_hints.Hex), - ("Create Time", datetime.datetime), - ("Exit Time", datetime.datetime), + ("StartAddress", format_hints.Hex), + ("CreateTime", datetime.datetime), + ("ExitTime", datetime.datetime), ], self._generator(), ) \ No newline at end of file From 9bdd249aadb5b60ca5e45c91c396f1c591f9b45e Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:39:12 +0300 Subject: [PATCH 015/130] added _version to ThrdScan --- volatility3/framework/plugins/windows/thrdscan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index ca3f4fc69..cbbe64988 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -19,6 +19,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) # cuz installed Framework interface version 2 _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) @classmethod def get_requirements(cls): From 5072728e897c1489e11f3422ea6974cf542d7e64 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 22 May 2023 15:50:22 +0300 Subject: [PATCH 016/130] formated with black --- .../framework/plugins/windows/thrdscan.py | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index cbbe64988..813883c2d 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -18,7 +18,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """Scans for windows threads.""" # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod @@ -33,7 +33,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) ), ] - + @classmethod def scan_threads( cls, @@ -53,7 +53,7 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """ constraints = poolscanner.PoolScanner.builtin_constraints( - symbol_table, [b"Thr\xe5", b"Thre"] + symbol_table, [b"Thr\xe5", b"Thre"] ) for result in poolscanner.PoolScanner.generate_pool_scan( @@ -69,12 +69,16 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) self.context, kernel.layer_name, kernel.symbol_table_name ): try: - thread_offset = ethread.vol.offset - owner_proc_pid = ethread.Cid.UniqueProcess - thread_tid = ethread.Cid.UniqueThread - thread_start_addr = ethread.StartAddress - thread_create_time = ethread.get_create_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object - thread_exit_time = ethread.get_exit_time() # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_offset = ethread.vol.offset + owner_proc_pid = ethread.Cid.UniqueProcess + thread_tid = ethread.Cid.UniqueThread + thread_start_addr = ethread.StartAddress + thread_create_time = ( + ethread.get_create_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object + thread_exit_time = ( + ethread.get_exit_time() + ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object except (ValueError, exceptions.InvalidAddressException): vollog.debug( "Thread :{}, invalid address {} in layer {}".format( @@ -86,13 +90,13 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) yield ( 0, ( - format_hints.Hex(thread_offset), - owner_proc_pid, - thread_tid, - format_hints.Hex(thread_start_addr), - thread_create_time, - thread_exit_time, - ) + format_hints.Hex(thread_offset), + owner_proc_pid, + thread_tid, + format_hints.Hex(thread_start_addr), + thread_create_time, + thread_exit_time, + ), ) def generate_timeline(self): @@ -112,13 +116,16 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) # - mainly system process threads if not isinstance(row_dict["CreateTime"], datetime.datetime): continue - description = (f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})") - + description = f"Thread: Tid {row_dict['TID']} in Pid {row_dict['PID']} (Offset {row_dict['Offset']})" + # yield created time, and if there is exit time, yield it too. yield (description, timeliner.TimeLinerType.CREATED, row_dict["CreateTime"]) if isinstance(row_dict["ExitTime"], datetime.datetime): - yield (description, timeliner.TimeLinerType.MODIFIED, row_dict["ExitTime"]) - + yield ( + description, + timeliner.TimeLinerType.MODIFIED, + row_dict["ExitTime"], + ) def run(self): return renderers.TreeGrid( @@ -131,4 +138,4 @@ class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) ("ExitTime", datetime.datetime), ], self._generator(), - ) \ No newline at end of file + ) From 5d32ca542c8918224f070163f02ff454f0bffffa Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 28 May 2023 20:52:26 +0300 Subject: [PATCH 017/130] fix black formatting --- .../framework/plugins/windows/poolscanner.py | 20 +++++++++---------- .../symbols/windows/extensions/__init__.py | 17 ++++++++-------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index ce1015789..5539e1e84 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -222,25 +222,25 @@ class PoolScanner(plugins.PluginInterface): type_name=symbol_table + constants.BANG + "_EPROCESS", object_type="Process", size=(600, None), - skip_type_test = True, + skip_type_test=True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows before windows8 PoolConstraint( - b'Thr\xe5', # -> “protected” allocation, MSB is set. - type_name = symbol_table + constants.BANG + "_ETHREAD", + b"Thr\xe5", # -> “protected” allocation, MSB is set. + type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size = (600, None), # -> 0x0258 - size of strcut in win5.1 - skip_type_test = True, - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + size=(600, None), # -> 0x0258 - size of strcut in win5.1 + skip_type_test=True, + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # threads on windows starting with windows8 PoolConstraint( - b'Thre', - type_name = symbol_table + constants.BANG + "_ETHREAD", + b"Thre", + type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size = (600, None), # -> 0x0258 - size of strcut in win5.1 - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE + size=(600, None), # -> 0x0258 - size of strcut in win5.1 + page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # files on windows before windows 8 PoolConstraint( diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 4ad74f61a..8790f41a7 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -499,17 +499,18 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): """Determine if the object is valid.""" try: - # validation by TID: - if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 + if self.Cid.UniqueThread % 4 != 0: # NT tids are divisible by 4 return False - + # validation by PID of parent process: if self.Cid.UniqueProcess % 4 != 0: return False - + # validation by thread creation time: - if self.Cid.UniqueProcess != 4: # The System process (PID 4) has no create time + if ( + self.Cid.UniqueProcess != 4 + ): # The System process (PID 4) has no create time ctime = self.get_create_time() if not isinstance(ctime, datetime.datetime): return False @@ -518,14 +519,14 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return False # passed all validations - return True - + return True + def get_create_time(self): return conversion.wintime_to_datetime(self.CreateTime.QuadPart) def get_exit_time(self): return conversion.wintime_to_datetime(self.ExitTime.QuadPart) - + def owning_process(self) -> interfaces.objects.ObjectInterface: """Return the EPROCESS that owns this thread.""" From 524ff59107ee857c6d4e86697fd6db7f92c05156 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Fri, 7 Jul 2023 20:22:58 +0300 Subject: [PATCH 018/130] bumped MINOR_VERSION to 2.5.2 after changes, and updated dependent thrdscan plugin's required version to this --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3a6b24ea8..09dded076 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,7 +44,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 4 # Number of changes that only add to the interface +VERSION_MINOR = 5 # Number of changes that only add to the interface VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 813883c2d..4afc29cdb 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # cuz installed Framework interface version 2 - _required_framework_version = (2, 0, 0) + # version 2.5.2 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 5, 2) _version = (1, 0, 0) @classmethod From abf1c2e03d67f02ea5c31cc9f4b16029cbbe946d Mon Sep 17 00:00:00 2001 From: RuBublik Date: Fri, 7 Jul 2023 20:25:02 +0300 Subject: [PATCH 019/130] fixed typos --- volatility3/framework/plugins/windows/poolscanner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 5539e1e84..1f70cfb8c 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -230,7 +230,7 @@ class PoolScanner(plugins.PluginInterface): b"Thr\xe5", # -> “protected” allocation, MSB is set. type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size=(600, None), # -> 0x0258 - size of strcut in win5.1 + size=(600, None), # -> 0x0258 - size of struct in win5.1 skip_type_test=True, page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), @@ -239,7 +239,7 @@ class PoolScanner(plugins.PluginInterface): b"Thre", type_name=symbol_table + constants.BANG + "_ETHREAD", object_type="Thread", - size=(600, None), # -> 0x0258 - size of strcut in win5.1 + size=(600, None), # -> 0x0258 - size of struct in win5.1 page_type=PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, ), # files on windows before windows 8 From 338238c6396aa4c53a579a61a64eb4139de6cc76 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Sun, 16 Jul 2023 00:10:31 +0300 Subject: [PATCH 020/130] fixed build number - resets when MINOR version goes up --- volatility3/framework/constants/__init__.py | 2 +- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09dded076..de1674885 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 4afc29cdb..6e19f9bd5 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # version 2.5.2 adds support for scanning for 'Ethread' structures by pool tags - _required_framework_version = (2, 5, 2) + # version 2.5.0 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 5, 0) _version = (1, 0, 0) @classmethod From 62466c7953cb0264bc3bb491a694a55732115d7a Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 3 Oct 2023 21:35:37 +0300 Subject: [PATCH 021/130] fixed merge conflicts with 'volatilityfoundation:develop' branch - bumped VERSION_PATCH --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index de1674885..c3ebaca27 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 57d995a81d3b6a9d1843439477c0be6933215df0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 8 Oct 2023 03:15:23 +0200 Subject: [PATCH 022/130] manually instantiate queue_entry for tasks symbol --- volatility3/framework/plugins/mac/pslist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 88045a277..c0e149fc2 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object_from_symbol(symbol_name="tasks") + queue_entry = kernel.object("queue_entry", kernel.get_symbol("tasks").address) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 6e5d41c38b3c494b9a43d5a7fca515aa84e1b6d4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 8 Oct 2023 03:37:17 +0200 Subject: [PATCH 023/130] manually instantiate queue_entry for tasks symbol --- volatility3/framework/plugins/mac/pslist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index c0e149fc2..e8c490dff 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object("queue_entry", kernel.get_symbol("tasks").address) + queue_entry = kernel.object(object_type="queue_entry", offset=kernel.get_symbol("tasks").address) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 3a656266711b21951ea8c48fe9d8ac4a4cf64775 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 23 Oct 2023 17:47:57 +0200 Subject: [PATCH 024/130] black formatting --- volatility3/framework/plugins/mac/pslist.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index e8c490dff..9835644b8 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -188,7 +188,9 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object(object_type="queue_entry", offset=kernel.get_symbol("tasks").address) + queue_entry = kernel.object( + object_type="queue_entry", offset=kernel.get_symbol("tasks").address + ) seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): From 604c23bcbc88679a48d1348ef3c8d2163726ab44 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Thu, 14 Dec 2023 14:50:17 +0100 Subject: [PATCH 025/130] Import Address Table Plugin --- volatility3/framework/plugins/windows/iat.py | 132 +++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 volatility3/framework/plugins/windows/iat.py diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py new file mode 100644 index 000000000..1cf51cfa2 --- /dev/null +++ b/volatility3/framework/plugins/windows/iat.py @@ -0,0 +1,132 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + +import logging +import io +from typing import Callable, List +from volatility3.framework.symbols import intermed +from volatility3.framework import renderers, interfaces, exceptions, constants +from volatility3.framework.configuration import requirements +from volatility3.plugins.windows import pslist +from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.symbols.windows.extensions import pe +import pefile + +vollog = logging.getLogger(__name__) + + +class IAT(interfaces.plugins.PluginInterface): + """Extract Import Address Table to list API (functions) used by a program contained in external libraries""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process ID to include (all other processes are excluded)", + optional=True, + ), + ] + + def _generator(self, procs): + kernel = self.context.modules[self.config["kernel"]] + + for proc in procs: + try: + proc_id = proc.UniqueProcessId + proc_layer_name = proc.add_process_layer() + peb = self.context.object( + kernel.symbol_table_name + constants.BANG + "_PEB", + layer_name=proc_layer_name, + offset=proc.Peb, + ) + + if proc_layer_name is None: + raise TypeError("Layer must be a string not None") + + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "windows", + "pe", + class_types=pe.class_types, + ) + pe_data = io.BytesIO() + + dos_header = self.context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=peb.ImageBaseAddress, + layer_name=proc_layer_name, + ) + + for offset, data in dos_header.reconstruct(): + pe_data.seek(offset) + pe_data.write(data) + + pe_obj = pefile.PE(data=pe_data.getvalue(), fast_load=True) + pe_obj.parse_data_directories( + [pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_IMPORT"]] + ) + if hasattr(pe_obj, "DIRECTORY_ENTRY_IMPORT"): + for entry in pe_obj.DIRECTORY_ENTRY_IMPORT: + dll_entry = entry.dll + if dll_entry: + dll_entry = dll_entry.decode() + else: + dll_entry = renderers.NotAvailableValue + + # Iterate over imported functions + for imp in entry.imports: + import_name = imp.name + if import_name: + import_name = imp.name.decode() + else: + import_name = renderers.NotAvailableValue() + yield ( + 0, + ( + proc_id, + proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ), + dll_entry, + import_name, + ), + ) + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) + ) + continue + + def run(self): + kernel = self.context.modules[self.config["kernel"]] + + return renderers.TreeGrid( + [("PID", int), ("Process", str), ("Library", str), ("Function", str)], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=pslist.PsList.create_pid_filter( + self.config.get("pid", None) + ), + ) + ), + ) From 059f2d012896d96fa0e2d20469d7771ec27b0b6d Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 16 Dec 2023 21:19:35 +0100 Subject: [PATCH 026/130] Adding function addr + bound info. Formatting code. --- volatility3/framework/plugins/windows/iat.py | 32 +++++++++++++++----- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 1cf51cfa2..11c273859 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,16 +1,13 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -import logging -import io -from typing import Callable, List +import logging, io, pefile from volatility3.framework.symbols import intermed from volatility3.framework import renderers, interfaces, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.plugins.windows import pslist -from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.renderers import format_hints from volatility3.framework.symbols.windows.extensions import pe -import pefile vollog = logging.getLogger(__name__) @@ -86,6 +83,12 @@ class IAT(interfaces.plugins.PluginInterface): else: dll_entry = renderers.NotAvailableValue + bound = True + # Initially set to 0 if not bound + time_date_stamp = entry.struct.TimeDateStamp + if not time_date_stamp: + bound = False + # Iterate over imported functions for imp in entry.imports: import_name = imp.name @@ -93,6 +96,12 @@ class IAT(interfaces.plugins.PluginInterface): import_name = imp.name.decode() else: import_name = renderers.NotAvailableValue() + function_address = ( + pe_obj.OPTIONAL_HEADER.ImageBase + imp.address + ) + if not function_address: + function_address = renderers.NotAvailableValue + yield ( 0, ( @@ -103,7 +112,9 @@ class IAT(interfaces.plugins.PluginInterface): errors="replace", ), dll_entry, + bound, import_name, + format_hints.Hex(function_address), ), ) except exceptions.InvalidAddressException as excp: @@ -118,7 +129,14 @@ class IAT(interfaces.plugins.PluginInterface): kernel = self.context.modules[self.config["kernel"]] return renderers.TreeGrid( - [("PID", int), ("Process", str), ("Library", str), ("Function", str)], + [ + ("PID", int), + ("Name", str), + ("Library", str), + ("Bound", bool), + ("Function", str), + ("Address", format_hints.Hex), + ], self._generator( pslist.PsList.list_processes( context=self.context, From 409ce95980b0e4c5a29222f4acb90b681886d16f Mon Sep 17 00:00:00 2001 From: hsarkey Date: Thu, 7 Dec 2023 14:04:44 -0500 Subject: [PATCH 027/130] Windows: Added '--refined' option to windows malfind plugin Also updated malfind to include "\x55\x48" and "\x55\x89" as part of the refined_criteria list. --- volatility3/framework/plugins/windows/malfind.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 6ed078996..1c73fdf1c 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -46,6 +46,12 @@ class Malfind(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) ), + requirements.BooleanRequirement( + name="refined", + description="Refine the output. Only show regions with an MZ header or that start with well known opcode combinations (i.e. PUSH EBP). WARNING: This can cause you to overlook regions with wiped headers or shell code blocks starting with NOP sleds, etc.. However, it will in general result in less noisy output.", + default=False, + optional=True, + ), ] @classmethod @@ -138,6 +144,9 @@ class Malfind(interfaces.plugins.PluginInterface): yield vad, data def _generator(self, procs): + # set refined criteria + refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] + # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] @@ -151,6 +160,10 @@ class Malfind(interfaces.plugins.PluginInterface): for vad, data in self.list_injections( self.context, kernel.layer_name, kernel.symbol_table_name, proc ): + # check if refined option was passed + if self.config["refined"] and data[0:2] not in refined_criteria: + continue + # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 if is_32bit_arch or proc.get_is_wow64(): architecture = "intel" From 3848fc69a6d93602b4836a87d1e69f493f7dbd49 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 25 Dec 2023 21:32:14 -0300 Subject: [PATCH 028/130] Add support for kernels earlier than version 3.11. * Add support for older ring buffer kernel implementation: - 3.5 < kernels - 3.5 <= kernels < 3.11 * Enabled support for wrapped-around ring buffers in all four kernel implementations. * Bugfix: wrapped around buffer issue with 3.11 <= kernel < 5.10. --- volatility3/framework/plugins/linux/kmsg.py | 187 ++++++++++++++------ 1 file changed, 132 insertions(+), 55 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 5136a00f6..e55d5f14e 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -1,6 +1,7 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import re import logging from abc import ABC, abstractmethod from enum import Enum @@ -9,7 +10,6 @@ from typing import Generator, Iterator, List, Tuple from volatility3.framework import ( class_subclasses, constants, - contexts, interfaces, renderers, ) @@ -143,7 +143,7 @@ class ABCKmsg(ABC): return "%lu.%06lu" % (nsec / 1000000000, (nsec % 1000000000) / 1000) def get_timestamp_in_sec_str(self, obj) -> str: - # obj could be printk_log or printk_info + # obj could be log, printk_log or printk_info return self.nsec_to_sec_str(obj.ts_nsec) def get_caller(self, obj): @@ -153,7 +153,7 @@ class ABCKmsg(ABC): if obj.has_member("caller_id"): return self.get_caller_text(obj.caller_id) else: - return "" + return renderers.NotAvailableValue() def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" @@ -161,7 +161,7 @@ class ABCKmsg(ABC): return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: - # obj could be printk_log or printk_info + # obj could be log, printk_log or printk_info return ( obj.facility, obj.level, @@ -186,39 +186,90 @@ class ABCKmsg(ABC): return str(facility) -class KmsgLegacy(ABCKmsg): - """Linux kernels prior to v5.10, the ringbuffer is initially kept in - __log_buf, and log_buf is a pointer to the former. __log_buf is declared as - a char array but it actually contains an array of printk_log structs. - The length of this array is defined in the kernel KConfig configuration via - the CONFIG_LOG_BUF_SHIFT value as a power of 2. - This can also be modified by the log_buf_len kernel boot parameter. - In SMP systems with more than 64 CPUs this ringbuffer size is dynamically - allocated according the number of CPUs based on the value of - CONFIG_LOG_CPU_MAX_BUF_SHIFT, and the log_buf pointer is updated - consequently to the new buffer. - In that case, the original static buffer in __log_buf is unused. +class Kmsg_pre_3_5(ABCKmsg): + """The kernel ring buffer (log_buf) is a char array that sequentially stores + log lines, each separated by newline (LF) characters. i.e: + <6>[ 9565.250411] line1!\n<6>[ 9565.250412] line2\n... """ @classmethod def symtab_checks(cls, vmlinux) -> bool: - return vmlinux.has_type("printk_log") + return ( + vmlinux.has_symbol("log_end") + and not vmlinux.has_symbol("log_first_idx") + and not ( + vmlinux.has_type("log") + and vmlinux.get_type("log").has_member("ts_nsec") + ) + ) - def get_text_from_printk_log(self, msg) -> str: - msg_offset = msg.vol.offset + self.vmlinux.get_type("printk_log").size + def run(self) -> Iterator[Tuple[str, str, str, str, str]]: + log_buf_ptr = self.vmlinux.object_from_symbol(symbol_name="log_buf") + log_buf_len = self.vmlinux.object_from_symbol(symbol_name="log_buf_len") + log_buf = utility.pointer_to_string(log_buf_ptr, count=log_buf_len) + log_end = self.vmlinux.object_from_symbol(symbol_name="log_end") + + if log_end > log_buf_len: + start = log_end - log_buf_len + first_half = log_buf[start:] + second_half = log_buf[:start] + log_buf = first_half + second_half + + log_buf_lines = log_buf.splitlines() + + for log_buf_line in log_buf_lines: + m = re.match(r"<(\d+)>\[\s*(\d+\.\d+)\]\s(.*?)$", log_buf_line) + if not m: + # If there was a wrap-around in the ring buffer, it will find + # remnants at the top. As those remnants do not conform to the + # expected line format, they are discarded + continue + + level_facility_str, timestamp_str, line = m.groups() + level_facility = int(level_facility_str) + # The lower 3 bit are the log level, the rest are the log facility + level = level_facility & 7 + facility = level_facility >> 3 + level_txt = self.get_level_text(level) + facility_txt = self.get_facility_text(facility) + caller = renderers.NotAvailableValue() + yield facility_txt, level_txt, timestamp_str, caller, line + + +class Kmsg_3_5_to_3_11(ABCKmsg): + """While 'log_buf' is declared as a pointer and '__log_buf' as a char array, + it essentially holds an array of 'log' structs. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return ( + vmlinux.has_type("log") + and vmlinux.get_type("log").has_member("ts_nsec") + and vmlinux.has_symbol("log_first_idx") + ) + + def _get_log_struct_name(self): + return "log" + + def get_text_from_log(self, msg) -> str: + log_struct_name = self._get_log_struct_name() + log_struct_size = self.vmlinux.get_type(log_struct_name).size + msg_offset = msg.vol.offset + log_struct_size return self.get_string(msg_offset, msg.text_len) def get_log_lines(self, msg) -> Generator[str, None, None]: if msg.text_len > 0: - text = self.get_text_from_printk_log(msg) + text = self.get_text_from_log(msg) yield from text.splitlines() def get_dict_lines(self, msg) -> Generator[str, None, None]: if msg.dict_len == 0: return None - dict_offset = ( - msg.vol.offset + self.vmlinux.get_type("printk_log").size + msg.text_len - ) + + log_struct_name = self._get_log_struct_name() + log_struct_size = self.vmlinux.get_type(log_struct_name).size + dict_offset = msg.vol.offset + log_struct_size + msg.text_len dict_data = self._context.layers[self.layer_name].read( dict_offset, msg.dict_len ) @@ -226,29 +277,41 @@ class KmsgLegacy(ABCKmsg): yield " " + chunk.decode() def run(self) -> Iterator[Tuple[str, str, str, str, str]]: - log_buf_ptr = self.vmlinux.object_from_symbol(symbol_name="log_buf") - if log_buf_ptr == 0: - # This is weird, let's fallback to check the static ringbuffer. - log_buf_ptr = self.vmlinux.object_from_symbol( - symbol_name="__log_buf" - ).vol.offset - if log_buf_ptr == 0: - raise ValueError("Log buffer is not available") + # First, the ring buffer size is determined in the kernel configuration + # by CONFIG_LOG_BUF_SHIFT. This static buffer is held in the '__log_buf' + # global variable, with 'log_buf' serving as a pointer to it. + # The user can also update this size using 'log_buf_len' in the + # kernel boot parameters. Additionally, in SMP systems with over 64 CPUs, + # the ring buffer size dynamically allocates based on the number of CPUs, + # following CONFIG_LOG_CPU_MAX_BUF_SHIFT. + # In the last two cases mentioned above, the 'log_buf' pointer is + # updated to this new buffer. The original static buffer in '__log_buf' + # remains unused. Therefore, it is crucial to read from 'log_buf' rather + # than '__log_buf'. + + log_buf_ptr = self.vmlinux.object_from_symbol("log_buf") + log_buf_len = self.vmlinux.object_from_symbol("log_buf_len") + + log_first_idx = int(self.vmlinux.object_from_symbol("log_first_idx")) + log_next_idx = int(self.vmlinux.object_from_symbol("log_next_idx")) + + log_struct_name = self._get_log_struct_name() - log_first_idx = int( - self.vmlinux.object_from_symbol(symbol_name="log_first_idx") - ) cur_idx = log_first_idx - end_idx = None # We don't need log_next_idx here. See below msg.len == 0 - while cur_idx != end_idx: - end_idx = log_first_idx + if log_first_idx < log_next_idx: + end_idx = log_next_idx + else: + end_idx = log_buf_len + + while cur_idx < end_idx: msg_offset = log_buf_ptr + cur_idx # type: ignore - msg = self.vmlinux.object(object_type="printk_log", offset=msg_offset) + msg = self.vmlinux.object(object_type=log_struct_name, offset=msg_offset) if msg.len == 0: - # As per kernel/printk/printk.c: + # As per kernel/printk.c: # A length == 0 for the next message indicates a wrap-around to # the beginning of the buffer. cur_idx = 0 + end_idx = log_next_idx else: facility, level, timestamp, caller = self.get_prefix(msg) level_txt = self.get_level_text(level) @@ -262,39 +325,53 @@ class KmsgLegacy(ABCKmsg): cur_idx += msg.len -class KmsgFiveTen(ABCKmsg): - """In 5.10 the kernel ringbuffer implementation changed. +class Kmsg_3_11_to_5_10(Kmsg_3_5_to_3_11): + """Starting from version 3.11, the struct 'log' was renamed to 'printk_log'. + While 'log_buf' is declared as a pointer and '__log_buf' as a char array, + it essentially holds an array of 'printk_log' structs. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return vmlinux.has_type("printk_log") + + def _get_log_struct_name(self): + return "printk_log" + + +class Kmsg_5_10_to_(ABCKmsg): + """In 5.10 the kernel ring buffer implementation changed. Previously only one process should read /proc/kmsg and it is permanently open and periodically read by the syslog daemon. A high level structure 'printk_ringbuffer' was added to represent the printk - ringbuffer which actually contains two ringbuffers. The descriptor ring + ring buffer which actually contains two ring buffers. The descriptor ring 'desc_ring' contains the records' metadata, text offsets and states. The data block ring 'text_data_ring' contains the records' text strings. A pointer to the high level structure is kept in the prb pointer which is - initialized to a static ringbuffer. + initialized to a static ring buffer. .. code-block:: c static struct printk_ringbuffer *prb = &printk_rb_static; - In SMP systems with more than 64 CPUs this ringbuffer size is dynamically + In SMP systems with more than 64 CPUs this ring buffer size is dynamically allocated according the number of CPUs based on the value of CONFIG_LOG_CPU_MAX_BUF_SHIFT. The prb pointer is updated consequently to - this dynamic ringbuffer in setup_log_buf(). + this dynamic ring buffer in setup_log_buf(). .. code-block:: c prb = &printk_rb_dynamic; - Behind scenes, log_buf is still used as external buffer. - When the static printk_ringbuffer struct is initialized, _DEFINE_PRINTKRB - sets text_data_ring.data pointer to the address in log_buf which points to - the static buffer __log_buff. - If a dynamic ringbuffer takes place, setup_log_buf() sets - text_data_ring.data of printk_rb_dynamic to the new allocated external - buffer via the prb_init function. - In that case, the original external static buffer in __log_buf and - printk_rb_static are unused. + Behind scenes, 'log_buf' is still used as external buffer. + When the static 'printk_ringbuffer' struct is initialized, _DEFINE_PRINTKRB + sets text_data_ring.data pointer to the address in 'log_buf' which points + to the static buffer '__log_buf'. + If a dynamic ring buffer takes place, setup_log_buf() sets + text_data_ring.data of 'printk_rb_dynamic' to the new allocated external + buffer via the 'prb_init' function. + In that case, the original external static buffer in '__log_buf' and + 'printk_rb_static' are unused. .. code-block:: c @@ -352,7 +429,7 @@ class KmsgFiveTen(ABCKmsg): def run(self) -> Iterator[Tuple[str, str, str, str, str]]: # static struct printk_ringbuffer *prb = &printk_rb_static; - ringbuffers = self.vmlinux.object_from_symbol(symbol_name="prb").dereference() + ringbuffers = self.vmlinux.object_from_symbol("prb").dereference() desc_ring = ringbuffers.desc_ring text_data_ring = ringbuffers.text_data_ring From 0d934d4991818cdec0102759e78b324fac509f79 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 26 Dec 2023 16:13:47 -0300 Subject: [PATCH 029/130] Fix error and comment --- volatility3/framework/plugins/linux/kmsg.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index e55d5f14e..2b3d70ccb 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -102,14 +102,13 @@ class ABCKmsg(ABC): subclass.__name__, ) kmsg_inst = subclass(context=context, config=config) - # More than one class could be executed for an specific kernel - # version i.e. Netfilter Ingress hooks - # We expect just one implementation to be executed for an specific kernel yield from kmsg_inst.run() + # So far, it allows only one implementation to be executed for each + # specific kernel. break if kmsg_inst is None: - vollog.error("Unsupported Netfilter kernel implementation") + vollog.error("Unsupported kernel ring buffer implementation") @abstractmethod def run(self) -> Iterator[Tuple[str, str, str, str, str]]: From e4c44698f801967bb7dd41dc87cfb9a501ad2924 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 2 Jan 2024 11:14:43 -0300 Subject: [PATCH 030/130] Reduce imports --- volatility3/framework/plugins/linux/kmsg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 2b3d70ccb..70d80835b 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -14,7 +14,6 @@ from volatility3.framework import ( renderers, ) from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility vollog = logging.getLogger(__name__) @@ -483,7 +482,7 @@ class Kmsg_5_10_to_(ABCKmsg): cur_id &= desc_id_mask -class Kmsg(plugins.PluginInterface): +class Kmsg(interfaces.plugins.PluginInterface): """Kernel log buffer reader""" _required_framework_version = (2, 0, 0) From 02a389a61b460c6ffa3b2cae177cad1f48f23234 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 2 Jan 2024 11:17:18 -0300 Subject: [PATCH 031/130] Improve docstrings --- volatility3/framework/plugins/linux/kmsg.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 70d80835b..d17e0d92d 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -81,7 +81,7 @@ class ABCKmsg(ABC): config: Core configuration Yields: - kmsg records + The kmsg records. Same as run() """ vmlinux = context.modules[config["kernel"]] @@ -102,7 +102,7 @@ class ABCKmsg(ABC): ) kmsg_inst = subclass(context=context, config=config) yield from kmsg_inst.run() - # So far, it allows only one implementation to be executed for each + # So far, it only allows a single implementation to be executed for each # specific kernel. break @@ -111,7 +111,16 @@ class ABCKmsg(ABC): @abstractmethod def run(self) -> Iterator[Tuple[str, str, str, str, str]]: - """Walks through the specific kernel implementation.""" + """Walks through the specific kernel implementation. + + Returns: + tuple: + facility [str]: The log facility: kern, user, etc. see FACILITIES + level [str]: The log level: info, debug, etc. see LEVELS + timestamp [str]: The message timestamp. See nsec_to_sec_str() + caller [str]: The Caller ID: CPU(1) or Task(1234). See get_caller() + line [str]: The log message. + """ @classmethod @abstractmethod @@ -121,7 +130,8 @@ class ABCKmsg(ABC): The first class returning True will be instantiated and called via the run() method. - :return: True is the kernel being analysed fulfill the class requirements. + Returns: + bool: True if the kernel being analysed fulfill the class requirements. """ def get_string(self, addr: int, length: int) -> str: From 8e58815c114e23b28acfe1ec188c703ee555fb1a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 2 Jan 2024 11:25:35 -0300 Subject: [PATCH 032/130] Bump plugin's patch version --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d17e0d92d..919d9145b 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -497,7 +497,7 @@ class Kmsg(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 66b408d6e848a1e566f1b17787298be923b8a783 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 2 Jan 2024 11:37:10 -0300 Subject: [PATCH 033/130] Fix minor docstring typos --- volatility3/framework/plugins/linux/kmsg.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 919d9145b..d1f17bf94 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -115,10 +115,10 @@ class ABCKmsg(ABC): Returns: tuple: - facility [str]: The log facility: kern, user, etc. see FACILITIES - level [str]: The log level: info, debug, etc. see LEVELS + facility [str]: The log facility: kern, user, etc. See FACILITIES + level [str]: The log level: info, debug, etc. See LEVELS timestamp [str]: The message timestamp. See nsec_to_sec_str() - caller [str]: The Caller ID: CPU(1) or Task(1234). See get_caller() + caller [str]: The caller ID: CPU(1) or Task(1234). See get_caller() line [str]: The log message. """ @@ -131,7 +131,7 @@ class ABCKmsg(ABC): run() method. Returns: - bool: True if the kernel being analysed fulfill the class requirements. + bool: True if the kernel being analyzed fulfill the class requirements. """ def get_string(self, addr: int, length: int) -> str: From 25637a41e05e0bc5fccded01cf1913d45668ac25 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 8 Jan 2024 23:35:21 +0200 Subject: [PATCH 034/130] added account for XP timestamps - bit shifted --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 8790f41a7..9a286bc26 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -522,6 +522,9 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return True def get_create_time(self): + # For Windows XPs + if self.has_member("ThreadsProcess"): + return conversion.wintime_to_datetime(self.CreateTime.QuadPart >> 3) return conversion.wintime_to_datetime(self.CreateTime.QuadPart) def get_exit_time(self): From 550112b848913954e8b03d8c40bff9e2fd7902d7 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Mon, 8 Jan 2024 23:37:38 +0200 Subject: [PATCH 035/130] added another sanity check to ETHREAD.is_valid --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 9a286bc26..f4e2cc485 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -515,6 +515,9 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False + if not (1998 < ctime.year < 2030): + return False + except exceptions.InvalidAddressException: return False From 7c370121181af5da71b75d2b844def89341ddecd Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 9 Jan 2024 00:16:18 +0200 Subject: [PATCH 036/130] bumped version constants to mark change of interface (add of support for ETHREAD) --- volatility3/framework/constants/__init__.py | 4 ++-- volatility3/framework/plugins/windows/thrdscan.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index c3ebaca27..9aaafb933 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 6 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index 6e19f9bd5..80906b3b9 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -17,8 +17,8 @@ vollog = logging.getLogger(__name__) class ThrdScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for windows threads.""" - # version 2.5.0 adds support for scanning for 'Ethread' structures by pool tags - _required_framework_version = (2, 5, 0) + # version 2.6.0 adds support for scanning for 'Ethread' structures by pool tags + _required_framework_version = (2, 6, 0) _version = (1, 0, 0) @classmethod From b0d84e55ab1846004318e87bd7b5f427e7d70551 Mon Sep 17 00:00:00 2001 From: RuBublik Date: Tue, 9 Jan 2024 21:32:29 +0200 Subject: [PATCH 037/130] fix indentation (typo) --- volatility3/framework/symbols/windows/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index f4e2cc485..a0af29d18 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -516,7 +516,7 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): return False if not (1998 < ctime.year < 2030): - return False + return False except exceptions.InvalidAddressException: return False From 52dcfb45f45b9442b944f124f66cb0c363b584af Mon Sep 17 00:00:00 2001 From: bbarnacle Date: Tue, 28 Nov 2023 11:20:51 -0500 Subject: [PATCH 038/130] Windows: Add regex filtering to dumpfiles In volatility2, the windows.dumpfiles plugin allows you to filter the dumped files using a regular expression. This PR adds the same functionality to volatility3. The regular expression is passed in using --regex=REGEX and all files matching REGEX will be dumped. The --ignore-case flag can be passed to make the search case-insensitive. The search is case-sensitive by default. The matching volatility2 functionality can be found here: https://github.com/volatilityfoundation/volatility/blob/a438e768194a9e05eb4d9ee 9338b881c0fa25937/volatility/plugins/dumpfiles.py#L844 Manual testing was performed on windows memory images across different windows versions to verify the expected output. --- .../framework/plugins/windows/dumpfiles.py | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index dd82d897e..4aef660da 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -4,11 +4,12 @@ import logging import ntpath +import re from typing import List, Tuple, Type, Optional, Generator from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints +from volatility3.framework.renderers import format_hints, UnreadableValue from volatility3.plugins.windows import handles from volatility3.plugins.windows import pslist @@ -53,6 +54,15 @@ class DumpFiles(interfaces.plugins.PluginInterface): description="Dump a single _FILE_OBJECT at this physical address", optional=True, ), + requirements.StringRequirement( + name="regex", description="Dump files matching REGEX", optional=True + ), + requirements.BooleanRequirement( + name="ignore-case", + description="Ignore case in pattern match", + default=False, + optional=True, + ), requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), @@ -208,6 +218,11 @@ class DumpFiles(interfaces.plugins.PluginInterface): def _generator(self, procs: List, offsets: List): kernel = self.context.modules[self.config["kernel"]] + if self.config["regex"]: + if self.config["ignore-case"]: + file_re = re.compile(self.config["regex"], re.I) + else: + file_re = re.compile(self.config["regex"]) if procs: # The handles plugin doesn't expose any staticmethod/classmethod, and it also requires stashing @@ -243,6 +258,14 @@ class DumpFiles(interfaces.plugins.PluginInterface): obj_type = entry.get_object_type(type_map, cookie) if obj_type == "File": file_obj = entry.Body.cast("_FILE_OBJECT") + + if self.config["regex"]: + name = file_obj.file_name_with_device() + if isinstance(name, UnreadableValue): + continue + if not file_re.search(name): + continue + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): @@ -272,6 +295,13 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_obj.is_valid(): continue + if self.config["regex"]: + name = file_obj.file_name_with_device() + if isinstance(name, UnreadableValue): + continue + if not file_re.search(name): + continue + for result in self.process_file_object( self.context, kernel.layer_name, self.open, file_obj ): From 62366d4f6f6b57e6e2f6920df67a88125dac9484 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 14 Jan 2024 22:37:56 +0000 Subject: [PATCH 039/130] Symbols: Add support for metadata verification of symbol tables --- volatility3/framework/constants/__init__.py | 4 +- volatility3/framework/plugins/linux/kmsg.py | 11 +- volatility3/framework/symbols/__init__.py | 49 +- volatility3/framework/symbols/intermed.py | 9 + volatility3/framework/symbols/metadata.py | 42 +- volatility3/schemas/schema-6.3.0.json | 503 ++++++++++++++++++++ 6 files changed, 613 insertions(+), 5 deletions(-) create mode 100644 volatility3/schemas/schema-6.3.0.json diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09dded076..9aaafb933 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_MINOR = 6 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d1f17bf94..d79248256 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -10,10 +10,12 @@ from typing import Generator, Iterator, List, Tuple from volatility3.framework import ( class_subclasses, constants, + exceptions, interfaces, renderers, ) from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility vollog = logging.getLogger(__name__) @@ -495,7 +497,7 @@ class Kmsg_5_10_to_(ABCKmsg): class Kmsg(interfaces.plugins.PluginInterface): """Kernel log buffer reader""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 6, 0) _version = (1, 0, 2) @@ -514,6 +516,13 @@ class Kmsg(interfaces.plugins.PluginInterface): yield (0, values) def run(self): + if not self.context.symbol_space.verify_table_versions( + "dwarf2json", lambda version, _: (not version) or version > (0, 4, 1) + ): + raise exceptions.SymbolSpaceError( + "Invalid symbol table, please ensure the ISF table produced by dwarf2json was produced using a version > 0.4.1" + ) + return renderers.TreeGrid( [ ("facility", str), diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index 10cf39cf1..49ff94f4b 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -4,9 +4,20 @@ import collections import collections.abc +import datetime import enum import logging -from typing import Any, Dict, Iterable, Iterator, TypeVar, List +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + Optional, + Tuple, + TypeVar, + List, +) from volatility3.framework import constants, exceptions, interfaces, objects @@ -113,6 +124,42 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self._resolved = {} del self._dict[key] + def verify_table_versions( + self, + producer: str, + validator: Callable[[Optional[Tuple], Optional[datetime.datetime]], bool], + tables: List[str] = None, + ) -> bool: + """Verifies the producer metadata and version of tables + + Args: + producer: String name of a table producer to have validation performed + validator: callable that takes an optional version and an optional datetime that returns False if table is invalid + + Returns: + False if an invalid table was found or True if no invalid table was found + """ + if tables is None: + tables = self._dict.keys() + for table_name in tables: + table = self._dict[table_name] + if not table.producer: + vollog.debug( + f"Symbol table {table_name} could not be validated because no producer metadata was found" + ) + continue + if table.producer.name == producer: + # Run the verification + if not validator( + table.producer.version, + table.producer.datetime, + ): + vollog.debug(f"Symbol table {table_name} does not pass validator") + return False + else: + continue + return True + ### Resolution functions class UnresolvedTemplate(objects.templates.ReferenceTemplate): diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 24e00bbd8..8c40e466a 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -183,6 +183,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): types = _construct_delegate_function("types", True) enumerations = _construct_delegate_function("enumerations", True) metadata = _construct_delegate_function("metadata", True) + producer = _construct_delegate_function("producer", True) clear_symbol_cache = _construct_delegate_function("clear_symbol_cache") get_type = _construct_delegate_function("get_type") get_symbol = _construct_delegate_function("get_symbol") @@ -372,6 +373,14 @@ class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass=ABCMeta): table.""" return None + @property + def producer(self) -> Optional["metadata.ProducerMetadata"]: + """Returns a metadata object containing information about the symbol + table.""" + return metadata.ProducerMetadata( + self._json_object.get("metadata", {}).get("producer", {}) + ) + def clear_symbol_cache(self) -> None: """Clears the symbol cache of the symbol table.""" self._symbol_cache.clear() diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 61947be69..f765e43c9 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -2,9 +2,49 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import datetime +import logging from typing import Optional, Tuple, Union -from volatility3.framework import interfaces +from volatility3.framework import constants, interfaces + +vollog = logging.getLogger(__name__) + + +class ProducerMetadata(interfaces.symbols.MetadataInterface): + """Class to handle the Producer metadata from an ISF""" + + @property + def name(self) -> Optional[str]: + return self._json_data.get("name", None) + + @property + def version(self) -> Optional[Tuple[int]]: + """Returns the version of the ISF file producer""" + version = self._json_data.get("version", None) + if not version: + return None + if all([x in "0123456789." for x in version]): + return tuple([int(x) for x in version.split(".")]) + else: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Metadata version contains unexpected characters: '{version}'", + ) + + @property + def datetime(self) -> Optional[datetime.datetime]: + """Returns a timestamp for when the file was produced""" + if "datetime" not in self._json_data: + return None + try: + timestamp = datetime.datetime.strptime( + self._json_data["datetime"], "YYYY-MM-DD" + ) + except (TypeError, ValueError): + vollog.debug("Invalid timestamp in producer information of symbol table") + return None + return timestamp class WindowsMetadata(interfaces.symbols.MetadataInterface): diff --git a/volatility3/schemas/schema-6.3.0.json b/volatility3/schemas/schema-6.3.0.json new file mode 100644 index 000000000..2cd8ebbda --- /dev/null +++ b/volatility3/schemas/schema-6.3.0.json @@ -0,0 +1,503 @@ +{ + "$schema": "http://json-schema.org/schema#", + "id": "http://volatilityfoundation.org/intermediate-format/schema", + "title": "Symbol Container", + "type": "object", + "definitions": { + "metadata_producer": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+.[0-9]+.[0-9]+$" + }, + "datetime": { + "type": "string", + "format": "date-time" + } + }, + "required":[ + "name", + "version" + ] + }, + "metadata_windows_pe": { + "type": "object", + "properties": { + "major": { + "type": "integer" + }, + "minor": { + "type": "integer" + }, + "revision": { + "type": "integer" + }, + "build": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "major", + "minor", + "revision" + ] + }, + "metadata_windows_pdb": { + "type": "object", + "properties": { + "GUID": { + "type": "string" + }, + "age": { + "type": "integer" + }, + "database": { + "type": "string" + }, + "machine_type": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "GUID", + "age", + "database", + "machine_type" + ] + }, + "metadata_windows": { + "type": "object", + "properties": { + "pe": { + "$ref": "#/definitions/metadata_windows_pe" + }, + "pdb": { + "$ref": "#/definitions/metadata_windows_pdb" + } + }, + "additionalProperties": false + }, + "metadata_nix": { + "type": "object", + "properties": { + "symbols": { + "type": "array", + "items": { + "$ref": "#/definitions/metadata_nix_item" + } + }, + "types": { + "type": "array", + "items": { + "$ref": "#/definitions/metadata_nix_item" + } + } + }, + "additionalProperties": false + }, + "metadata_format": { + "type": "string", + "pattern": "^6.[0-9]+.[0-9]+$" + }, + "metadata_nix_item": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "pattern": "^(dwarf|symtab|system-map)$" + }, + "name": { + "type": "string" + }, + "hash_type": { + "type": "string", + "pattern": "^(sha256)$" + }, + "hash_value": { + "type": "string", + "pattern": "^[a-fA-F0-9]+$" + } + }, + "additionalProperties": false + }, + "element_metadata": { + "type": "object", + "oneOf": [ + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + } + }, + "required": [ + "format" + ], + "additionalProperties": false + }, + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + }, + "windows": { + "$ref": "#/definitions/metadata_windows" + } + }, + "required": [ + "format", + "windows" + ], + "additionalProperties": false + }, + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + }, + "linux": { + "$ref": "#/definitions/metadata_nix" + } + }, + "required": [ + "format", + "linux" + ], + "additionalProperties": false + }, + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + }, + "mac": { + "$ref": "#/definitions/metadata_nix" + } + }, + "required": [ + "format", + "mac" + ], + "additionalProperties": false + } + ] + }, + "element_enum": { + "properties": { + "size": { + "type": "integer" + }, + "base": { + "type": "string" + }, + "constants": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + } + }, + "required": [ + "size", + "base", + "constants" + ], + "additionalProperties": false + }, + "element_symbol": { + "properties": { + "address": { + "type": "number" + }, + "linkage_name": { + "type": "string" + }, + "type": { + "$ref": "#/definitions/type_descriptor" + }, + "constant_data": { + "type": "string", + "media": { + "binaryEncoding": "base64", + "readOnly": true + } + } + }, + "required": [ + "address" + ], + "additionalProperties": false + }, + "element_base_type": { + "properties": { + "size": { + "type": "integer" + }, + "signed": { + "type": "boolean" + }, + "kind": { + "type": "string", + "pattern": "^(void|int|float|char|bool)$" + }, + "endian": { + "type": "string", + "pattern": "^(little|big)$" + } + }, + "required": [ + "size", + "kind", + "signed", + "endian" + ], + "additionalProperties": false + }, + "element_user_type": { + "properties": { + "kind": { + "type": "string", + "pattern": "^(struct|union|class)$" + }, + "size": { + "type": "integer" + }, + "fields": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/field" + } + } + }, + "required": [ + "kind", + "size", + "fields" + ], + "additionalProperties": false + }, + "field": { + "properties": { + "type": { + "$ref": "#/definitions/type_descriptor" + }, + "offset": { + "type": "integer" + }, + "anonymous": { + "type": "boolean" + } + }, + "required": [ + "type", + "offset" + ], + "additionalProperties": false + }, + "type_descriptor": { + "oneOf": [ + { + "$ref": "#/definitions/type_pointer" + }, + { + "$ref": "#/definitions/type_base" + }, + { + "$ref": "#/definitions/type_array" + }, + { + "$ref": "#/definitions/type_struct" + }, + { + "$ref": "#/definitions/type_enum" + }, + { + "$ref": "#/definitions/type_function" + }, + { + "$ref": "#/definitions/type_bitfield" + } + ] + }, + "type_pointer": { + "properties": { + "kind": { + "type": "string", + "pattern": "^pointer$" + }, + "base": { + "type": "string" + }, + "subtype": { + "$ref": "#/definitions/type_descriptor" + } + }, + "required": [ + "kind", + "subtype" + ], + "additionalProperties": false + }, + "type_base": { + "properties": { + "kind": { + "type": "string", + "pattern": "^base$" + }, + "name": { + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "additionalProperties": false + }, + "type_array": { + "properties": { + "kind": { + "type": "string", + "pattern": "^array$" + }, + "subtype": { + "$ref": "#/definitions/type_descriptor" + }, + "count": { + "type": "integer" + } + }, + "required": [ + "kind", + "subtype", + "count" + ], + "additionalProperties": false + }, + "type_struct": { + "properties": { + "kind": { + "type": "string", + "pattern": "^(struct|class|union)$" + }, + "name": { + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "additionalProperties": false + }, + "type_enum": { + "properties": { + "kind": { + "type": "string", + "pattern": "^enum$" + }, + "name": { + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "additionalProperties": false + }, + "type_function": { + "properties": { + "kind": { + "type": "string", + "pattern": "^function$" + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + "type_bitfield": { + "properties": { + "kind": { + "type": "string", + "pattern": "^bitfield$" + }, + "bit_position": { + "type": "integer" + }, + "bit_length": { + "type": "integer" + }, + "type": { + "oneOf": [ + { + "$ref": "#/definitions/type_base" + }, + { + "$ref": "#/definitions/type_enum" + } + ] + } + }, + "required": [ + "kind", + "bit_position", + "bit_length", + "type" + ], + "additionalProperties": false + } + }, + "properties": { + "metadata": { + "$ref": "#/definitions/element_metadata" + }, + "base_types": { + "additionalProperties": { + "$ref": "#/definitions/element_base_type" + } + }, + "user_types": { + "additionalProperties": { + "$ref": "#/definitions/element_user_type" + } + }, + "enums": { + "additionalProperties": { + "$ref": "#/definitions/element_enum" + } + }, + "symbols": { + "additionalProperties": { + "$ref": "#/definitions/element_symbol" + } + } + }, + "required": [ + "metadata", + "base_types", + "user_types", + "enums", + "symbols" + ], + "additionalProperties": false +} From 7c8f4ca4ac5c23da3afa1d0ec6404c3fada69a27 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 14 Jan 2024 22:44:29 +0000 Subject: [PATCH 040/130] Schemas: As of 6.3.0 we require a producer metadata, not just format --- volatility3/schemas/schema-6.3.0.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/schemas/schema-6.3.0.json b/volatility3/schemas/schema-6.3.0.json index 2cd8ebbda..ba6e970bb 100644 --- a/volatility3/schemas/schema-6.3.0.json +++ b/volatility3/schemas/schema-6.3.0.json @@ -139,7 +139,8 @@ } }, "required": [ - "format" + "format", + "producer" ], "additionalProperties": false }, @@ -157,6 +158,7 @@ }, "required": [ "format", + "producer", "windows" ], "additionalProperties": false @@ -175,6 +177,7 @@ }, "required": [ "format", + "producer", "linux" ], "additionalProperties": false @@ -193,6 +196,7 @@ }, "required": [ "format", + "producer", "mac" ], "additionalProperties": false From 269058021dc171c7b46a6a2cb20d78d646b9f8fd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jan 2024 20:34:39 +0000 Subject: [PATCH 041/130] Plugins: Fix up kmsg code issues --- volatility3/framework/plugins/linux/kmsg.py | 1 - volatility3/framework/symbols/metadata.py | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d79248256..dd707a7ff 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -15,7 +15,6 @@ from volatility3.framework import ( renderers, ) from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility vollog = logging.getLogger(__name__) diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index f765e43c9..149829371 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -26,11 +26,11 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): return None if all([x in "0123456789." for x in version]): return tuple([int(x) for x in version.split(".")]) - else: - vollog.log( - constants.LOGLEVEL_VVVV, - f"Metadata version contains unexpected characters: '{version}'", - ) + vollog.log( + constants.LOGLEVEL_VVVV, + f"Metadata version contains unexpected characters: '{version}'", + ) + return None @property def datetime(self) -> Optional[datetime.datetime]: From 614d4d6f2b9eac50992c04cd08e7389be19ede99 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jan 2024 20:35:54 +0000 Subject: [PATCH 042/130] Layers: Fix cloudstorage unnecessary import --- volatility3/framework/layers/cloudstorage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/layers/cloudstorage.py b/volatility3/framework/layers/cloudstorage.py index 3f88ef34f..97ed54231 100644 --- a/volatility3/framework/layers/cloudstorage.py +++ b/volatility3/framework/layers/cloudstorage.py @@ -20,7 +20,6 @@ try: except ImportError: HAS_GCSFS = False -from volatility3.framework import exceptions from volatility3.framework.layers import resources vollog = logging.getLogger(__file__) From 01ffcd9634af3d6ef90c130f1dfd1a8910554d4d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jan 2024 20:37:26 +0000 Subject: [PATCH 043/130] Core: Fixing None equality test in MapleTree implementation --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d8a2867cc..d73d0cfb9 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -362,7 +362,7 @@ class maple_tree(objects.StructType): # None. If however you wanted to parse from a node, but ignore some parts of the tree below it then # this could be populated with the addresses of the nodes you wish to ignore. - if seen == None: + if seen is None: seen = set() # protect against unlikely loop From 4b86b9ea89b5610c581a574045309b93ea8f3ef5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jan 2024 20:40:27 +0000 Subject: [PATCH 044/130] Plugins: Remove unnecessary variable from windows.mftscan --- volatility3/framework/plugins/windows/mftscan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 7e4e1ca18..4298b4e4e 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -53,7 +53,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" - header_object = symbol_table + constants.BANG + "ATTR_HEADER" si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" From 4e89040dadfacac01ba0c0a3727969a7b0fb0047 Mon Sep 17 00:00:00 2001 From: hsarkey Date: Sun, 14 Jan 2024 13:20:56 -0500 Subject: [PATCH 045/130] Updated changes to the windows.malfind plugin. Eliminated --refined as a command line option and instead added an additional column called "Notes" to provide info on common headers. --- .../framework/plugins/windows/malfind.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 1c73fdf1c..a14f8889d 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -46,12 +46,6 @@ class Malfind(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="vadinfo", component=vadinfo.VadInfo, version=(2, 0, 0) ), - requirements.BooleanRequirement( - name="refined", - description="Refine the output. Only show regions with an MZ header or that start with well known opcode combinations (i.e. PUSH EBP). WARNING: This can cause you to overlook regions with wiped headers or shell code blocks starting with NOP sleds, etc.. However, it will in general result in less noisy output.", - default=False, - optional=True, - ), ] @classmethod @@ -144,9 +138,6 @@ class Malfind(interfaces.plugins.PluginInterface): yield vad, data def _generator(self, procs): - # set refined criteria - refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] - # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] @@ -155,14 +146,21 @@ class Malfind(interfaces.plugins.PluginInterface): ) for proc in procs: + # by default, "Notes" column will be set to none + notes = "None" process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( self.context, kernel.layer_name, kernel.symbol_table_name, proc ): - # check if refined option was passed - if self.config["refined"] and data[0:2] not in refined_criteria: - continue + # Check for unique headers and update "Notes" column if criteria is met + match data[0:2]: + case b"MZ" | b"\x55\x8B": + notes = "MZ header" + case b"\x55\x8B": + notes = "PE header" + case b"\x55\x48" | b"\x55\x89": + notes = "Function prologue" # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 if is_32bit_arch or proc.get_is_wow64(): @@ -209,6 +207,7 @@ class Malfind(interfaces.plugins.PluginInterface): vad.get_commit_charge(), vad.get_private_memory(), file_output, + notes, format_hints.HexBytes(data), disasm, ), @@ -229,6 +228,7 @@ class Malfind(interfaces.plugins.PluginInterface): ("CommitCharge", int), ("PrivateMemory", int), ("File output", str), + ("Notes", str), ("Hexdump", format_hints.HexBytes), ("Disasm", interfaces.renderers.Disassembly), ], From e6e138f12d06509da829008867392bc2346f18f9 Mon Sep 17 00:00:00 2001 From: hsarkey Date: Wed, 17 Jan 2024 17:24:26 -0500 Subject: [PATCH 046/130] Updated windows.malfind to have a "Notes" column which will indicate if a process meets a "refined criteria", meaning it has a common header type (MZ, PE, or a function prologue). --- volatility3/framework/plugins/windows/malfind.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index a14f8889d..e5a842611 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -140,6 +140,9 @@ class Malfind(interfaces.plugins.PluginInterface): def _generator(self, procs): # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] + + # set refined criteria to know when to add to "Notes" column + refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] is_32bit_arch = not symbols.symbol_table_is_64bit( self.context, kernel.symbol_table_name @@ -154,12 +157,12 @@ class Malfind(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, proc ): # Check for unique headers and update "Notes" column if criteria is met - match data[0:2]: - case b"MZ" | b"\x55\x8B": + if data[0:2] in refined_criteria: + if data[0:2] == b"MZ": notes = "MZ header" - case b"\x55\x8B": + elif data[0:2] == b"\x55\x8B": notes = "PE header" - case b"\x55\x48" | b"\x55\x89": + else: notes = "Function prologue" # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 From ef36cb5a6c6a30f0bc986cc275c83a734b9d535b Mon Sep 17 00:00:00 2001 From: hsarkey Date: Wed, 17 Jan 2024 17:51:27 -0500 Subject: [PATCH 047/130] Updated windows.malfind to have a "Notes" column to indicate if a process meets "refined" criteria, which includes common headers like MZ,PE or function prologues. Fixed black formatting issue. --- volatility3/framework/plugins/windows/malfind.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index e5a842611..284144077 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -140,7 +140,7 @@ class Malfind(interfaces.plugins.PluginInterface): def _generator(self, procs): # determine if we're on a 32 or 64 bit kernel kernel = self.context.modules[self.config["kernel"]] - + # set refined criteria to know when to add to "Notes" column refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] @@ -150,7 +150,7 @@ class Malfind(interfaces.plugins.PluginInterface): for proc in procs: # by default, "Notes" column will be set to none - notes = "None" + notes = "None" process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( From a50ebb6014ff11ea165454e0a08b967d01cce9e8 Mon Sep 17 00:00:00 2001 From: Calvin Kusek Date: Tue, 2 Jan 2024 12:00:41 -0500 Subject: [PATCH 048/130] Windows: Display additional process info for windows.pstree --- .../framework/plugins/windows/pstree.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/pstree.py b/volatility3/framework/plugins/windows/pstree.py index a39fe7485..2be96277c 100644 --- a/volatility3/framework/plugins/windows/pstree.py +++ b/volatility3/framework/plugins/windows/pstree.py @@ -5,7 +5,7 @@ import datetime import logging from typing import Callable, Dict, Set, Tuple -from volatility3.framework import objects, interfaces, renderers +from volatility3.framework import objects, interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import pslist @@ -132,6 +132,25 @@ class PsTree(interfaces.plugins.PluginInterface): proc.get_exit_time(), ) + try: + audit = proc.SeAuditProcessCreationInfo.ImageFileName.Name + # If 'audit' is set to the empty string, display NotAvailableValue + row += (audit.get_string() or renderers.NotAvailableValue(),) + except exceptions.InvalidAddressException: + row += (renderers.NotAvailableValue(),) + + try: + process_params = proc.get_peb().ProcessParameters + row += ( + process_params.CommandLine.get_string(), + process_params.ImagePathName.get_string(), + ) + except exceptions.InvalidAddressException: + row += ( + renderers.NotAvailableValue(), + renderers.NotAvailableValue(), + ) + yield (self._levels[pid] - 1, row) for child_pid in self._children.get(pid, []): yield from yield_processes( @@ -161,6 +180,9 @@ class PsTree(interfaces.plugins.PluginInterface): ("Wow64", bool), ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime), + ("Audit", str), + ("Cmd", str), + ("Path", str), ], self._generator( filter_func=pslist.PsList.create_pid_filter( From ae5375a622c82177f6b29adfb8b0f410ecb03059 Mon Sep 17 00:00:00 2001 From: Brandon Barnacle Date: Wed, 17 Jan 2024 09:49:39 -0500 Subject: [PATCH 049/130] PR comment changes Change the --regex flag to --filter. Add a check so that --filter cannot be used with --physaddr or --virtaddr. Change self.config["filter"] check to check if file_re has been set. --- .../framework/plugins/windows/dumpfiles.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 4aef660da..8865007d8 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -55,11 +55,11 @@ class DumpFiles(interfaces.plugins.PluginInterface): optional=True, ), requirements.StringRequirement( - name="regex", description="Dump files matching REGEX", optional=True + name="filter", description="Dump files matching regular expression FILTER", optional=True ), requirements.BooleanRequirement( name="ignore-case", - description="Ignore case in pattern match", + description="Ignore case in filter match", default=False, optional=True, ), @@ -218,11 +218,11 @@ class DumpFiles(interfaces.plugins.PluginInterface): def _generator(self, procs: List, offsets: List): kernel = self.context.modules[self.config["kernel"]] - if self.config["regex"]: - if self.config["ignore-case"]: - file_re = re.compile(self.config["regex"], re.I) - else: - file_re = re.compile(self.config["regex"]) + file_re = None + if self.config["filter"]: + flags = re.I if self.config["ignore-case"] else 0 + file_re = re.compile(self.config["filter"], flags) + if procs: # The handles plugin doesn't expose any staticmethod/classmethod, and it also requires stashing @@ -259,7 +259,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if obj_type == "File": file_obj = entry.Body.cast("_FILE_OBJECT") - if self.config["regex"]: + if file_re: name = file_obj.file_name_with_device() if isinstance(name, UnreadableValue): continue @@ -295,7 +295,7 @@ class DumpFiles(interfaces.plugins.PluginInterface): if not file_obj.is_valid(): continue - if self.config["regex"]: + if file_re: name = file_obj.file_name_with_device() if isinstance(name, UnreadableValue): continue @@ -345,6 +345,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): procs = list() kernel = self.context.modules[self.config["kernel"]] + if self.config["filter"] and (self.config["virtaddr"] or self.config["physaddr"]): + raise ValueError("Cannot use filter flag with an address flag") + if self.config.get("virtaddr", None) is not None: offsets.append((self.config["virtaddr"], True)) elif self.config.get("physaddr", None) is not None: From 4dc6f637b055f912bafaa9f6079759539277d60f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 28 Jan 2024 23:50:45 +0000 Subject: [PATCH 050/130] Core: Apply black 24.1.0 to the whole codebase --- volatility3/cli/text_renderer.py | 8 +++--- volatility3/framework/__init__.py | 6 ++--- volatility3/framework/automagic/mac.py | 6 ++--- volatility3/framework/automagic/module.py | 6 ++--- .../framework/automagic/symbol_finder.py | 12 ++++----- volatility3/framework/automagic/windows.py | 20 +++++++------- .../framework/configuration/requirements.py | 6 ++--- volatility3/framework/layers/avml.py | 6 ++--- volatility3/framework/layers/elf.py | 6 ++--- volatility3/framework/layers/intel.py | 6 ++--- volatility3/framework/layers/lime.py | 6 ++--- volatility3/framework/layers/qemu.py | 6 ++--- volatility3/framework/layers/xen.py | 6 ++--- volatility3/framework/objects/__init__.py | 6 ++--- volatility3/framework/plugins/mac/pslist.py | 4 +-- .../framework/plugins/windows/crashinfo.py | 6 ++--- .../framework/plugins/windows/mftscan.py | 1 - .../framework/plugins/windows/netscan.py | 10 ++++--- .../plugins/windows/registry/printkey.py | 6 ++--- volatility3/framework/renderers/conversion.py | 6 ++--- volatility3/framework/symbols/__init__.py | 26 +++++++++---------- .../symbols/windows/extensions/__init__.py | 6 ++--- volatility3/plugins/windows/statistics.py | 10 +++---- 23 files changed, 88 insertions(+), 93 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ffb8d516b..6e58ee68d 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -389,9 +389,11 @@ class JsonRenderer(CLIRenderer): interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), - datetime.datetime: lambda x: x.isoformat() - if not isinstance(x, interfaces.renderers.BaseAbsentValue) - else None, + datetime.datetime: lambda x: ( + x.isoformat() + if not isinstance(x, interfaces.renderers.BaseAbsentValue) + else None + ), "default": lambda x: x, } diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 9c17846a8..1565b2267 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -206,9 +206,9 @@ def _zipwalk(path: str): if not file.is_dir(): dirlist = zip_results.get(os.path.dirname(file.filename), []) dirlist.append(os.path.basename(file.filename)) - zip_results[ - os.path.join(path, os.path.dirname(file.filename)) - ] = dirlist + zip_results[os.path.join(path, os.path.dirname(file.filename))] = ( + dirlist + ) for value in zip_results: yield value, zip_results[value] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index aa75fbc3d..e51753139 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -138,9 +138,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): config_path = join("automagic", "MacIntelHelper", new_layer_name) context.config[join(config_path, "memory_layer")] = layer_name context.config[join(config_path, "page_map_offset")] = dtb - context.config[ - join(config_path, MacSymbolFinder.banner_config_key) - ] = str(banner, "latin-1") + context.config[join(config_path, MacSymbolFinder.banner_config_key)] = ( + str(banner, "latin-1") + ) new_layer = intel.Intel32e( context, diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index ee56a040c..ff13db905 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -34,9 +34,9 @@ class KernelModule(interfaces.automagic.AutomagicInterface): return None # The requirement is unfulfilled and is a ModuleRequirement - context.config[ - interfaces.configuration.path_join(new_config_path, "class") - ] = "volatility3.framework.contexts.Module" + context.config[interfaces.configuration.path_join(new_config_path, "class")] = ( + "volatility3.framework.contexts.Module" + ) for req in requirement.requirements: if ( diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index bf1c8ff16..21e594549 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -150,12 +150,12 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join - context.config[ - path_join(config_path, requirement.name, "class") - ] = clazz - context.config[ - path_join(config_path, requirement.name, "isf_url") - ] = isf_path + context.config[path_join(config_path, requirement.name, "class")] = ( + clazz + ) + context.config[path_join(config_path, requirement.name, "isf_url")] = ( + isf_path + ) context.config[ path_join(config_path, requirement.name, "symbol_mask") ] = layer.address_mask diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index a8530829b..52296f5ad 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -402,19 +402,19 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): if swap_location: context.config[current_layer_path] = current_layer_name try: - context.config[ - layer_loc_path - ] = requirements.URIRequirement.location_from_file( - swap_location + context.config[layer_loc_path] = ( + requirements.URIRequirement.location_from_file( + swap_location + ) ) except ValueError: vollog.warning( f"Volatility swap_location {swap_location} could not be validated - swap layer disabled" ) continue - context.config[ - layer_class_path - ] = "volatility3.framework.layers.physical.FileLayer" + context.config[layer_class_path] = ( + "volatility3.framework.layers.physical.FileLayer" + ) # Add the requirement new_req = requirements.TranslationLayerRequirement( @@ -424,9 +424,9 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface): ) swap_req.add_requirement(new_req) - context.config[ - path_join(swap_sub_config, "number_of_elements") - ] = counter + context.config[path_join(swap_sub_config, "number_of_elements")] = ( + counter + ) context.config[swap_sub_config] = True swap_req.construct(context, swap_config) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index abdffdbe4..1c0622574 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -550,9 +550,9 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} - context.config[ - interfaces.configuration.path_join(config_path, self.name) - ] = True + context.config[interfaces.configuration.path_join(config_path, self.name)] = ( + True + ) return {} @classmethod diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index c825464cc..2e5572192 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -224,7 +224,7 @@ class AVMLStacker(interfaces.automagic.StackerLayerInterface): except exceptions.LayerException: return None new_name = context.layers.free_layer_name("AVMLLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return AVMLLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index a10d36592..b2fd6d4d1 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -115,9 +115,9 @@ class Elf64Stacker(interfaces.automagic.StackerLayerInterface): vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("Elf64Layer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) try: return Elf64Layer(context, new_name, new_name) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 7d3b86a12..ae477854d 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -277,9 +277,9 @@ class Intel(linear.LinearlyMappedLayer): This allows translation layers to provide maps of contiguous regions in one layer """ - stashed_offset = ( - stashed_mapped_offset - ) = stashed_size = stashed_mapped_size = stashed_map_layer = None + stashed_offset = stashed_mapped_offset = stashed_size = stashed_mapped_size = ( + stashed_map_layer + ) = None for offset, size, mapped_offset, mapped_size, map_layer in self._mapping( offset, length, ignore_errors ): diff --git a/volatility3/framework/layers/lime.py b/volatility3/framework/layers/lime.py index 28d646640..8b93932ab 100644 --- a/volatility3/framework/layers/lime.py +++ b/volatility3/framework/layers/lime.py @@ -104,7 +104,7 @@ class LimeStacker(interfaces.automagic.StackerLayerInterface): except LimeFormatException: return None new_name = context.layers.free_layer_name("LimeLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return LimeLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 501b8655e..ff483291c 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -486,9 +486,9 @@ class QemuStacker(interfaces.automagic.StackerLayerInterface): except exceptions.LayerException: return None new_name = context.layers.free_layer_name("QemuSuspendLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) layer = QemuSuspendLayer(context, new_name, new_name) cls.stacker_slow_warning() return layer diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index f7881a091..927b30430 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -173,8 +173,8 @@ class XenCoreDumpStacker(elf.Elf64Stacker): vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("XenCoreDumpLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return XenCoreDumpLayer(context, new_name, new_name) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 831856e3d..316a30bec 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -768,12 +768,10 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): raise IndexError(f"Member not present in array template: {child}") @overload - def __getitem__(self, i: int) -> interfaces.objects.Template: - ... + def __getitem__(self, i: int) -> interfaces.objects.Template: ... @overload - def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: - ... + def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: ... def __getitem__(self, i): """Returns the i-th item from the array.""" diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 88045a277..9b570f3f9 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -49,9 +49,7 @@ class PsList(interfaces.plugins.PluginInterface): ] @classmethod - def get_list_tasks( - cls, method: str - ) -> Callable[ + def get_list_tasks(cls, method: str) -> Callable[ [interfaces.context.ContextInterface, str, Callable[[int], bool]], Iterable[interfaces.objects.ObjectInterface], ]: diff --git a/volatility3/framework/plugins/windows/crashinfo.py b/volatility3/framework/plugins/windows/crashinfo.py index 4ecd85087..862eb6080 100644 --- a/volatility3/framework/plugins/windows/crashinfo.py +++ b/volatility3/framework/plugins/windows/crashinfo.py @@ -46,9 +46,9 @@ class Crashinfo(interfaces.plugins.PluginInterface): bitmap_size = format_hints.Hex(summary_header.BitmapSize) bitmap_pages = format_hints.Hex(summary_header.Pages) else: - bitmap_header_size = ( - bitmap_size - ) = bitmap_pages = renderers.NotApplicableValue() + bitmap_header_size = bitmap_size = bitmap_pages = ( + renderers.NotApplicableValue() + ) yield ( 0, diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 4298b4e4e..91a2e9152 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -175,7 +175,6 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): class ADS(interfaces.plugins.PluginInterface): - """Scans for Alternate Data Stream""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index d0bbd5cbd..62ead3ab7 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -487,10 +487,12 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if not isinstance(row_data[9], datetime.datetime): continue row_data = [ - "N/A" - if isinstance(i, renderers.UnreadableValue) - or isinstance(i, renderers.UnparsableValue) - else i + ( + "N/A" + if isinstance(i, renderers.UnreadableValue) + or isinstance(i, renderers.UnparsableValue) + else i + ) for i in row_data ] description = ( diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index e248c19bc..180f8f9d9 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -193,9 +193,9 @@ class PrintKey(interfaces.plugins.PluginInterface): vollog.debug( "Couldn't read registry value type, so data is unreadable" ) - value_data: Union[ - interfaces.renderers.BaseAbsentValue, bytes - ] = renderers.UnreadableValue() + value_data: Union[interfaces.renderers.BaseAbsentValue, bytes] = ( + renderers.UnreadableValue() + ) else: try: value_data = node.decode_data() diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index bf7da9ecb..bb18fcc8a 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -28,9 +28,9 @@ def wintime_to_datetime( def unixtime_to_datetime( unixtime: int, ) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: - ret: Union[ - interfaces.renderers.BaseAbsentValue, datetime.datetime - ] = renderers.UnparsableValue() + ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = ( + renderers.UnparsableValue() + ) if unixtime > 0: with contextlib.suppress(ValueError): diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index 10cf39cf1..d1e7a104d 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -35,9 +35,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): def __init__(self) -> None: super().__init__() - self._dict: Dict[ - str, interfaces.symbols.BaseSymbolTableInterface - ] = collections.OrderedDict() + self._dict: Dict[str, interfaces.symbols.BaseSymbolTableInterface] = ( + collections.OrderedDict() + ) # Permanently cache all resolved symbols self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} @@ -73,9 +73,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self, offset: int, size: int = 0, table_name: str = None ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" - table_list: Iterable[ - interfaces.symbols.BaseSymbolTableInterface - ] = self._dict.values() + table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = ( + self._dict.values() + ) if table_name is not None: if table_name in self._dict: table_list = [self._dict[table_name]] @@ -179,15 +179,15 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): if child.vol.type_name not in self._resolved: traverse_list.append(child.vol.type_name) try: - self._resolved[ - child.vol.type_name - ] = self._weak_resolve( - SymbolType.TYPE, child.vol.type_name + self._resolved[child.vol.type_name] = ( + self._weak_resolve( + SymbolType.TYPE, child.vol.type_name + ) ) except exceptions.SymbolError: - self._resolved[ - child.vol.type_name - ] = self.UnresolvedTemplate(child.vol.type_name) + self._resolved[child.vol.type_name] = ( + self.UnresolvedTemplate(child.vol.type_name) + ) # Stash the replacement replacements.add((traverser, child)) elif child.children: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d435851d7..846e5bd90 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -452,9 +452,9 @@ class FILE_OBJECT(objects.StructType, pool.ExecutiveObject): ].is_valid(self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name: Union[ - str, interfaces.renderers.BaseAbsentValue - ] = renderers.UnreadableValue() + name: Union[str, interfaces.renderers.BaseAbsentValue] = ( + renderers.UnreadableValue() + ) # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 9915312e3..7f56b75f8 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -31,13 +31,9 @@ class Statistics(plugins.PluginInterface): # Do mass mapping and determine the number of different layers and how many pages go to each one layer = self.context.layers[self.config["primary"]] - page_count = ( - swap_count - ) = ( - invalid_page_count - ) = ( - large_page_count - ) = large_swap_count = large_invalid_count = other_invalid = 0 + page_count = swap_count = invalid_page_count = large_page_count = ( + large_swap_count + ) = large_invalid_count = other_invalid = 0 if isinstance(layer, intel.Intel): page_addr = 0 From 497d291ef4393e2580052a3dfddbef10e4dc2338 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 29 Jan 2024 00:07:40 +0000 Subject: [PATCH 051/130] Windows: Fix black on dumpfiles --- volatility3/framework/plugins/windows/dumpfiles.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 8865007d8..48539c752 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -55,7 +55,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): optional=True, ), requirements.StringRequirement( - name="filter", description="Dump files matching regular expression FILTER", optional=True + name="filter", + description="Dump files matching regular expression FILTER", + optional=True, ), requirements.BooleanRequirement( name="ignore-case", @@ -223,7 +225,6 @@ class DumpFiles(interfaces.plugins.PluginInterface): flags = re.I if self.config["ignore-case"] else 0 file_re = re.compile(self.config["filter"], flags) - if procs: # The handles plugin doesn't expose any staticmethod/classmethod, and it also requires stashing # private variables, so we need an instance (for now, anyway). We _could_ call Handles._generator() @@ -345,7 +346,9 @@ class DumpFiles(interfaces.plugins.PluginInterface): procs = list() kernel = self.context.modules[self.config["kernel"]] - if self.config["filter"] and (self.config["virtaddr"] or self.config["physaddr"]): + if self.config["filter"] and ( + self.config["virtaddr"] or self.config["physaddr"] + ): raise ValueError("Cannot use filter flag with an address flag") if self.config.get("virtaddr", None) is not None: From 837d3b7b9fce65d5ef6543b6db9ecbad5092d4b0 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 14 Dec 2023 13:47:00 -0600 Subject: [PATCH 052/130] Windows: Add parsing of service binary/dll Parses the binary or dll associated with each service from the Windows SYSTEM hive and includes it in the output for each service. --- .../framework/plugins/windows/svcscan.py | 139 +++++++++++++++++- 1 file changed, 133 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 60562915e..720f745d8 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -4,20 +4,30 @@ import logging import os -from typing import List +from typing import Dict, List, NamedTuple, Union -from volatility3.framework import interfaces, renderers, constants, symbols, exceptions +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions from volatility3.framework.symbols.windows.extensions import services -from volatility3.plugins.windows import poolscanner, vadyarascan, pslist +from volatility3.plugins.windows import poolscanner, pslist, vadyarascan +from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) +ServiceBinaryInfo = NamedTuple( + "ServiceBinaryInfo", + [ + ("dll", Union[str, interfaces.renderers.BaseAbsentValue]), + ("binary", Union[str, interfaces.renderers.BaseAbsentValue]), + ], +) + + class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" @@ -42,10 +52,16 @@ class SvcScan(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="vadyarascan", plugin=vadyarascan.VadYaraScan, version=(1, 0, 0) ), + requirements.PluginRequirement( + name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) + ), ] @staticmethod - def get_record_tuple(service_record: interfaces.objects.ObjectInterface): + def get_record_tuple( + service_record: interfaces.objects.ObjectInterface, + binary_info: ServiceBinaryInfo, + ): return ( format_hints.Hex(service_record.vol.offset), service_record.Order, @@ -56,6 +72,8 @@ class SvcScan(interfaces.plugins.PluginInterface): service_record.get_name(), service_record.get_display(), service_record.get_binary(), + binary_info.binary, + binary_info.dll, ) @staticmethod @@ -150,6 +168,86 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types=native_types, ) + def _get_service_key(self, kernel): + for hive in hivelist.HiveList.list_hives( + context=self.context, + base_config_path=self.config_path, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_string="machine\\system", + hive_offsets=None, + ): + # Get ControlSet\Services. + try: + return hive.get_key(r"CurrentControlSet\Services") + except (KeyError, exceptions.InvalidAddressException): + try: + return hive.get_key(r"ControlSet001\Services") + except (KeyError, exceptions.InvalidAddressException): + pass + + return None + + @staticmethod + def _get_service_dll( + service_key, + ) -> Union[str, interfaces.renderers.BaseAbsentValue]: + try: + param_key = next( + key + for key in service_key.get_subkeys() + if key.get_name() == "Parameters" + ) + return ( + next( + val + for val in param_key.get_values() + if val.get_name() == "ServiceDll" + ) + .decode_data() + .decode("utf-16") + .rstrip("\x00") + ) + + except UnicodeDecodeError: + return renderers.UnparsableValue() + except StopIteration: + return renderers.UnreadableValue() + + @staticmethod + def _get_service_binary( + service_key, + ) -> Union[str, interfaces.renderers.BaseAbsentValue]: + try: + return ( + next( + val + for val in service_key.get_values() + if val.get_name() == "ImagePath" + ) + .decode_data() + .decode("utf-16") + .rstrip("\x00") + ) + + except UnicodeDecodeError: + return renderers.UnparsableValue() + except StopIteration: + return renderers.UnreadableValue() + + @staticmethod + def _get_service_binary_map( + services_key: interfaces.objects.ObjectInterface, + ) -> Dict[str, ServiceBinaryInfo]: + services = services_key.get_subkeys() + return { + service_key.get_name(): ServiceBinaryInfo( + SvcScan._get_service_dll(service_key), + SvcScan._get_service_binary(service_key), + ) + for service_key in services + } + def _generator(self): kernel = self.context.modules[self.config["kernel"]] @@ -157,6 +255,15 @@ class SvcScan(interfaces.plugins.PluginInterface): self.context, kernel.symbol_table_name, self.config_path ) + # Building the dictionary ahead of time is much better for performance + # vs looking up each service's DLL individually. + services_key = self._get_service_key(kernel) + service_binary_dll_map = ( + self._get_service_binary_map(services_key) + if services_key is not None + else {} + ) + relative_tag_offset = self.context.symbol_space.get_type( service_table_name + constants.BANG + "_SERVICE_RECORD" ).relative_child_offset("Tag") @@ -209,7 +316,16 @@ class SvcScan(interfaces.plugins.PluginInterface): if not service_record.is_valid(): continue - yield (0, self.get_record_tuple(service_record)) + service_info = service_binary_dll_map.get( + service_record.get_name(), + ServiceBinaryInfo( + renderers.UnreadableValue(), renderers.UnreadableValue() + ), + ) + yield ( + 0, + self.get_record_tuple(service_record, service_info), + ) else: service_header = self.context.object( service_table_name + constants.BANG + "_SERVICE_HEADER", @@ -227,7 +343,16 @@ class SvcScan(interfaces.plugins.PluginInterface): if service_record in seen: break seen.append(service_record) - yield (0, self.get_record_tuple(service_record)) + service_info = service_binary_dll_map.get( + service_record.get_name(), + ServiceBinaryInfo( + renderers.UnreadableValue(), renderers.UnreadableValue() + ), + ) + yield ( + 0, + self.get_record_tuple(service_record, service_info), + ) def run(self): return renderers.TreeGrid( @@ -241,6 +366,8 @@ class SvcScan(interfaces.plugins.PluginInterface): ("Name", str), ("Display", str), ("Binary", str), + ("Binary (Registry)", str), + ("Dll", str), ], self._generator(), ) From 77b01106356ab30ec531799a66d38db3951033c4 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 14 Dec 2023 14:16:40 -0600 Subject: [PATCH 053/130] Windows: Fixes svc symbols and OS version detection There were some changes made to the service structures between certain versions of Windows that caused the PID/binary path to fail to parse across quite a few samples. This adds the appropriate version detection logic and updated symbol files to ensure that services are parsed correctly across Windows versions. --- volatility3/framework/interfaces/context.py | 2 +- .../framework/plugins/windows/svcscan.py | 50 +++- .../services/services-win10-17763-x86.json | 248 +++++++++++++++++ .../services/services-win10-18362-x64.json | 255 ++++++++++++++++++ .../services/services-win10-18362-x86.json | 248 +++++++++++++++++ .../services/services-win10-19041-x64.json | 255 ++++++++++++++++++ .../services/services-win10-19041-x86.json | 248 +++++++++++++++++ .../services/services-win10-25398-x64.json | 255 ++++++++++++++++++ .../framework/symbols/windows/versions.py | 34 +++ 9 files changed, 1584 insertions(+), 11 deletions(-) create mode 100644 volatility3/framework/symbols/windows/services/services-win10-17763-x86.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-18362-x64.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-18362-x86.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-19041-x64.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-19041-x86.json create mode 100644 volatility3/framework/symbols/windows/services/services-win10-25398-x64.json diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 29cb41379..a95f2b464 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -87,7 +87,7 @@ class ContextInterface(metaclass=ABCMeta): offset: int, native_layer_name: str = None, **arguments, - ): + ) -> "interfaces.objects.ObjectInterface": """Object factory, takes a context, symbol, offset and optional layer_name. diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 720f745d8..74fe2d802 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -106,26 +106,46 @@ class SvcScan(interfaces.plugins.PluginInterface): and is_64bit ): symbol_filename = "services-xp-2003-x64" + elif ( + versions.is_win10_25398_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win10-25398-x64" + elif ( + versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win10-19041-x64" + elif ( + versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win10-19041-x86" + elif ( + versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win10-18362-x64" + elif ( + versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win10-18362-x86" elif ( versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) and is_64bit ): symbol_filename = "services-win10-16299-x64" + elif ( + versions.is_win10_17763_or_later(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win10-17763-x86" elif ( versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) and not is_64bit ): symbol_filename = "services-win10-16299-x86" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win8-x64" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win8-x86" elif ( versions.is_win10_15063(context=context, symbol_table=symbol_table) and is_64bit @@ -136,6 +156,16 @@ class SvcScan(interfaces.plugins.PluginInterface): and not is_64bit ): symbol_filename = "services-win10-15063-x86" + elif ( + versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) + and is_64bit + ): + symbol_filename = "services-win8-x64" + elif ( + versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) + and not is_64bit + ): + symbol_filename = "services-win8-x86" elif ( versions.is_windows_8_or_later(context=context, symbol_table=symbol_table) and is_64bit diff --git a/volatility3/framework/symbols/windows/services/services-win10-17763-x86.json b/volatility3/framework/symbols/windows/services/services-win10-17763-x86.json new file mode 100644 index 000000000..8f2854721 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-17763-x86.json @@ -0,0 +1,248 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 4 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 12 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 20 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 12 + }, + "_SERVICE_RECORD": { + "fields": { + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 48 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 160 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 24 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 56 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 44 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 160 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 52 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 156 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-18362-x64.json b/volatility3/framework/symbols/windows/services/services-win10-18362-x64.json new file mode 100644 index 000000000..a6a80c1d3 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-18362-x64.json @@ -0,0 +1,255 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 16 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 24 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 40 + } + }, + "kind": "struct", + "size": 40 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_RECORD": { + "fields": { + "ServiceList": { + "type": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + }, + "offset": 0 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 32 + }, + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 64 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 240 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 36 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 76 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 56 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 240 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 72 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 248 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-18362-x86.json b/volatility3/framework/symbols/windows/services/services-win10-18362-x86.json new file mode 100644 index 000000000..4684dfe5b --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-18362-x86.json @@ -0,0 +1,248 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 4 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 12 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 20 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + } + }, + "kind": "struct", + "size": 12 + }, + "_SERVICE_RECORD": { + "fields": { + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 48 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 164 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 24 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 56 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 44 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 164 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 52 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 156 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-19041-x64.json b/volatility3/framework/symbols/windows/services/services-win10-19041-x64.json new file mode 100644 index 000000000..e44dbbd37 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-19041-x64.json @@ -0,0 +1,255 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 16 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 24 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 40 + } + }, + "kind": "struct", + "size": 40 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_RECORD": { + "fields": { + "ServiceList": { + "type": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + }, + "offset": 0 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 32 + }, + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 64 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 296 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 36 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 76 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 56 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 296 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 72 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 296 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "David McDonald", + "datetime": "2023-11-16T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-19041-x86.json b/volatility3/framework/symbols/windows/services/services-win10-19041-x86.json new file mode 100644 index 000000000..cc5ed9a73 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-19041-x86.json @@ -0,0 +1,248 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 4 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 8 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 12 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 20 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 12 + }, + "_SERVICE_RECORD": { + "fields": { + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 48 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 192 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 12 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 24 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 56 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 44 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 192 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 52 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 20 + } + }, + "kind": "struct", + "size": 192 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "vtypes_to_json.py", + "datetime": "2019-04-17T13:45:16.417006" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/services/services-win10-25398-x64.json b/volatility3/framework/symbols/windows/services/services-win10-25398-x64.json new file mode 100644 index 000000000..cd29abc43 --- /dev/null +++ b/volatility3/framework/symbols/windows/services/services-win10-25398-x64.json @@ -0,0 +1,255 @@ +{ + "symbols": {}, + "enums": { + "StateEnum": { + "base": "long", + "constants": { + "SERVICE_START_PENDING": 2, + "SERVICE_STOP_PENDING": 3, + "SERVICE_STOPPED": 1, + "SERVICE_CONTINUE_PENDING": 5, + "SERVICE_PAUSE_PENDING": 6, + "SERVICE_PAUSED": 7, + "SERVICE_RUNNING": 4 + }, + "size": 4 + }, + "StartEnum": { + "base": "long", + "constants": { + "SERVICE_DEMAND_START": 3, + "SERVICE_AUTO_START": 2, + "SERVICE_BOOT_START": 0, + "SERVICE_DISABLED": 4, + "SERVICE_SYSTEM_START": 1 + }, + "size": 4 + } + }, + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "user_types": { + "_SERVICE_LIST_ENTRY": { + "fields": { + "Flink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 16 + }, + "Blink": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + } + }, + "offset": 0 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_PROCESS": { + "fields": { + "BinaryPath": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 24 + }, + "ProcessId": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 40 + } + }, + "kind": "struct", + "size": 40 + }, + "_SERVICE_HEADER": { + "fields": { + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 0 + }, + "ServiceRecord": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + } + }, + "kind": "struct", + "size": 16 + }, + "_SERVICE_RECORD": { + "fields": { + "ServiceList": { + "type": { + "kind": "struct", + "name": "_SERVICE_LIST_ENTRY" + }, + "offset": 0 + }, + "Tag": { + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + }, + "offset": 32 + }, + "DisplayName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 64 + }, + "ServiceProcess": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_PROCESS" + } + }, + "offset": 336 + }, + "PrevEntry": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SERVICE_RECORD" + } + }, + "offset": 16 + }, + "Start": { + "type": { + "kind": "enum", + "name": "StartEnum" + }, + "offset": 36 + }, + "State": { + "type": { + "kind": "enum", + "name": "StateEnum" + }, + "offset": 84 + }, + "ServiceName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 56 + }, + "DriverName": { + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "unsigned short" + } + }, + "offset": 296 + }, + "Type": { + "type": { + "kind": "base", + "name": "unsigned long" + }, + "offset": 80 + }, + "Order": { + "type": { + "kind": "base", + "name": "unsigned int" + }, + "offset": 32 + } + }, + "kind": "struct", + "size": 336 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "David McDonald", + "datetime": "2023-11-16T15:05:35-06:00" + }, + "format": "4.1.0" + } +} diff --git a/volatility3/framework/symbols/windows/versions.py b/volatility3/framework/symbols/windows/versions.py index 84ce65432..6e2c2e9bd 100644 --- a/volatility3/framework/symbols/windows/versions.py +++ b/volatility3/framework/symbols/windows/versions.py @@ -151,11 +151,45 @@ is_win10_16299_or_later = OsDistinguisher( ], ) +is_win10_17763_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 17763), + fallback_checks=[ + ("_EPROCESS", "TrustletIdentity", False), + ("ParentSecurityDomain", None, False), + ], +) + +is_win10_18362_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 18362), + fallback_checks=[ + ("ObHeaderCookie", None, True), + ("_CM_CACHED_VALUE_INDEX", None, False), + ("_WNF_PROCESS_CONTEXT", None, True), + ], +) + is_win10_18363_or_later = OsDistinguisher( version_check=lambda x: x >= (10, 0, 18363), fallback_checks=[("_KQOS_GROUPING_SETS", None, True)], ) +is_win10_19041_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 19041), + fallback_checks=[ + ("_EPROCESS", "TimerResolutionIgnore", True), + ("_EPROCESS", "VmProcessorHostTransition", True), + ("_KQOS_GROUPING_SETS", None, True), + ], +) + +is_win10_25398_or_later = OsDistinguisher( + version_check=lambda x: x >= (10, 0, 25398), + fallback_checks=[ + ("_EPROCESS", "MmSlabIdentity", True), + ("_EPROCESS", "EnableProcessImpersonationLogging", True), + ], +) + is_windows_10 = OsDistinguisher( version_check=lambda x: x >= (10, 0), fallback_checks=[("ObHeaderCookie", None, True)], From 86d6b4f7245a553bb2b7db6a3d844375a57f8cd4 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:08:14 -0600 Subject: [PATCH 054/130] Bump plugin version --- volatility3/framework/plugins/windows/svcscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 74fe2d802..7688594d2 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -32,7 +32,7 @@ class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 4b400b6df612fe0f3a796c06dbfb7fa272e5d0fd Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:09:36 -0600 Subject: [PATCH 055/130] Refactor out code duplication in OS version checks --- .../framework/plugins/windows/svcscan.py | 132 ++++++------------ 1 file changed, 39 insertions(+), 93 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 7688594d2..4f20ed346 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -4,9 +4,16 @@ import logging import os -from typing import Dict, List, NamedTuple, Union +from typing import Dict, List, NamedTuple, Optional, Tuple, Union, cast -from volatility3.framework import constants, exceptions, interfaces, renderers, symbols +from volatility3.framework import ( + constants, + exceptions, + interfaces, + objects, + renderers, + symbols, +) from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints @@ -76,6 +83,28 @@ class SvcScan(interfaces.plugins.PluginInterface): binary_info.dll, ) + # These checks must be completed from newest -> oldest OS version. + _win_version_file_map: List[Tuple[versions.OsDistinguisher, bool, str]] = [ + (versions.is_win10_25398_or_later, True, "services-win10-25398-x64"), + (versions.is_win10_19041_or_later, True, "services-win10-19041-x64"), + (versions.is_win10_19041_or_later, False, "services-win10-19041-x86"), + (versions.is_win10_18362_or_later, True, "services-win10-18362-x64"), + (versions.is_win10_18362_or_later, False, "services-win10-18362-x86"), + (versions.is_win10_17763_or_later, False, "services-win10-17763-x86"), + (versions.is_win10_16299_or_later, True, "services-win10-16299-x64"), + (versions.is_win10_16299_or_later, False, "services-win10-16299-x86"), + (versions.is_win10_15063, True, "services-win10-15063-x64"), + (versions.is_win10_15063, False, "services-win10-15063-x86"), + (versions.is_win10_up_to_15063, True, "services-win8-x64"), + (versions.is_win10_up_to_15063, False, "services-win8-x86"), + (versions.is_windows_8_or_later, True, "services-win8-x64"), + (versions.is_windows_8_or_later, True, "services-win8-x86"), + (versions.is_vista_or_later, True, "services-vista-x64"), + (versions.is_vista_or_later, False, "services-vista-x86"), + (versions.is_windows_xp, False, "services-xp-x86"), + (versions.is_xp_or_2003, True, "services-xp-2003-x64"), + ] + @staticmethod def create_service_table( context: interfaces.context.ContextInterface, @@ -96,97 +125,14 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types = context.symbol_space[symbol_table].natives is_64bit = symbols.symbol_table_is_64bit(context, symbol_table) - if ( - versions.is_windows_xp(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-xp-x86" - elif ( - versions.is_xp_or_2003(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-xp-2003-x64" - elif ( - versions.is_win10_25398_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-25398-x64" - elif ( - versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-19041-x64" - elif ( - versions.is_win10_19041_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-19041-x86" - elif ( - versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-18362-x64" - elif ( - versions.is_win10_18362_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-18362-x86" - elif ( - versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-16299-x64" - elif ( - versions.is_win10_17763_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-17763-x86" - elif ( - versions.is_win10_16299_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-16299-x86" - elif ( - versions.is_win10_15063(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win10-15063-x64" - elif ( - versions.is_win10_15063(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win10-15063-x86" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win8-x64" - elif ( - versions.is_win10_up_to_15063(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win8-x86" - elif ( - versions.is_windows_8_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-win8-x64" - elif ( - versions.is_windows_8_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-win8-x86" - elif ( - versions.is_vista_or_later(context=context, symbol_table=symbol_table) - and is_64bit - ): - symbol_filename = "services-vista-x64" - elif ( - versions.is_vista_or_later(context=context, symbol_table=symbol_table) - and not is_64bit - ): - symbol_filename = "services-vista-x86" - else: + try: + symbol_filename = next( + filename + for version_check, for_64bit, filename in SvcScan._win_version_file_map + if is_64bit == for_64bit + and version_check(context=context, symbol_table=symbol_table) + ) + except StopIteration: raise NotImplementedError("This version of Windows is not supported!") return intermed.IntermediateSymbolTable.create( From 5541378afe86d41734c05f6b84bbafcd26d46644 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:09:59 -0600 Subject: [PATCH 056/130] Create unique config path --- volatility3/framework/plugins/windows/svcscan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 4f20ed346..d16bee113 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -147,7 +147,9 @@ class SvcScan(interfaces.plugins.PluginInterface): def _get_service_key(self, kernel): for hive in hivelist.HiveList.list_hives( context=self.context, - base_config_path=self.config_path, + base_config_path=interfaces.configuration.path_join( + self.config_path, "hivelist" + ), layer_name=kernel.layer_name, symbol_table=kernel.symbol_table_name, filter_string="machine\\system", From f5c1bf0f1f5c177b2dcee9d09d3ab522ba83f41e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:10:20 -0600 Subject: [PATCH 057/130] Remove argument that matches default --- volatility3/framework/plugins/windows/svcscan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index d16bee113..b77491a65 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -153,7 +153,6 @@ class SvcScan(interfaces.plugins.PluginInterface): layer_name=kernel.layer_name, symbol_table=kernel.symbol_table_name, filter_string="machine\\system", - hive_offsets=None, ): # Get ControlSet\Services. try: From c517a44cde3cb87d48da3a72d685630eca7acaa1 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:10:54 -0600 Subject: [PATCH 058/130] Add type hints/casts to _get_service_key method --- volatility3/framework/plugins/windows/svcscan.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index b77491a65..dd2f76890 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -144,7 +144,7 @@ class SvcScan(interfaces.plugins.PluginInterface): native_types=native_types, ) - def _get_service_key(self, kernel): + def _get_service_key(self, kernel) -> Optional[objects.StructType]: for hive in hivelist.HiveList.list_hives( context=self.context, base_config_path=interfaces.configuration.path_join( @@ -156,10 +156,14 @@ class SvcScan(interfaces.plugins.PluginInterface): ): # Get ControlSet\Services. try: - return hive.get_key(r"CurrentControlSet\Services") + return cast( + objects.StructType, hive.get_key(r"CurrentControlSet\Services") + ) except (KeyError, exceptions.InvalidAddressException): try: - return hive.get_key(r"ControlSet001\Services") + return cast( + objects.StructType, hive.get_key(r"ControlSet001\Services") + ) except (KeyError, exceptions.InvalidAddressException): pass From 42c2a86a37ace33cf93d28efd491b00e8c29dd53 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 16 Jan 2024 10:11:16 -0600 Subject: [PATCH 059/130] Add logging for when a Services key cannot be retrieved --- volatility3/framework/plugins/windows/svcscan.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index dd2f76890..095661880 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -165,6 +165,10 @@ class SvcScan(interfaces.plugins.PluginInterface): objects.StructType, hive.get_key(r"ControlSet001\Services") ) except (KeyError, exceptions.InvalidAddressException): + vollog.log( + constants.LOGLEVEL_VVVV, + "Could not retrieve any control set from SYSTEM hive", + ) pass return None From 4192e8a711aff985352f3347dc8cd289894a812f Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 18 Jan 2024 12:22:23 -0600 Subject: [PATCH 060/130] Fix condition in OsDistinguisher --- volatility3/framework/symbols/windows/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/versions.py b/volatility3/framework/symbols/windows/versions.py index 6e2c2e9bd..e1e74afc0 100644 --- a/volatility3/framework/symbols/windows/versions.py +++ b/volatility3/framework/symbols/windows/versions.py @@ -155,7 +155,7 @@ is_win10_17763_or_later = OsDistinguisher( version_check=lambda x: x >= (10, 0, 17763), fallback_checks=[ ("_EPROCESS", "TrustletIdentity", False), - ("ParentSecurityDomain", None, False), + ("ParentSecurityDomain", None, True), ], ) From 43e8e01daff8dc1ff88b9edfc6d5d6b2fac33417 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 30 Jan 2024 09:26:17 -0600 Subject: [PATCH 061/130] Removes unnecessary pass and deindents return stmt --- volatility3/framework/plugins/windows/svcscan.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 095661880..10de46e2a 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -169,9 +169,8 @@ class SvcScan(interfaces.plugins.PluginInterface): constants.LOGLEVEL_VVVV, "Could not retrieve any control set from SYSTEM hive", ) - pass - return None + return None @staticmethod def _get_service_dll( From 470c750d82ee0208f673472bcf2bf6632ae4190f Mon Sep 17 00:00:00 2001 From: hsarkey Date: Wed, 31 Jan 2024 16:00:29 -0500 Subject: [PATCH 062/130] Updated to use dictionary for refined criteria and BaseAbsentValue for when criteria is not met --- .../framework/plugins/windows/malfind.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 284144077..8c0cb68ac 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -142,15 +142,20 @@ class Malfind(interfaces.plugins.PluginInterface): kernel = self.context.modules[self.config["kernel"]] # set refined criteria to know when to add to "Notes" column - refined_criteria = [b"MZ", b"\x55\x8B", b"\x55\x48", b"\x55\x89"] + refined_criteria = { + b"MZ": "MZ header", + b"\x55\x8B": "PE header", + b"\x55\x48": "Function prologue", + b"\x55\x89": "Function prologue", + } is_32bit_arch = not symbols.symbol_table_is_64bit( self.context, kernel.symbol_table_name ) for proc in procs: - # by default, "Notes" column will be set to none - notes = "None" + # by default, "Notes" column will be set to N/A + notes = renderers.NotApplicableValue() process_name = utility.array_to_string(proc.ImageFileName) for vad, data in self.list_injections( @@ -158,12 +163,7 @@ class Malfind(interfaces.plugins.PluginInterface): ): # Check for unique headers and update "Notes" column if criteria is met if data[0:2] in refined_criteria: - if data[0:2] == b"MZ": - notes = "MZ header" - elif data[0:2] == b"\x55\x8B": - notes = "PE header" - else: - notes = "Function prologue" + notes = refined_criteria[data[0:2]] # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64 if is_32bit_arch or proc.get_is_wow64(): From 6d093a4e4338aaee5b69384f1652cda994a3e1d3 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 31 Jan 2024 21:04:16 +0000 Subject: [PATCH 063/130] Documentation: Update copyright in README.md for 2024 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 471735af8..f69acb3bb 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ The latest generated copy of the documentation can be found at: Date: Wed, 31 Jan 2024 21:13:00 +0000 Subject: [PATCH 064/130] Documentation: Bump the copyright here to 2024 --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index d601c1eee..cabfdc327 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -169,7 +169,7 @@ master_doc = "index" # General information about the project. project = "Volatility 3" -copyright = "2012-2022, Volatility Foundation" +copyright = "2012-2024, Volatility Foundation" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From d1c7f14a75dc36c32523caa907aa49375c772160 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 2 Feb 2024 13:28:04 +0000 Subject: [PATCH 065/130] Linux: attempt to fix issue 1089 --- .../symbols/linux/extensions/__init__.py | 75 ++++++++++++++----- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d8a2867cc..f67e3962b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -26,42 +26,79 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): + def _get_mod_mem_type(self): + """Attempt to get the mod_mem_type enum once to allow repeated access from other functions""" + if not self.has_member("mod_mem_type"): + try: + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type( + self._context, self + ) + # mod_mem_type and module_memory were added in kernel 6.4 which replaces + # module_layout for storing the information around core_layout etc. + # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information + self.mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices + except exceptions.SymbolError: + vollog.debug( + f"Unable to find mod_mem_type enum. This is expected on kernels <6.4 but may cause issues with later kernels" + ) + self.mod_mem_type = None + def get_module_base(self): - if self.has_member("core_layout"): - return self.core_layout.base + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base else: - return self.module_core + if self.has_member("core_layout"): + return self.core_layout.base + else: + return self.module_core def get_init_size(self): - if self.has_member("init_layout"): - return self.init_layout.size - elif self.has_member("init_size"): - return self.init_size + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + else: + if self.has_member("init_layout"): + return self.init_layout.size + elif self.has_member("init_size"): + return self.init_size raise AttributeError( "module -> get_init_size: Unable to determine .init section size of module" ) def get_core_size(self): - if self.has_member("core_layout"): - return self.core_layout.size - elif self.has_member("core_size"): - return self.core_size + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_TEXT"]].size + else: + if self.has_member("core_layout"): + return self.core_layout.size + elif self.has_member("core_size"): + return self.core_size raise AttributeError( "module -> get_core_size: Unable to determine core size of module" ) def get_module_core(self): - if self.has_member("core_layout"): - return self.core_layout.base - elif self.has_member("module_core"): - return self.module_core + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base + else: + if self.has_member("core_layout"): + return self.core_layout.base + elif self.has_member("module_core"): + return self.module_core raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - if self.has_member("init_layout"): - return self.init_layout.base - elif self.has_member("module_init"): - return self.module_init + self._get_mod_mem_type() + if self.mod_mem_type: + return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base + else: + if self.has_member("init_layout"): + return self.init_layout.base + elif self.has_member("module_init"): + return self.module_init raise AttributeError("module -> get_module_core: Unable to get module init") def get_name(self): From ecf20d99b4ddc3d28c577ee383902877c3090d45 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 2 Feb 2024 13:28:57 +0000 Subject: [PATCH 066/130] Linux: fix typo in error message in module.get_module_init extension --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f67e3962b..9c15acef5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -99,7 +99,7 @@ class module(generic.GenericIntelProcess): return self.init_layout.base elif self.has_member("module_init"): return self.module_init - raise AttributeError("module -> get_module_core: Unable to get module init") + raise AttributeError("module -> get_module_init: Unable to get module init") def get_name(self): """Get the name of the module as a string""" From 261d7ffbf2e6837980989292b3966ddd2042d277 Mon Sep 17 00:00:00 2001 From: Eve Date: Fri, 2 Feb 2024 14:03:55 +0000 Subject: [PATCH 067/130] Linux: include all mod mem types in size calculations --- .../framework/symbols/linux/extensions/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9c15acef5..e3ff4bdd5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -56,7 +56,11 @@ class module(generic.GenericIntelProcess): def get_init_size(self): self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + return ( + self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size + + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size + ) else: if self.has_member("init_layout"): return self.init_layout.size @@ -69,7 +73,12 @@ class module(generic.GenericIntelProcess): def get_core_size(self): self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_TEXT"]].size + return ( + self.mem[self.mod_mem_type["MOD_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_DATA"]].size + + self.mem[self.mod_mem_type["MOD_RODATA"]].size + + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size + ) else: if self.has_member("core_layout"): return self.core_layout.size From 130621a2e7655015f220ac561de974e99f571fca Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Feb 2024 20:04:36 +0100 Subject: [PATCH 068/130] Changing comment when Type error is occuring --- volatility3/framework/plugins/windows/iat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 11c273859..73976fb86 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -50,7 +50,7 @@ class IAT(interfaces.plugins.PluginInterface): ) if proc_layer_name is None: - raise TypeError("Layer must be a string not None") + raise TypeError("add_process_layer failed") pe_table_name = intermed.IntermediateSymbolTable.create( self.context, From ab8ed049a4c4bddbc4c2a2129cb425b9113b5d37 Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sat, 3 Feb 2024 11:32:00 +0300 Subject: [PATCH 069/130] Windows: add TrueCrypt plugin --- .../framework/plugins/windows/truecrypt.py | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 volatility3/framework/plugins/windows/truecrypt.py diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py new file mode 100644 index 000000000..988c900ea --- /dev/null +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -0,0 +1,141 @@ +from typing import Iterable, Generator, List, Tuple + +from volatility3.framework import constants, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces.configuration import RequirementInterface +from volatility3.framework.interfaces.objects import ObjectInterface +from volatility3.framework.objects import Bytes, DataFormatInfo, Integer, StructType +from volatility3.framework.objects.templates import ObjectTemplate +from volatility3.framework.objects.utility import array_to_string +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import pe + +from volatility3.plugins.windows import modules + +class Passphrase(interfaces.plugins.PluginInterface): + """TrueCrypt Cached Passphrase Finder""" + + _version = (0, 1, 0) + _required_framework_version = (2, 5, 2) + + @classmethod + def get_requirements(cls) -> List[RequirementInterface]: + return [ + requirements.ModuleRequirement( + 'kernel', + description='Windows kernel', + architectures=['Intel32', 'Intel64'] + ), + requirements.VersionRequirement( + name='modules', + component=modules.Modules, + version=(1, 1, 0) + ), + requirements.IntRequirement( + name="min-length", + description="Minimum length of passphrases to identify", + default=5, + optional=True + ), + ] + + def scan_module(self, module_base: int, layer_name: str) -> Generator[Tuple[int, str], None, None]: + """Scans the TrueCrypt kernel module for cached passphrases. + + Args: + module_base: the module's DLL base + layer_name: the name of the layer in which the module resides + + Generates: + A tuple of the offset at which a password is found, and the password + """ + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "windows", + "pe", + class_types=pe.class_types + ) + dos_header: pe.IMAGE_DOS_HEADER = self.context.object( + pe_table_name + constants.BANG + '_IMAGE_DOS_HEADER', + layer_name, + module_base, + ) + data_section: StructType = next( + sec for sec in dos_header.get_nt_header().get_sections() + if array_to_string(sec.Name) == '.data' + ) + base: int = data_section.VirtualAddress + module_base + size: int = data_section.Misc.VirtualSize + # Looking at `Length` in TrueCrypt/Common/Password.h::Password struct + DWORD_SIZE_BYTES: int = 4 + format = DataFormatInfo(length=DWORD_SIZE_BYTES, byteorder="little", signed=True) + int32 = ObjectTemplate( + Integer, + pe_table_name + constants.BANG + 'int', + data_format=format + ) + count, not_aligned = divmod(size, DWORD_SIZE_BYTES) + if not_aligned: + raise ValueError("PE data section not DWORD-aligned!") + lengths = self.context.object( + pe_table_name + constants.BANG + 'array', + layer_name, + base, + count=count, + subtype=int32, + ) + min_length = self.config.get('min-length') + for length in lengths: + # TrueCrypt maximum password length is 64 + # (see TrueCrypt/Common/Password.h) + if not min_length <= length <= 64: + continue + offset = length.vol['offset'] + DWORD_SIZE_BYTES + passphrase: Bytes = self.context.object( + pe_table_name + constants.BANG + 'bytes', + layer_name, + offset, + length=length, + ) + # TrueCrypt/Common/Password.c permits chars in the range + # [0x20, 0x7F). + if not all(0x20 <= c < 0x7F for c in passphrase): + continue + # TrueCrypt/Common/Password.h::Password struct is padded with + # 3 zero bytes to keep 64-byte alignment. + buf: Bytes = self.context.object( + pe_table_name + constants.BANG + 'bytes', + layer_name, + offset + length + 1, # +1 for '\0'-terminated password string + length=3 + ) + if any(buf): + continue + # Password found. + yield offset, passphrase.decode(encoding='ascii') + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + mods: Iterable[ObjectInterface] = modules.Modules.list_modules( + self.context, + kernel.layer_name, + kernel.symbol_table_name + ) + truecrypt_module_base = next( + mod.DllBase for mod in mods + if mod.BaseDllName.get_string().lower() == 'truecrypt.sys' + ) + for offset, password in self.scan_module(truecrypt_module_base, kernel.layer_name): + yield (0, (format_hints.Hex(offset), len(password), password)) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Length", int), + ("Password", str), + ], + self._generator() + ) From 41cbfbfa340ef8949d20e1717aa5baaee02ef984 Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sat, 3 Feb 2024 11:58:25 +0300 Subject: [PATCH 070/130] add license --- volatility3/framework/plugins/windows/truecrypt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py index 988c900ea..feba56965 100644 --- a/volatility3/framework/plugins/windows/truecrypt.py +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -1,3 +1,7 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from typing import Iterable, Generator, List, Tuple from volatility3.framework import constants, interfaces, renderers From cf9029fd85ae090034a253e3dcd819a0381b6442 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 4 Feb 2024 00:43:33 +0100 Subject: [PATCH 071/130] Fixing the year in the Copyright --- volatility3/framework/plugins/windows/iat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 73976fb86..d2fdc0ad8 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,4 +1,4 @@ -# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 import logging, io, pefile From d44c0fef36dfe1f7a83cc25211073489a56820be Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sun, 4 Feb 2024 11:21:14 +0300 Subject: [PATCH 072/130] run Black formatter --- .../framework/plugins/windows/truecrypt.py | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py index feba56965..81250a749 100644 --- a/volatility3/framework/plugins/windows/truecrypt.py +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -17,6 +17,7 @@ from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import modules + class Passphrase(interfaces.plugins.PluginInterface): """TrueCrypt Cached Passphrase Finder""" @@ -27,78 +28,75 @@ class Passphrase(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[RequirementInterface]: return [ requirements.ModuleRequirement( - 'kernel', - description='Windows kernel', - architectures=['Intel32', 'Intel64'] + "kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name='modules', - component=modules.Modules, - version=(1, 1, 0) + name="modules", component=modules.Modules, version=(1, 1, 0) ), requirements.IntRequirement( name="min-length", description="Minimum length of passphrases to identify", default=5, - optional=True + optional=True, ), ] - - def scan_module(self, module_base: int, layer_name: str) -> Generator[Tuple[int, str], None, None]: + + def scan_module( + self, module_base: int, layer_name: str + ) -> Generator[Tuple[int, str], None, None]: """Scans the TrueCrypt kernel module for cached passphrases. - + Args: module_base: the module's DLL base layer_name: the name of the layer in which the module resides - + Generates: A tuple of the offset at which a password is found, and the password """ pe_table_name = intermed.IntermediateSymbolTable.create( - self.context, - self.config_path, - "windows", - "pe", - class_types=pe.class_types + self.context, self.config_path, "windows", "pe", class_types=pe.class_types ) dos_header: pe.IMAGE_DOS_HEADER = self.context.object( - pe_table_name + constants.BANG + '_IMAGE_DOS_HEADER', + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", layer_name, module_base, ) data_section: StructType = next( - sec for sec in dos_header.get_nt_header().get_sections() - if array_to_string(sec.Name) == '.data' + sec + for sec in dos_header.get_nt_header().get_sections() + if array_to_string(sec.Name) == ".data" ) base: int = data_section.VirtualAddress + module_base size: int = data_section.Misc.VirtualSize # Looking at `Length` in TrueCrypt/Common/Password.h::Password struct DWORD_SIZE_BYTES: int = 4 - format = DataFormatInfo(length=DWORD_SIZE_BYTES, byteorder="little", signed=True) + format = DataFormatInfo( + length=DWORD_SIZE_BYTES, byteorder="little", signed=True + ) int32 = ObjectTemplate( - Integer, - pe_table_name + constants.BANG + 'int', - data_format=format + Integer, pe_table_name + constants.BANG + "int", data_format=format ) count, not_aligned = divmod(size, DWORD_SIZE_BYTES) if not_aligned: raise ValueError("PE data section not DWORD-aligned!") lengths = self.context.object( - pe_table_name + constants.BANG + 'array', + pe_table_name + constants.BANG + "array", layer_name, base, count=count, subtype=int32, ) - min_length = self.config.get('min-length') + min_length = self.config.get("min-length") for length in lengths: # TrueCrypt maximum password length is 64 # (see TrueCrypt/Common/Password.h) if not min_length <= length <= 64: continue - offset = length.vol['offset'] + DWORD_SIZE_BYTES + offset = length.vol["offset"] + DWORD_SIZE_BYTES passphrase: Bytes = self.context.object( - pe_table_name + constants.BANG + 'bytes', + pe_table_name + constants.BANG + "bytes", layer_name, offset, length=length, @@ -110,30 +108,31 @@ class Passphrase(interfaces.plugins.PluginInterface): # TrueCrypt/Common/Password.h::Password struct is padded with # 3 zero bytes to keep 64-byte alignment. buf: Bytes = self.context.object( - pe_table_name + constants.BANG + 'bytes', + pe_table_name + constants.BANG + "bytes", layer_name, - offset + length + 1, # +1 for '\0'-terminated password string - length=3 + offset + length + 1, # +1 for '\0'-terminated password string + length=3, ) if any(buf): continue # Password found. - yield offset, passphrase.decode(encoding='ascii') - + yield offset, passphrase.decode(encoding="ascii") + def _generator(self): kernel = self.context.modules[self.config["kernel"]] mods: Iterable[ObjectInterface] = modules.Modules.list_modules( - self.context, - kernel.layer_name, - kernel.symbol_table_name + self.context, kernel.layer_name, kernel.symbol_table_name ) truecrypt_module_base = next( - mod.DllBase for mod in mods - if mod.BaseDllName.get_string().lower() == 'truecrypt.sys' + mod.DllBase + for mod in mods + if mod.BaseDllName.get_string().lower() == "truecrypt.sys" ) - for offset, password in self.scan_module(truecrypt_module_base, kernel.layer_name): + for offset, password in self.scan_module( + truecrypt_module_base, kernel.layer_name + ): yield (0, (format_hints.Hex(offset), len(password), password)) - + def run(self) -> renderers.TreeGrid: return renderers.TreeGrid( [ @@ -141,5 +140,5 @@ class Passphrase(interfaces.plugins.PluginInterface): ("Length", int), ("Password", str), ], - self._generator() + self._generator(), ) From 6a384c197c7cc7429ea14417cec8703a5f759395 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 7 Feb 2024 09:04:55 +0000 Subject: [PATCH 073/130] Linux: update mm_struct extension so that _get_maple_tree_iter and _get_mmap_iter are more clearly identified, showing that get_vma_iter should be used instead --- .../symbols/linux/extensions/__init__.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d73d0cfb9..b57803d20 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -447,12 +447,14 @@ class maple_tree(objects.StructType): class mm_struct(objects.StructType): - def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns an iterator for the mmap list member of an mm_struct.""" + def _get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns an iterator for the mmap list member of an mm_struct. Use this only if + required, get_vma_iter() will choose the correct _get_maple_tree_iter() or + _get_mmap_iter() automatically as required.""" if not self.has_member("mmap"): raise AttributeError( - "get_mmap_iter called on mm_struct where no mmap member exists." + "_get_mmap_iter called on mm_struct where no mmap member exists." ) if not self.mmap: return None @@ -466,12 +468,14 @@ class mm_struct(objects.StructType): seen.add(link.vol.offset) link = link.vm_next - def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns an iterator for the mm_mt member of an mm_struct.""" + def _get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns an iterator for the mm_mt member of an mm_struct. Use this only if + required, get_vma_iter() will choose the correct _get_maple_tree_iter() or + get_mmap_iter() automatically as required.""" if not self.has_member("mm_mt"): raise AttributeError( - "get_maple_tree_iter called on mm_struct where no mm_mt member exists." + "_get_maple_tree_iter called on mm_struct where no mm_mt member exists." ) symbol_table_name = self.get_symbol_table_name() for vma_pointer in self.mm_mt.get_slot_iter(): @@ -487,9 +491,9 @@ class mm_struct(objects.StructType): """Returns an iterator for the VMAs in an mm_struct. Automatically choosing the mmap or mm_mt as required.""" if self.has_member("mmap"): - yield from self.get_mmap_iter() + yield from self._get_mmap_iter() elif self.has_member("mm_mt"): - yield from self.get_maple_tree_iter() + yield from self._get_maple_tree_iter() else: raise AttributeError("Unable to find mmap or mm_mt in mm_struct") From 18220498ae572fadf6e47c3e38ffe65c24447183 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 7 Feb 2024 09:07:19 +0000 Subject: [PATCH 074/130] Linux: update linux.pslist plugin to use task.mm.get_vma_iter() so that it works correctly on newer kernels --- volatility3/framework/plugins/linux/pslist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 9afd13e5a..771040dc0 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -17,7 +17,7 @@ class PsList(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 2, 0) + _version = (2, 2, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -128,7 +128,7 @@ class PsList(interfaces.plugins.PluginInterface): else: # Find the vma that belongs to the main ELF of the process file_output = "Error outputting file" - for v in task.mm.get_mmap_iter(): + for v in task.mm.get_vma_iter(): if v.vm_start == task.mm.start_code: file_handle = elfs.Elfs.elf_dump( self.context, From 9127509db9a8bcfffc971e7b959d429ffcecea6e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:13:14 +0000 Subject: [PATCH 075/130] CLI: Reblack after updating PR --- volatility3/cli/__init__.py | 45 ++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 0e7ba41ee..9393c0805 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -106,7 +106,7 @@ class CommandLine: ) # Load up system defaults - delayed_logs, default_config = self.load_system_defaults('vol.json') + delayed_logs, default_config = self.load_system_defaults("vol.json") parser = volargparse.HelpfulArgParser( add_help=False, @@ -280,7 +280,7 @@ class CommandLine: if partial_args.cache_path: constants.CACHE_PATH = partial_args.cache_path - + vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") @@ -473,28 +473,47 @@ class CommandLine: ) return requirements.URIRequirement.location_from_file(filename) - def load_system_defaults(self, filename: str) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: + def load_system_defaults( + self, filename: str + ) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: """Modify the main configuration based on the default configuration override""" # Build the config path - default_config_path = os.path.join(os.path.expanduser("~"), ".config", "volatility3", filename) - if sys.platform == 'win32': - default_config_path = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3", - filename) + default_config_path = os.path.join( + os.path.expanduser("~"), ".config", "volatility3", filename + ) + if sys.platform == "win32": + default_config_path = os.path.join( + os.environ.get("APPDATA", os.path.expanduser("~")), + "volatility3", + filename, + ) delayed_logs = [] # Process it if the files exist if os.path.exists(default_config_path): - with open(default_config_path, 'rb') as config_json: + with open(default_config_path, "rb") as config_json: result = json.load(config_json) if not isinstance(result, dict): - delayed_logs.append((logging.INFO, - f'Default configuration file {default_config_path} does not contain a dictionary')) + delayed_logs.append( + ( + logging.INFO, + f"Default configuration file {default_config_path} does not contain a dictionary", + ) + ) else: delayed_logs.append( - (logging.INFO, f"Loading default configuration options from {default_config_path}")) - delayed_logs.append((logging.DEBUG, - f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}")) + ( + logging.INFO, + f"Loading default configuration options from {default_config_path}", + ) + ) + delayed_logs.append( + ( + logging.DEBUG, + f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}", + ) + ) return delayed_logs, result return delayed_logs, {} From a66a4461906e7661b957559634559a56ee6356f6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:16:35 +0000 Subject: [PATCH 076/130] CLI: Reblack after updating PR - take 2 --- volatility3/cli/volshell/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 40f42f3af..9e74acfec 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -56,7 +56,7 @@ class VolShell(cli.CommandLine): framework.require_interface_version(2, 0, 0) # Load up system defaults - delayed_logs, default_config = self.load_system_defaults('volshell.json') + delayed_logs, default_config = self.load_system_defaults("volshell.json") parser = argparse.ArgumentParser( prog=self.CLI_NAME, @@ -151,10 +151,12 @@ class VolShell(cli.CommandLine): default=constants.CACHE_PATH, type=str, ) - parser.add_argument("--offline", - help = "Do not search online for additional JSON files", - default = False, - action = 'store_true') + parser.add_argument( + "--offline", + help="Do not search online for additional JSON files", + default=False, + action="store_true", + ) # Volshell specific flags os_specific = parser.add_mutually_exclusive_group(required=False) From 7b42ef0a9f23ecb6b4fff2425c0ac33c85b30d04 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:59:50 +0000 Subject: [PATCH 077/130] Documentation: Add in more information about overriding configuration values --- doc/source/vol-cli.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index cc6f7fe6a..7b91e815d 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -143,3 +143,18 @@ Options `hivescan` would match `windows.registry.hivescan.HiveScan`, but `pslist` is ambiguous because it could match `windows.pslist` or `linux.pslist`. + +Overriding options +------------------ + +The default values for the command line interface are defined by constants within the code, +but can be overridden by creating a JSON file (`%APPDATA%/volatility3/vol.json` for Windows +systems, or `~/.config/volatility3/vol.json` or `volshell.json` for all others). + +The format of this file is a JSON dictionary, containing the options above and their value. +It should be noted that the ordering is (`<` means is overridden by): + +`in-built default value < config file value < command line parameter` + +It should also be noted that boolean flags (such as `offline`) that are overridden as true will +not be unset by not specifying the command line flag. From 63a325c5157845d607d119971928657fb02ecdc2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 23:44:12 +0000 Subject: [PATCH 078/130] CLI: Add support for filtering lines from output --- volatility3/cli/__init__.py | 12 +++++++++++- volatility3/cli/text_renderer.py | 9 +++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 91bda7c66..2c6dde00a 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -22,6 +22,7 @@ import traceback from typing import Any, Dict, Type, Union from urllib import parse, request +from volatility3.cli import text_filter import volatility3.plugins import volatility3.symbols from volatility3 import framework @@ -230,6 +231,12 @@ class CommandLine: default=False, action="store_true", ) + parser.add_argument( + "--filters", + help="List of filters to apply to the output (in the form of [+-]columname,pattern[!])", + default=[], + action="append", + ) # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. @@ -444,7 +451,10 @@ class CommandLine: try: # Construct and run the plugin if constructed: - renderers[args.renderer]().render(constructed.run()) + grid = constructed.run() + renderer = renderers[args.renderer]() + renderer.filter = text_filter.CLIFilter(grid, args.filters) + renderer.render(grid) except exceptions.VolatilityException as excp: self.process_exceptions(excp) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 6e58ee68d..b0ba6baa7 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -10,6 +10,7 @@ import string import sys from functools import wraps from typing import Any, Callable, Dict, List, Tuple +from volatility3.cli import text_filter from volatility3.framework import interfaces, renderers from volatility3.framework.renderers import format_hints @@ -134,6 +135,7 @@ class CLIRenderer(interfaces.renderers.Renderer): name = "unnamed" structured_output = False + filter: text_filter.CLIFilter = None class QuickTextRenderer(CLIRenderer): @@ -172,6 +174,9 @@ class QuickTextRenderer(CLIRenderer): outfd.write("\n{}\n".format("\t".join(line))) def visitor(node: interfaces.renderers.TreeNode, accumulator): + if self.filter and self.filter.filter(node.values): + return accumulator + accumulator.write("\n") # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case accumulator.write( @@ -306,6 +311,10 @@ class PrettyTextRenderer(CLIRenderer): max_column_widths[tree_indent_column] = max( max_column_widths.get(tree_indent_column, 0), node.path_depth ) + + if self.filter and self.filter.filter(node.values): + return accumulator + line = {} for column_index in range(len(grid.columns)): column = grid.columns[column_index] From d8d3e157afab853fbffff394d7d1bc371025664e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 23:46:30 +0000 Subject: [PATCH 079/130] CLI: Don't forget the core text filter class --- volatility3/cli/text_filter.py | 96 ++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 volatility3/cli/text_filter.py diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py new file mode 100644 index 000000000..314af9a7a --- /dev/null +++ b/volatility3/cli/text_filter.py @@ -0,0 +1,96 @@ +import logging +from typing import Any, List, Optional +from volatility3.framework import constants, interfaces +import re + +vollog = logging.getLogger(__name__) + + +class CLIFilter: + def __init__(self, treegrid, filters: List[str]): + self._filters = self._prepare(treegrid, filters) + + def _prepare(self, treegrid: interfaces.renderers.TreeGrid, filters: List[str]): + """Runs through the filter strings and creates the necessary filter objects""" + output = [] + + for filter in filters: + exclude = False + regex = False + pattern = None + column_name = None + if filter.startswith("-"): + exclude = True + filter = filter[1:] + elif filter.startswith("+"): + filter = filter[1:] + components = filter.split(",") + if len(components) < 2: + pattern = components[0] + else: + column_name = components[0] + pattern = ",".join(components[1:]) + if pattern and pattern.endswith("!"): + regex = True + pattern = pattern[:-1] + column_num = None + if column_name: + for num, column in enumerate(treegrid.columns): + if column_name.lower() in column.name.lower(): + column_num = num + break + if pattern: + output.append(ColumnFilter(column_num, pattern, regex, exclude)) + + vollog.log(constants.LOGLEVEL_VVV, "Filters:\n" + repr(output)) + + return output + + def filter( + self, + row: List[Any], + ) -> bool: + """Filters the row based on each of the column_filters""" + found = any([column_filter.found(row) for column_filter in self._filters]) + return not found + + +class ColumnFilter: + def __init__( + self, + column_num: Optional[int], + pattern: str, + regex: bool = False, + exclude: bool = False, + ) -> None: + self.column_num = column_num + self.pattern = pattern + self.exclude = exclude + self.regex = regex + + def find(self, item) -> bool: + """Identifies whether an item is found in the appropriate column""" + try: + if self.regex: + return re.search(self.pattern, f"{item}") + return self.pattern in f"{item}" + except IOError: + return False + + def found(self, row: List[Any]) -> bool: + """Determines whether a row should be filtered + + If the classes exclude value is false, and the necessary pattern is found, the row is not filtered, + otherwise it is filtered. + """ + if self.column_num is None: + found = any([self.find(x) for x in row]) + else: + found = self.find(row[self.column_num]) + if self.exclude: + return not found + return found + + def __repr__(self) -> str: + """Returns a display of a column filter""" + return f"ColumnFilter(column={self.column_num},exclude={self.exclude},regex={self.regex},pattern={self.pattern})" From 865f004f5c6628c230c3dedde0e04573aa4e1339 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 9 Feb 2024 00:00:21 +0000 Subject: [PATCH 080/130] CLI: Ensure no filters still returns results --- volatility3/cli/text_filter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 314af9a7a..948e969df 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -51,6 +51,8 @@ class CLIFilter: row: List[Any], ) -> bool: """Filters the row based on each of the column_filters""" + if not self._filters: + return False found = any([column_filter.found(row) for column_filter in self._filters]) return not found From 683319bd7f96cef33e4b1cc4333d4c0835310113 Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 12 Feb 2024 08:47:02 +0000 Subject: [PATCH 081/130] Linux: add deprecation warning for get_maple_tree_iter and get_mmap_iter mm_struct extensions. --- .../symbols/linux/extensions/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b57803d20..f9db30498 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -447,6 +447,16 @@ class maple_tree(objects.StructType): class mm_struct(objects.StructType): + + def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """ + Deprecated: Use either get_vma_iter() or _get_mmap_iter(). + """ + vollog.warning( + "This method has been deprecated in favour of using the get_vma_iter() method." + ) + yield from self.get_vma_iter() + def _get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or @@ -468,6 +478,15 @@ class mm_struct(objects.StructType): seen.add(link.vol.offset) link = link.vm_next + def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """ + Deprecated: Use either get_vma_iter() or _get_maple_tree_iter(). + """ + vollog.warning( + "This method has been deprecated in favour of using the get_vma_iter() method." + ) + yield from self.get_vma_iter() + def _get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mm_mt member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or From 0e75140ec5a36e8f9933eee90cc74f0e06789242 Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 12 Feb 2024 09:35:06 +0000 Subject: [PATCH 082/130] Linux: update module extension with mod_mem_type property and add try/except for get methods --- .../symbols/linux/extensions/__init__.py | 75 ++++++++++++------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3557f3a17..fb60e3b97 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -15,6 +15,7 @@ from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATE from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES from volatility3.framework.constants.linux import CAPABILITIES from volatility3.framework import exceptions, objects, interfaces, symbols +from volatility3.framework.renderers import UnparsableValue from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed @@ -26,9 +27,10 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): - def _get_mod_mem_type(self): - """Attempt to get the mod_mem_type enum once to allow repeated access from other functions""" - if not self.has_member("mod_mem_type"): + @property + def mod_mem_type(self): + """Return the mod_mem_type enum choices if available or None if not""" + if not self.has_member("_mod_mem_type"): try: vmlinux = linux.LinuxUtilities.get_module_from_volobj_type( self._context, self @@ -36,31 +38,41 @@ class module(generic.GenericIntelProcess): # mod_mem_type and module_memory were added in kernel 6.4 which replaces # module_layout for storing the information around core_layout etc. # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information - self.mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices + self._mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices except exceptions.SymbolError: vollog.debug( f"Unable to find mod_mem_type enum. This is expected on kernels <6.4 but may cause issues with later kernels" ) - self.mod_mem_type = None + self._mod_mem_type = None + return self._mod_mem_type def get_module_base(self): - self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base + try: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base + except: + raise AttributeError( + "module -> get_module_base: Unable to get module base. Cannot read base from MOD_TEXT." + ) else: if self.has_member("core_layout"): return self.core_layout.base - else: + elif self.has_member("module_core"): return self.module_core + raise AttributeError("module -> get_module_base: Unable to get module base") def get_init_size(self): - self._get_mod_mem_type() if self.mod_mem_type: - return ( - self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size - + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size - ) + try: + return ( + self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size + + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size + ) + except: + raise AttributeError( + "module -> get_init_size: Unable to determine .init section size of module. Cannot read size of MOD_INIT_TEXT, MOD_INIT_DATA, and MOD_INIT_RODATA" + ) else: if self.has_member("init_layout"): return self.init_layout.size @@ -71,14 +83,19 @@ class module(generic.GenericIntelProcess): ) def get_core_size(self): - self._get_mod_mem_type() if self.mod_mem_type: - return ( - self.mem[self.mod_mem_type["MOD_TEXT"]].size - + self.mem[self.mod_mem_type["MOD_DATA"]].size - + self.mem[self.mod_mem_type["MOD_RODATA"]].size - + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size - ) + try: + return ( + self.mem[self.mod_mem_type["MOD_TEXT"]].size + + self.mem[self.mod_mem_type["MOD_DATA"]].size + + self.mem[self.mod_mem_type["MOD_RODATA"]].size + + self.mem[self.mod_mem_type["MOD_RO_AFTER_INIT"]].size + ) + except KeyError: + raise AttributeError( + "module -> get_core_size: Unable to determine core size of module. Cannot read size of MOD_TEXT, MOD_DATA, MOD_RODATA, and MOD_RO_AFTER_INIT." + ) + else: if self.has_member("core_layout"): return self.core_layout.size @@ -89,9 +106,13 @@ class module(generic.GenericIntelProcess): ) def get_module_core(self): - self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_TEXT"]].base + try: + return self.mem[self.mod_mem_type["MOD_TEXT"]].base + except KeyError: + raise AttributeError( + "module -> get_module_core: Unable to get module core. Cannot read base from MOD_TEXT." + ) else: if self.has_member("core_layout"): return self.core_layout.base @@ -100,9 +121,13 @@ class module(generic.GenericIntelProcess): raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - self._get_mod_mem_type() if self.mod_mem_type: - return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base + try: + return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base + except KeyError: + raise AttributeError( + "module -> get_module_core: Unable to get module init. Cannot read base from MOD_INIT_TEXT." + ) else: if self.has_member("init_layout"): return self.init_layout.base From 53d4ec66ebb774db76ef1999832662ecc541e56b Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 12 Feb 2024 09:36:33 +0000 Subject: [PATCH 083/130] Linux: Remove unnecessary import from linux extension. --- volatility3/framework/symbols/linux/extensions/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index fb60e3b97..c259d16aa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -15,7 +15,6 @@ from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATE from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES from volatility3.framework.constants.linux import CAPABILITIES from volatility3.framework import exceptions, objects, interfaces, symbols -from volatility3.framework.renderers import UnparsableValue from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed From cc35fecf91b1d2704c87031427c623e066ac968a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 19:40:37 +1100 Subject: [PATCH 084/130] Linux: Add library_list plugin and other ELF related code enhacements. - Add library_list plugin - Add ELF dynamic table enum types in elf.json - Update missing program header enum types in elf.json - Add PAGE constants - Add ELF ident and class enums - Replace ELF hardcoded type numbers for enum description matching - Fix unmanaged ValueError exception issue in Elf64Layer::_load_segments() --- .../framework/constants/linux/__init__.py | 26 +++ volatility3/framework/layers/elf.py | 11 +- volatility3/framework/plugins/linux/elfs.py | 34 ++-- .../framework/plugins/linux/library_list.py | 169 ++++++++++++++++++ volatility3/framework/symbols/linux/elf.json | 138 +++++++++++++- .../symbols/linux/extensions/__init__.py | 3 +- .../framework/symbols/linux/extensions/elf.py | 122 +++++++++++-- 7 files changed, 464 insertions(+), 39 deletions(-) create mode 100644 volatility3/framework/plugins/linux/library_list.py diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 6e8883f19..5e82e580e 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -5,11 +5,15 @@ Linux-specific values that aren't found in debug symbols """ +from enum import IntEnum KERNEL_NAME = "__kernel__" # arch/x86/include/asm/page_types.h PAGE_SHIFT = 12 +PAGE_SIZE = 1 << PAGE_SHIFT +PAGE_MASK = ~(PAGE_SIZE - 1) + """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" # include/linux/sched.h @@ -281,3 +285,25 @@ CAPABILITIES = ( ) ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1 + + +class ELF_IDENT(IntEnum): + """ELF header e_ident indexes""" + + EI_MAG0 = 0 + EI_MAG1 = 1 + EI_MAG2 = 2 + EI_MAG3 = 3 + EI_CLASS = 4 + EI_DATA = 5 + EI_VERSION = 6 + EI_OSABI = 7 + EI_PAD = 8 + + +class ELF_CLASS(IntEnum): + """ELF header class types""" + + ELFCLASSNONE = 0 + ELFCLASS32 = 1 + ELFCLASS64 = 2 diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index b2fd6d4d1..6bd5c2d63 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -6,9 +6,11 @@ import struct from typing import Optional from volatility3.framework import exceptions, interfaces, constants +from volatility3.framework.constants.linux import ELF_CLASS from volatility3.framework.layers import segmented from volatility3.framework.symbols import intermed + vollog = logging.getLogger(__name__) @@ -21,7 +23,7 @@ class Elf64Layer(segmented.SegmentedLayer): _header_struct = struct.Struct(" 0 ): diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index e688ecb42..7171a6616 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -14,8 +14,14 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux.extensions import elf +from volatility3.framework.constants.linux import ( + PAGE_SIZE, + PAGE_MASK, + ELF_MAX_EXTRACTION_SIZE, +) from volatility3.plugins.linux import pslist + vollog = logging.getLogger(__name__) @@ -23,7 +29,7 @@ class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -87,7 +93,10 @@ class Elfs(plugins.PluginInterface): sections = {} # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ? for phdr in elf_object.get_program_headers(): - if phdr.p_type != 1: # PT_LOAD = 1 + try: + if phdr.p_type.description != "PT_LOAD": + continue + except ValueError: continue start = phdr.p_vaddr @@ -95,18 +104,18 @@ class Elfs(plugins.PluginInterface): end = start + size # Use complete memory pages for dumping - # If start isn't a multiple of 4096, stick to the highest multiple < start - # If end isn't a multiple of 4096, stick to the lowest multiple > end - if start % 4096: - start = start & ~0xFFF + # If start isn't a multiple of a page, stick to the highest multiple < start + # If end isn't a multiple of a page, stick to the lowest multiple > end + if start % PAGE_SIZE: + start = start & PAGE_MASK - if end % 4096: - end = (end & ~0xFFF) + 4096 + if end % PAGE_SIZE: + end = (end & PAGE_MASK) + PAGE_SIZE real_size = end - start # Check if ELF has a legitimate size - if real_size < 0 or real_size > constants.linux.ELF_MAX_EXTRACTION_SIZE: + if real_size < 0 or real_size > ELF_MAX_EXTRACTION_SIZE: raise ValueError(f"The claimed size of the ELF is invalid: {real_size}") sections[start] = real_size @@ -140,12 +149,7 @@ class Elfs(plugins.PluginInterface): for vma in task.mm.get_vma_iter(): hdr = proc_layer.read(vma.vm_start, 4, pad=True) - if not ( - hdr[0] == 0x7F - and hdr[1] == 0x45 - and hdr[2] == 0x4C - and hdr[3] == 0x46 - ): + if hdr != b"\x7fELF": continue path = vma.get_name(self.context, task) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py new file mode 100644 index 000000000..ed5545347 --- /dev/null +++ b/volatility3/framework/plugins/linux/library_list.py @@ -0,0 +1,169 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from typing import Iterable, Tuple + +from volatility3.framework import interfaces, renderers, constants, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.framework.objects import utility +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.linux.extensions import elf +from volatility3.plugins.linux import pslist + + +vollog = logging.getLogger(__name__) + + +class LibraryList(interfaces.plugins.PluginInterface): + """Enumerate libraries loaded into processes""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + ), + requirements.ListRequirement( + name="pids", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + ] + + def get_libdl_libraries( + self, proc_layer_name: str, vma_start: int + ) -> interfaces.objects.ObjectInterface: + """Get the ELF link map objects for the given VMA address + + Args: + proc_layer_name (str): Name of the process layer + vma_start (int): VMA start address + + Yields: + ELF link map objects for the given VMA address + """ + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "linux", + "elf", + class_types=elf.class_types, + ) + elf_object = self.context.object( + elf_table_name + constants.BANG + "Elf", + offset=vma_start, + layer_name=proc_layer_name, + ) + + if not elf_object or not elf_object.is_valid(): + return None + + kernel = self.context.modules[self.config["kernel"]] + + try: + for link_map in elf_object.get_link_maps(kernel.symbol_table_name): + if link_map.l_addr and link_map.l_name: + yield link_map + except exceptions.InvalidAddressException: + # Protection against memory smear in this VMA + pass + + def get_libdl_maps( + self, task: interfaces.objects.ObjectInterface, proc_layer_name: str + ) -> interfaces.objects.ObjectInterface: + """Get the ELF link maps objects for a task + + Args: + task (task_struct): A reference task + proc_layer_name (str): Name of the process layer + + Yields: + ELF link map objects + """ + + link_map_seen = set() + for vma in task.mm.get_vma_iter(): + for link_map in self.get_libdl_libraries(proc_layer_name, vma.vm_start): + if link_map.l_addr in link_map_seen: + continue + + yield link_map + link_map_seen.add(link_map.l_addr) + + def get_task_libraries( + self, task: interfaces.objects.ObjectInterface + ) -> Tuple[int, str]: + """Get the task libraries from the ELF headers found within the memory maps + + Args: + task (task_struct): The reference task + + Yields: + Tuples with a ELF link map address and name + """ + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + return + + for elf_link_map in self.get_libdl_maps(task, proc_layer_name): + name = elf_link_map.get_name() + if not name: + continue + yield elf_link_map.l_addr, name + + def get_tasks_libraries( + self, + tasks: Iterable[interfaces.objects.ObjectInterface], + ) -> Iterable[Tuple[str, int, int, str]]: + """Get the task libraries from the ELF headers found within the memory maps for + all the tasks. + + Args: + tasks: An iterable of tasks + + Yields: + Tuples with a task name, task tgid, an ELF link map address and name + """ + for task in tasks: + task_name = utility.array_to_string(task.comm) + for linkmap_addr, linkmap_name in self.get_task_libraries(task): + yield task_name, task.tgid, linkmap_addr, linkmap_name + + def _format_fields(self, fields): + task_name, task_pid, addr, name = fields + return task_name, task_pid, format_hints.Hex(addr), name + + def _generator( + self, tasks: Iterable[interfaces.objects.ObjectInterface] + ) -> Iterable[Tuple[int, Tuple]]: + for fields in self.get_tasks_libraries(tasks): + yield 0, self._format_fields(fields) + + def run(self): + pids = self.config.get("pids") + pid_filter = pslist.PsList.create_pid_filter(pids) + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=pid_filter + ) + + headers = [ + ("Name", str), + ("Pid", int), + ("LoadAddress", format_hints.Hex), + ("Path", str), + ] + + return renderers.TreeGrid(headers, self._generator(tasks)) diff --git a/volatility3/framework/symbols/linux/elf.json b/volatility3/framework/symbols/linux/elf.json index 76cd8a2ec..e0a95bbba 100644 --- a/volatility3/framework/symbols/linux/elf.json +++ b/volatility3/framework/symbols/linux/elf.json @@ -270,8 +270,8 @@ "d_tag": { "offset": 0, "type": { - "kind": "base", - "name": "long long" + "kind": "enum", + "name": "DtypeEnum64" } }, "d_ptr": { @@ -699,8 +699,8 @@ "d_tag": { "offset": 0, "type": { - "kind": "base", - "name": "long" + "kind": "enum", + "name": "DtypeEnum32" } }, "d_ptr": { @@ -905,11 +905,139 @@ "PT_PHDR": 6, "PT_TLS": 7, "PT_LOOS": 1610612736, + "PT_GNU_EH_FRAME": 1685382480, + "PT_GNU_STACK": 1685382481, + "PT_GNU_RELRO": 1685382482, + "PT_GNU_PROPERTY": 1685382483, "PT_HIOS": 1879048191, "PT_LOWPROC": 1879048192, "PT_HIPROC": 2147483647 }, "size": 4 + }, + "DtypeEnum32": { + "base": "long", + "constants": { + "DT_NULL": 0, + "DT_NEEDED": 1, + "DT_PLTRELSZ": 2, + "DT_PLTGOT": 3, + "DT_HASH": 4, + "DT_STRTAB": 5, + "DT_SYMTAB": 6, + "DT_RELA": 7, + "DT_RELASZ": 8, + "DT_RELAENT": 9, + "DT_STRSZ": 10, + "DT_SYMENT": 11, + "DT_INIT": 12, + "DT_FINI": 13, + "DT_SONAME": 14, + "DT_RPATH": 15, + "DT_SYMBOLIC": 16, + "DT_REL": 17, + "DT_RELSZ": 18, + "DT_RELENT": 19, + "DT_PLTREL": 20, + "DT_DEBUG": 21, + "DT_TEXTREL": 22, + "DT_JMPREL": 23, + "DT_BIND_NOW": 24, + "DT_INIT_ARRAY": 25, + "DT_FINI_ARRAY": 26, + "DT_INIT_ARRAYSZ": 27, + "DT_FINI_ARRAYSZ": 28, + "DT_RUNPATH": 29, + "DT_FLAGS": 30, + "DT_ENCODING": 32, + "DT_PREINIT_ARRAYSZ": 33, + "DT_SYMTAB_SHNDX": 34, + "DT_RELRSZ": 35, + "DT_RELR": 36, + "DT_RELRENT": 37, + "DT_NUM": 38, + "OLD_DT_LOOS": 1610612736, + "DT_LOOS": 1610612749, + "DT_HIOS": 1879044096, + "DT_VALRNGLO": 1879047424, + "DT_VALRNGHI": 1879047679, + "DT_ADDRRNGLO": 1879047680, + "DT_GNU_HASH": 1879047925, + "DT_ADDRRNGHI": 1879047935, + "DT_VERSYM": 1879048176, + "DT_RELACOUNT": 1879048185, + "DT_RELCOUNT": 1879048186, + "DT_FLAGS_1": 1879048187, + "DT_VERDEF": 1879048188, + "DT_VERDEFNUM": 1879048189, + "DT_VERNEED": 1879048190, + "DT_VERNEEDNUM": 1879048191, + "DT_LOPROC": 1879048192, + "DT_HIPROC": 2147483647 + }, + "size": 4 + }, + "DtypeEnum64": { + "base": "long long", + "constants": { + "DT_NULL": 0, + "DT_NEEDED": 1, + "DT_PLTRELSZ": 2, + "DT_PLTGOT": 3, + "DT_HASH": 4, + "DT_STRTAB": 5, + "DT_SYMTAB": 6, + "DT_RELA": 7, + "DT_RELASZ": 8, + "DT_RELAENT": 9, + "DT_STRSZ": 10, + "DT_SYMENT": 11, + "DT_INIT": 12, + "DT_FINI": 13, + "DT_SONAME": 14, + "DT_RPATH": 15, + "DT_SYMBOLIC": 16, + "DT_REL": 17, + "DT_RELSZ": 18, + "DT_RELENT": 19, + "DT_PLTREL": 20, + "DT_DEBUG": 21, + "DT_TEXTREL": 22, + "DT_JMPREL": 23, + "DT_BIND_NOW": 24, + "DT_INIT_ARRAY": 25, + "DT_FINI_ARRAY": 26, + "DT_INIT_ARRAYSZ": 27, + "DT_FINI_ARRAYSZ": 28, + "DT_RUNPATH": 29, + "DT_FLAGS": 30, + "DT_ENCODING": 32, + "DT_PREINIT_ARRAYSZ": 33, + "DT_SYMTAB_SHNDX": 34, + "DT_RELRSZ": 35, + "DT_RELR": 36, + "DT_RELRENT": 37, + "DT_NUM": 38, + "OLD_DT_LOOS": 1610612736, + "DT_LOOS": 1610612749, + "DT_HIOS": 1879044096, + "DT_VALRNGLO": 1879047424, + "DT_VALRNGHI": 1879047679, + "DT_ADDRRNGLO": 1879047680, + "DT_GNU_HASH": 1879047925, + "DT_ADDRRNGHI": 1879047935, + "DT_VERSYM": 1879048176, + "DT_RELACOUNT": 1879048185, + "DT_RELCOUNT": 1879048186, + "DT_FLAGS_1": 1879048187, + "DT_VERDEF": 1879048188, + "DT_VERDEFNUM": 1879048189, + "DT_VERNEED": 1879048190, + "DT_VERNEEDNUM": 1879048191, + "DT_LOPROC": 1879048192, + "DT_HIPROC": 2147483647 + }, + "size": 8 } }, "base_types": { @@ -958,7 +1086,7 @@ }, "metadata": { "producer": { - "version": "0.0.1", + "version": "0.0.2", "name": "ikelos-by-hand", "datetime": "2019-10-21T22:52:00" }, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d73d0cfb9..0a3db9fcd 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -7,14 +7,13 @@ import logging import socket as socket_module from typing import Generator, Iterable, Iterator, Optional, Tuple, List -from volatility3.framework import constants +from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.constants.linux import SOCK_TYPES, SOCK_FAMILY from volatility3.framework.constants.linux import IP_PROTOCOLS, IPV6_PROTOCOLS from volatility3.framework.constants.linux import TCP_STATES, NETLINK_PROTOCOLS from volatility3.framework.constants.linux import ETH_PROTOCOLS, BLUETOOTH_STATES from volatility3.framework.constants.linux import BLUETOOTH_PROTOCOLS, SOCKET_STATES from volatility3.framework.constants.linux import CAPABILITIES -from volatility3.framework import exceptions, objects, interfaces, symbols from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index fe85b194f..4b2b29b54 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -6,6 +6,10 @@ from typing import Dict, Tuple import logging from volatility3.framework import constants +from volatility3.framework.constants.linux import ( + ELF_IDENT, + ELF_CLASS, +) from volatility3.framework import objects, interfaces, exceptions vollog = logging.getLogger(__name__) @@ -59,13 +63,15 @@ class elf(objects.StructType): ei_class = self._context.object( symbol_table_name + constants.BANG + "unsigned char", layer_name=layer_name, - offset=object_info.offset + 0x4, + offset=object_info.offset + ELF_IDENT.EI_CLASS, ) - if ei_class == 1: + if ei_class == ELF_CLASS.ELFCLASS32: self._type_prefix = "Elf32_" - elif ei_class == 2: + self._ei_class_size = 32 + elif ei_class == ELF_CLASS.ELFCLASS64: self._type_prefix = "Elf64_" + self._ei_class_size = 64 else: raise ValueError(f"Unsupported ei_class value {ei_class}") @@ -140,36 +146,103 @@ class elf(objects.StructType): ) return section_headers + def get_link_maps(self, kernel_symbol_table_name): + """Get the ELF link map objects for the given VMA address + + Args: + kernel_symbol_table_name (str): Kernel symbol table name + + Yields: + The ELF link map objects + """ + got_entry_size = self._ei_class_size // 8 + + elf_symbol_table = self.get_symbol_table_name() + + link_maps_seen = set() + for phdr in self.get_program_headers(): + try: + if phdr.p_type.description != "PT_DYNAMIC": + continue + except ValueError: + continue + + for dsec in phdr.dynamic_sections(): + try: + if dsec.d_tag.description != "DT_PLTGOT": + continue + except ValueError: + continue + + got_start = dsec.d_ptr + + # link_map is stored at the second GOT entry + link_map_addr = got_start + got_entry_size + + # It needs the kernel symbol table to create a pointer + link_map_ptr = self._context.object( + kernel_symbol_table_name + constants.BANG + "pointer", + offset=link_map_addr, + layer_name=self.vol.layer_name, + ) + if not link_map_ptr: + continue + + linkmap_symname = ( + elf_symbol_table + constants.BANG + self._type_prefix + "LinkMap" + ) + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map_ptr, + layer_name=self.vol.layer_name, + ) + + while link_map and link_map.vol.offset != 0: + if link_map.vol.offset in link_maps_seen: + break + link_maps_seen.add(link_map.vol.offset) + + yield link_map + + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map.l_next, + layer_name=self.vol.layer_name, + ) + def _find_symbols(self): dt_strtab = None dt_symtab = None dt_strent = None for phdr in self.get_program_headers(): + # Find PT_DYNAMIC segment try: - # Find PT_DYNAMIC segment - if str(phdr.p_type.description) != "PT_DYNAMIC": + if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: - # If the p_type value is outside the ones declared in the enumeration, an - # exception is raised - return None + continue # This section contains pointers to the strtab, symtab, and strent sections for dsec in phdr.dynamic_sections(): - if dsec.d_tag == 5: + try: + dtag = dsec.d_tag.description + except ValueError: + continue + + if dtag == "DT_STRTAB": dt_strtab = dsec.d_ptr - elif dsec.d_tag == 6: + elif dtag == "DT_SYMTAB": dt_symtab = dsec.d_ptr - elif dsec.d_tag == 11: + elif dtag == "DT_SYMENT": # Size of the symtab symbol entry dt_strent = dsec.d_ptr break - if dt_strtab is None or dt_symtab is None or dt_strent is None: + if not (dt_strtab and dt_symtab and dt_strent): return None self._cached_symtab = dt_symtab @@ -274,15 +347,18 @@ class elf_phdr(objects.StructType): def get_vaddr(self): offset = self.__getattr__("p_vaddr") - if self._parent_e_type == 3: # ET_DYN - offset = self._parent_offset + offset + try: + if self._parent_e_type.description == "ET_DYN": + offset = self._parent_offset + offset + except ValueError: + pass return offset def dynamic_sections(self): # sanity check try: - if str(self.p_type.description) != "PT_DYNAMIC": + if self.p_type.description != "PT_DYNAMIC": return None except ValueError: # If the value is outside the ones declared in the enumeration, an @@ -314,10 +390,26 @@ class elf_phdr(objects.StructType): break +class elf_linkmap(objects.StructType): + def get_name(self): + try: + buf = self._context.layers.read(self.vol.layer_name, self.l_name, 256) + except exceptions.PagedInvalidAddressException: + # Protection against memory smear + return None + + idx = buf.find(b"\x00") + if idx != -1: + buf = buf[:idx] + return buf.decode() + + class_types = { "Elf": elf, "Elf64_Phdr": elf_phdr, "Elf32_Phdr": elf_phdr, "Elf32_Sym": elf_sym, "Elf64_Sym": elf_sym, + "Elf32_LinkMap": elf_linkmap, + "Elf64_LinkMap": elf_linkmap, } From 6e7b5b59416a0f0830660d6415144ec8ee0039ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 19:41:30 +1100 Subject: [PATCH 085/130] Linux: Add linux_library_list test case --- test/test_volatility.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/test_volatility.py b/test/test_volatility.py index aaad615bc..7b151fd6c 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -6,6 +6,7 @@ # import os +import re import subprocess import sys import shutil @@ -331,6 +332,32 @@ def test_linux_tty_check(image, volatility, python): assert rc == 0 +def test_linux_library_list(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.library_list.LibraryList", image, volatility, python + ) + + assert re.search( + rb"NetworkManager\s2363\s0x7f52cdda0000\s/lib/x86_64-linux-gnu/libnss_files.so.2", + out, + ) + assert re.search( + rb"gnome-settings-\s3807\s0x7f7e660b5000\s/lib/x86_64-linux-gnu/libbz2.so.1.0", + out, + ) + assert re.search( + rb"gdu-notificatio\s3878\s0x7f25ce33e000\s/usr/lib/x86_64-linux-gnu/libXau.so.6", + out, + ) + assert re.search( + rb"bash\s8600\s0x7fe78a85f000\s/lib/x86_64-linux-gnu/libnss_files.so.2", + out, + ) + + assert out.count(b"\n") >= 2677 + assert rc == 0 + + # MAC From cf81ceda8262b1bdfc528277cd405c24d572e320 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 13 Feb 2024 20:20:25 +1100 Subject: [PATCH 086/130] Fix CodeQL suggestion --- volatility3/framework/symbols/linux/extensions/elf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 4b2b29b54..828370fe8 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -351,6 +351,8 @@ class elf_phdr(objects.StructType): if self._parent_e_type.description == "ET_DYN": offset = self._parent_offset + offset except ValueError: + # Unknown ELF object file type. Anyway, if the ELF object file type is not a + # shared object (ET_DYN), the virtual address is 'p_vaddr'. pass return offset From cd08cb95fdf9d2503162020694fd4e3eddaf51d3 Mon Sep 17 00:00:00 2001 From: Alejandro Diego Date: Tue, 6 Feb 2024 09:26:49 -0500 Subject: [PATCH 087/130] Windows: Add filtering by offset to psscan Add capability to the psscan plugin to filter by specific offset. The filter would be used by other plugins to find specific offset using the psscan capabilities, like find hidden processes as a result of some dkom for example. The `--offset` flag argument should represent a physical address space. The flag is included in the following plugins: - dlllist - handles --- .../framework/plugins/windows/dlllist.py | 37 +++++++++++++---- .../framework/plugins/windows/handles.py | 41 ++++++++++++++----- .../framework/plugins/windows/psscan.py | 40 +++++++++++++++++- 3 files changed, 97 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index d73cea652..27eddc991 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -13,7 +13,7 @@ from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins import timeliner -from volatility3.plugins.windows import info, pslist +from volatility3.plugins.windows import info, pslist, psscan vollog = logging.getLogger(__name__) @@ -36,6 +36,9 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="pslist", component=pslist.PsList, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="psscan", component=psscan.PsScan, version=(1, 1, 0) + ), requirements.VersionRequirement( name="info", component=info.Info, version=(1, 0, 0) ), @@ -45,6 +48,11 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): description="Process IDs to include (all other processes are excluded)", optional=True, ), + requirements.IntRequirement( + name="offset", + description="Process offset in the physical address space", + optional=True, + ), requirements.BooleanRequirement( name="dump", description="Extract listed DLLs", @@ -221,6 +229,24 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) kernel = self.context.modules[self.config["kernel"]] + if self.config["offset"]: + procs = psscan.PsScan.scan_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=psscan.PsScan.create_offset_filter( + self.context.layers[kernel.layer_name], + self.config["offset"], + ), + ) + else: + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + return renderers.TreeGrid( [ ("PID", int), @@ -232,12 +258,5 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ("LoadTime", datetime.datetime), ("File output", str), ], - self._generator( - pslist.PsList.list_processes( - context=self.context, - layer_name=kernel.layer_name, - symbol_table=kernel.symbol_table_name, - filter_func=filter_func, - ) - ), + self._generator(procs=procs), ) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index ddd9cb78e..2c5fde1c9 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -9,7 +9,7 @@ from volatility3.framework import constants, exceptions, renderers, interfaces, from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints -from volatility3.plugins.windows import pslist +from volatility3.plugins.windows import pslist, psscan vollog = logging.getLogger(__name__) @@ -43,14 +43,22 @@ class Handles(interfaces.plugins.PluginInterface): description="Windows kernel", architectures=["Intel32", "Intel64"], ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="psscan", component=psscan.PsScan, version=(1, 1, 0) + ), requirements.ListRequirement( name="pid", element_type=int, description="Process IDs to include (all other processes are excluded)", optional=True, ), - requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + requirements.IntRequirement( + name="offset", + description="Process offset in the physical address space", + optional=True, ), ] @@ -416,6 +424,24 @@ class Handles(interfaces.plugins.PluginInterface): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) kernel = self.context.modules[self.config["kernel"]] + if self.config["offset"]: + procs = psscan.PsScan.scan_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=psscan.PsScan.create_offset_filter( + self.context.layers[kernel.layer_name], + self.config["offset"], + ), + ) + else: + procs = pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=filter_func, + ) + return renderers.TreeGrid( [ ("PID", int), @@ -426,12 +452,5 @@ class Handles(interfaces.plugins.PluginInterface): ("GrantedAccess", format_hints.Hex), ("Name", str), ], - self._generator( - pslist.PsList.list_processes( - self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func=filter_func, - ) - ), + self._generator(procs=procs), ) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 3d9ae5c1e..8b3afaf7c 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Iterable, Callable, Optional, Tuple +from typing import Iterable, Callable, List, Optional, Tuple from volatility3.framework import renderers, interfaces, layers, exceptions from volatility3.framework.configuration import requirements @@ -59,6 +59,44 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] + @classmethod + def create_offset_filter( + cls, + memory: interfaces.layers.DataLayerInterface, + offset: int = None, + exclude: bool = False, + ) -> Callable[[interfaces.objects.ObjectInterface], bool]: + """A factory for producing filter functions that filter based on the physical offset of the process. + + Args: + offset: A number that is the physical offset to be filtered out + memory: Memory object needed to do the offset mapping to physical. + exclude: Accept only tasks that are not the offset argument + Returns: + Filter function to be passed to the list of processes. + """ + if not isinstance(memory, interfaces.layers.DataLayerInterface): + raise TypeError("memory object requires an instance of DataLayerInterface") + + filter_func = lambda _: False + + # return physical offset in tuple -> (_, _, physical_offset, _, _) + # from the first item in the memory mapping list + virtual_to_physical_offset = lambda virtual_offset, memory: list( + memory.mapping(offset=virtual_offset, length=0) + )[0][2] + + if offset: + if exclude: + filter_func = ( + lambda x: virtual_to_physical_offset(x.vol.offset, memory) == offset + ) + else: + filter_func = ( + lambda x: virtual_to_physical_offset(x.vol.offset, memory) != offset + ) + return filter_func + @classmethod def scan_processes( cls, From 437a91375db004de5d0df26443d1bd71ee9f2b94 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 20 Feb 2024 06:39:42 +0000 Subject: [PATCH 088/130] Linux: add TODO for mm_struct for methods that should be removed when moving to version 3.0.0 --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f9db30498..27113965f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -448,6 +448,7 @@ class maple_tree(objects.StructType): class mm_struct(objects.StructType): + # TODO: As of version 3.0.0 this method should be removed def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """ Deprecated: Use either get_vma_iter() or _get_mmap_iter(). @@ -478,6 +479,7 @@ class mm_struct(objects.StructType): seen.add(link.vol.offset) link = link.vm_next + # TODO: As of version 3.0.0 this method should be removed def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """ Deprecated: Use either get_vma_iter() or _get_maple_tree_iter(). From a597ee17768b1513f8587642c0d00c9170007f46 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 20 Feb 2024 06:48:16 +0000 Subject: [PATCH 089/130] Linux: update minor version number due to API changes with linux extension for mm_struct --- volatility3/framework/constants/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09dded076..9aaafb933 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_MINOR = 6 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From 5e38ffa811bcf385c2b470351ed2518118c7d875 Mon Sep 17 00:00:00 2001 From: Alejandro Diego Date: Tue, 20 Feb 2024 15:07:15 -0500 Subject: [PATCH 090/130] Added support for virtual address filtering and validation of ph/v space --- .../framework/plugins/windows/dlllist.py | 3 +- .../framework/plugins/windows/handles.py | 3 +- .../framework/plugins/windows/psscan.py | 67 ++++++++++++++----- 3 files changed, 53 insertions(+), 20 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 27eddc991..f876dc1a2 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -235,7 +235,8 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kernel.layer_name, kernel.symbol_table_name, filter_func=psscan.PsScan.create_offset_filter( - self.context.layers[kernel.layer_name], + self.context, + kernel.layer_name, self.config["offset"], ), ) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 2c5fde1c9..d43d26ef1 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -430,7 +430,8 @@ class Handles(interfaces.plugins.PluginInterface): kernel.layer_name, kernel.symbol_table_name, filter_func=psscan.PsScan.create_offset_filter( - self.context.layers[kernel.layer_name], + self.context, + kernel.layer_name, self.config["offset"], ), ) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 8b3afaf7c..82a481070 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -59,42 +59,73 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] + @classmethod + def physical_offset_from_virtual(cls, context, layer_name, proc): + """Calculate the physical offset from the virtual offset of a process. + + Args: + context: The context containing layers and modules information. + layer_name: The name of the layer containing the process memory. + proc: The process object for which to calculate the physical offset. + + Returns: + int: The physical offset of the process. + Raises: + TypeError: If the primary layer is not an Intel layer. + """ + memory = context.layers[layer_name] + + if not isinstance(memory, layers.intel.Intel): + raise TypeError("Primary layer is not an intel layer") + + (_, _, ph_offset, _, _) = list( + memory.mapping(offset=proc.vol.offset, length=0) + )[0] + + return ph_offset + @classmethod def create_offset_filter( cls, - memory: interfaces.layers.DataLayerInterface, + context: interfaces.context.ContextInterface, + layer_name: str, offset: int = None, + physical: bool = True, exclude: bool = False, ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on the physical offset of the process. Args: offset: A number that is the physical offset to be filtered out - memory: Memory object needed to do the offset mapping to physical. exclude: Accept only tasks that are not the offset argument + Returns: Filter function to be passed to the list of processes. """ - if not isinstance(memory, interfaces.layers.DataLayerInterface): - raise TypeError("memory object requires an instance of DataLayerInterface") - filter_func = lambda _: False - # return physical offset in tuple -> (_, _, physical_offset, _, _) - # from the first item in the memory mapping list - virtual_to_physical_offset = lambda virtual_offset, memory: list( - memory.mapping(offset=virtual_offset, length=0) - )[0][2] - if offset: - if exclude: - filter_func = ( - lambda x: virtual_to_physical_offset(x.vol.offset, memory) == offset - ) + if physical: + if exclude: + filter_func = ( + lambda proc: cls.physical_offset_from_virtual( + context, layer_name, proc + ) + == offset + ) + else: + filter_func = ( + lambda proc: cls.physical_offset_from_virtual( + context, layer_name, proc + ) + != offset + ) else: - filter_func = ( - lambda x: virtual_to_physical_offset(x.vol.offset, memory) != offset - ) + if exclude: + lambda proc: proc.vol.offset == offset + else: + lambda proc: proc.vol.offset != offset + return filter_func @classmethod From 42c6a9ede6bbe3ff52b868144516679753c9fc8c Mon Sep 17 00:00:00 2001 From: Alejandro Diego Date: Tue, 20 Feb 2024 20:07:17 -0500 Subject: [PATCH 091/130] Remove unnecessary import and correctly filter_func var --- volatility3/framework/plugins/windows/psscan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 82a481070..5634298a8 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Iterable, Callable, List, Optional, Tuple +from typing import Iterable, Callable, Optional, Tuple from volatility3.framework import renderers, interfaces, layers, exceptions from volatility3.framework.configuration import requirements @@ -122,9 +122,9 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) else: if exclude: - lambda proc: proc.vol.offset == offset + filter_func = lambda proc: proc.vol.offset == offset else: - lambda proc: proc.vol.offset != offset + filter_func = lambda proc: proc.vol.offset != offset return filter_func From e10cff979af3917b3d0a67ec050c181e37bffed8 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 21 Feb 2024 12:50:05 +0000 Subject: [PATCH 092/130] Linux: update module extension with memoization for _mod_mem_type --- .../symbols/linux/extensions/__init__.py | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 3673b66df..8d11374c9 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -26,27 +26,33 @@ vollog = logging.getLogger(__name__) class module(generic.GenericIntelProcess): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._mod_mem_type = None # Initialize _mod_mem_type to None for memoization + @property def mod_mem_type(self): - """Return the mod_mem_type enum choices if available or None if not""" - if not self.has_member("_mod_mem_type"): + """Return the mod_mem_type enum choices if available or an empty dict if not""" + # mod_mem_type and module_memory were added in kernel 6.4 which replaces + # module_layout for storing the information around core_layout etc. + # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information + + if self._mod_mem_type is None: try: - vmlinux = linux.LinuxUtilities.get_module_from_volobj_type( - self._context, self - ) - # mod_mem_type and module_memory were added in kernel 6.4 which replaces - # module_layout for storing the information around core_layout etc. - # see commit ac3b43283923440900b4f36ca5f9f0b1ca43b70e for more information - self._mod_mem_type = vmlinux.get_enumeration("mod_mem_type").choices + self._mod_mem_type = self._context.symbol_space.get_enumeration( + self.get_symbol_table_name() + constants.BANG + "mod_mem_type" + ).choices except exceptions.SymbolError: vollog.debug( - f"Unable to find mod_mem_type enum. This is expected on kernels <6.4 but may cause issues with later kernels" + f"Unable to find mod_mem_type enum. This message can be ignored for kernels < 6.4" ) - self._mod_mem_type = None + # set to empty dict to show that the enum was not found, and so shouldn't be searched for again + self._mod_mem_type = {} return self._mod_mem_type def get_module_base(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base except: @@ -61,7 +67,7 @@ class module(generic.GenericIntelProcess): raise AttributeError("module -> get_module_base: Unable to get module base") def get_init_size(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size @@ -82,7 +88,7 @@ class module(generic.GenericIntelProcess): ) def get_core_size(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_TEXT"]].size @@ -105,7 +111,7 @@ class module(generic.GenericIntelProcess): ) def get_module_core(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base except KeyError: @@ -120,7 +126,7 @@ class module(generic.GenericIntelProcess): raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - if self.mod_mem_type: + if self.mod_mem_type: # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base except KeyError: From 3b5fde94208bbfc98c7b2a147cdd2341427b7097 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 21 Feb 2024 12:58:24 +0000 Subject: [PATCH 093/130] Linux: update module extension logic for choosing the correct values for different kernel versions --- .../symbols/linux/extensions/__init__.py | 60 +++++++++---------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 8d11374c9..751c80cb5 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -52,43 +52,41 @@ class module(generic.GenericIntelProcess): return self._mod_mem_type def get_module_base(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base - except: + except KeyError: raise AttributeError( "module -> get_module_base: Unable to get module base. Cannot read base from MOD_TEXT." ) - else: - if self.has_member("core_layout"): - return self.core_layout.base - elif self.has_member("module_core"): - return self.module_core + elif self.has_member("core_layout"): + return self.core_layout.base + elif self.has_member("module_core"): + return self.module_core raise AttributeError("module -> get_module_base: Unable to get module base") def get_init_size(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].size + self.mem[self.mod_mem_type["MOD_INIT_DATA"]].size + self.mem[self.mod_mem_type["MOD_INIT_RODATA"]].size ) - except: + except KeyError: raise AttributeError( "module -> get_init_size: Unable to determine .init section size of module. Cannot read size of MOD_INIT_TEXT, MOD_INIT_DATA, and MOD_INIT_RODATA" ) - else: - if self.has_member("init_layout"): - return self.init_layout.size - elif self.has_member("init_size"): - return self.init_size + elif self.has_member("init_layout"): + return self.init_layout.size + elif self.has_member("init_size"): + return self.init_size raise AttributeError( "module -> get_init_size: Unable to determine .init section size of module" ) def get_core_size(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return ( self.mem[self.mod_mem_type["MOD_TEXT"]].size @@ -100,44 +98,40 @@ class module(generic.GenericIntelProcess): raise AttributeError( "module -> get_core_size: Unable to determine core size of module. Cannot read size of MOD_TEXT, MOD_DATA, MOD_RODATA, and MOD_RO_AFTER_INIT." ) - - else: - if self.has_member("core_layout"): - return self.core_layout.size - elif self.has_member("core_size"): - return self.core_size + elif self.has_member("core_layout"): + return self.core_layout.size + elif self.has_member("core_size"): + return self.core_size raise AttributeError( "module -> get_core_size: Unable to determine core size of module" ) def get_module_core(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_TEXT"]].base except KeyError: raise AttributeError( "module -> get_module_core: Unable to get module core. Cannot read base from MOD_TEXT." ) - else: - if self.has_member("core_layout"): - return self.core_layout.base - elif self.has_member("module_core"): - return self.module_core + elif self.has_member("core_layout"): + return self.core_layout.base + elif self.has_member("module_core"): + return self.module_core raise AttributeError("module -> get_module_core: Unable to get module core") def get_module_init(self): - if self.mod_mem_type: # kernels 6.4+ + if self.has_member("mem"): # kernels 6.4+ try: return self.mem[self.mod_mem_type["MOD_INIT_TEXT"]].base except KeyError: raise AttributeError( "module -> get_module_core: Unable to get module init. Cannot read base from MOD_INIT_TEXT." ) - else: - if self.has_member("init_layout"): - return self.init_layout.base - elif self.has_member("module_init"): - return self.module_init + elif self.has_member("init_layout"): + return self.init_layout.base + elif self.has_member("module_init"): + return self.module_init raise AttributeError("module -> get_module_init: Unable to get module init") def get_name(self): From 77675d584d2e07df733c0700c8a3b7dd974252c0 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 21 Feb 2024 14:26:48 +0000 Subject: [PATCH 094/130] Update patch version --- volatility3/framework/constants/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 9aaafb933..7764dc44c 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 6 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature From a82b93523de768b68fed529064113340920e52fe Mon Sep 17 00:00:00 2001 From: Eve Date: Thu, 22 Feb 2024 09:47:23 +0000 Subject: [PATCH 095/130] Linux: psscan tiny update so that memory_layer_name is reused --- volatility3/framework/plugins/linux/psscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 462577e58..40784a647 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -28,7 +28,7 @@ class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -139,7 +139,7 @@ class PsScan(interfaces.plugins.PluginInterface): kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" ) memory_layer_name = kernel_layer.dependencies[0] - memory_layer = context.layers[kernel_layer.dependencies[0]] + memory_layer = context.layers[memory_layer_name] # scan the memory_layer for these needles for address, _ in memory_layer.scan( From 0899ba8d7fedece67b520c8a2a31c45868177ce1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 28 Feb 2024 23:09:46 +0000 Subject: [PATCH 096/130] Documentation: Fix up syntax errors involving * character --- volatility3/framework/interfaces/plugins.py | 2 +- volatility3/framework/plugins/linux/kmsg.py | 4 +++- volatility3/framework/plugins/linux/pslist.py | 6 +++--- volatility3/framework/plugins/linux/sockstat.py | 4 ++-- volatility3/framework/plugins/windows/mftscan.py | 4 ++-- .../framework/symbols/linux/extensions/__init__.py | 8 ++++---- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 29395aadf..697e4cdc3 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -60,7 +60,7 @@ class FileHandlerInterface(io.RawIOBase): @staticmethod def sanitize_filename(filename: str) -> str: """Sanititizes the filename to ensure only a specific whitelist of characters is allowed through""" - allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]\{\}!$%^:#~?<>,|" + allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]{}!$%^:#~?<>,|" result = "" for char in filename: if char in allowed: diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index dd707a7ff..c5e0fc302 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -198,7 +198,9 @@ class ABCKmsg(ABC): class Kmsg_pre_3_5(ABCKmsg): """The kernel ring buffer (log_buf) is a char array that sequentially stores log lines, each separated by newline (LF) characters. i.e: - <6>[ 9565.250411] line1!\n<6>[ 9565.250412] line2\n... + + <6>[ 9565.250411] line1!\\n<6>[ 9565.250412] line2\\n... + """ @classmethod diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 771040dc0..046ee43d8 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -83,11 +83,11 @@ class PsList(interfaces.plugins.PluginInterface): cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False ) -> Tuple[int, int, int, str]: """Extract the fields needed for the final output + Args: task: A task object from where to get the fields. - decorate_comm: If True, it decorates the comm string of - - User threads: in curly brackets, - - Kernel threads: in square brackets + decorate_comm: If True, it decorates the comm string of user threads in curly brackets, + and of Kernel threads in square brackets. Defaults to False. Returns: A tuple with the fields to show in the plugin output. diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index e9c98a227..78217fbec 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -83,7 +83,7 @@ class SockHandlers(interfaces.configuration.VersionableInterface): sock: Kernel generic `sock` object Returns a tuple with: - sock: The respective kernel's \*_sock object for that socket family + sock: The respective kernel's \\*_sock object for that socket family sock_stat: A tuple with the source and destination (address and port) along with its state string socket_filter: A dictionary with information about the socket filter """ @@ -501,7 +501,7 @@ class Sockstat(plugins.PluginInterface): family: Socket family string (AF_UNIX, AF_INET, etc) sock_type: Socket type string (STREAM, DGRAM, etc) protocol: Protocol string (UDP, TCP, etc) - sock_fields: A tuple with the \*_sock object, the sock stats and the extended info dictionary + sock_fields: A tuple with the \\*_sock object, the sock stats and the extended info dictionary """ vmlinux = context.modules[symbol_table] diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 91a2e9152..85c072037 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -38,7 +38,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\*|BAAD/"} + {"yara_rules": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File @@ -197,7 +197,7 @@ class ADS(interfaces.plugins.PluginInterface): # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( - {"yara_rules": "/FILE0|FILE\*|BAAD/"} + {"yara_rules": "/FILE0|FILE\\*|BAAD/"} ) # Read in the Symbol File diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 751c80cb5..588ce3874 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1137,17 +1137,17 @@ class vfsmount(objects.StructType): """Helper to make sure it is comparing two pointers to 'vfsmount'. Depending on the kernel version, the calling object (self) could be - a 'vfsmount \*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust + a 'vfsmount \\*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust in the framework "auto" dereferencing ability to assure that when we reach this point 'self' will be a 'vfsmount' already and self.vol.offset - a 'vfsmount \*' and not a 'vfsmount \*\*'. The argument must be a 'vfsmount \*'. + a 'vfsmount \\*' and not a 'vfsmount \\*\\*'. The argument must be a 'vfsmount \\*'. Typically, it's called from do_get_path(). Args: - vfsmount_ptr (vfsmount \*): A pointer to a 'vfsmount' + vfsmount_ptr (vfsmount *): A pointer to a 'vfsmount' Raises: - exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \*' + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \\*' Returns: bool: 'True' if the given argument points to the the same 'vfsmount' From 338106dfe8e0667a07b0ab0ba4d52fbf5f4d51e7 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 14:46:35 +1100 Subject: [PATCH 097/130] Move the memory page parameters to the Intel layer --- volatility3/framework/constants/linux/__init__.py | 5 ----- volatility3/framework/layers/intel.py | 12 ++++++++++++ volatility3/framework/plugins/linux/elfs.py | 14 +++++--------- .../framework/symbols/linux/extensions/__init__.py | 2 +- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 5e82e580e..3eabc2341 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -9,11 +9,6 @@ from enum import IntEnum KERNEL_NAME = "__kernel__" -# arch/x86/include/asm/page_types.h -PAGE_SHIFT = 12 -PAGE_SIZE = 1 << PAGE_SHIFT -PAGE_MASK = ~(PAGE_SIZE - 1) - """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" # include/linux/sched.h diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index ae477854d..75e561b33 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -67,6 +67,12 @@ class Intel(linear.LinearlyMappedLayer): math.ceil(math.log2(struct.calcsize(self._entry_format))) ) + @classproperty + @functools.lru_cache() + def page_shift(cls) -> int: + """Page shift for the intel memory layers.""" + return cls._page_size_in_bits + @classproperty @functools.lru_cache() def page_size(cls) -> int: @@ -76,6 +82,12 @@ class Intel(linear.LinearlyMappedLayer): """ return 1 << cls._page_size_in_bits + @classproperty + @functools.lru_cache() + def page_mask(cls) -> int: + """Page mask for the intel memory layers.""" + return ~(cls.page_size - 1) + @classproperty @functools.lru_cache() def bits_per_register(cls) -> int: diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 7171a6616..43cd6bb8b 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -14,11 +14,7 @@ from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux.extensions import elf -from volatility3.framework.constants.linux import ( - PAGE_SIZE, - PAGE_MASK, - ELF_MAX_EXTRACTION_SIZE, -) +from volatility3.framework.constants.linux import ELF_MAX_EXTRACTION_SIZE from volatility3.plugins.linux import pslist @@ -106,11 +102,11 @@ class Elfs(plugins.PluginInterface): # Use complete memory pages for dumping # If start isn't a multiple of a page, stick to the highest multiple < start # If end isn't a multiple of a page, stick to the lowest multiple > end - if start % PAGE_SIZE: - start = start & PAGE_MASK + if start % proc_layer.page_size: + start = start & proc_layer.page_mask - if end % PAGE_SIZE: - end = (end & PAGE_MASK) + PAGE_SIZE + if end % proc_layer.page_size: + end = (end & proc_layer.page_mask) + proc_layer.page_size real_size = end - start diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0a3db9fcd..1faafc267 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -640,7 +640,7 @@ class vm_area_struct(objects.StructType): elif flags_str == "r-x" and self.vm_file.dereference().vol.offset == 0: ret = True elif proclayer and "x" in flags_str: - for i in range(self.vm_start, self.vm_end, 1 << constants.linux.PAGE_SHIFT): + for i in range(self.vm_start, self.vm_end, proclayer.page_size): try: if proclayer.is_dirty(i): vollog.warning( From d5a0543a2d2d950c6743455f3f0ffa0b5afc2097 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 15:34:20 +1100 Subject: [PATCH 098/130] Use the ELF class constant instead of hardcoding a value. Fixed some f-strings --- volatility3/framework/layers/xen.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index 927b30430..e7aa0ccec 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -5,6 +5,7 @@ from typing import Optional from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.layers import elf from volatility3.framework.symbols import intermed +from volatility3.framework.constants.linux import ELF_CLASS vollog = logging.getLogger(__name__) @@ -14,7 +15,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): _header_struct = struct.Struct(" Date: Thu, 29 Feb 2024 15:34:48 +1100 Subject: [PATCH 099/130] Update author and modification time --- volatility3/framework/symbols/linux/elf.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/elf.json b/volatility3/framework/symbols/linux/elf.json index e0a95bbba..79a96e07a 100644 --- a/volatility3/framework/symbols/linux/elf.json +++ b/volatility3/framework/symbols/linux/elf.json @@ -1087,8 +1087,8 @@ "metadata": { "producer": { "version": "0.0.2", - "name": "ikelos-by-hand", - "datetime": "2019-10-21T22:52:00" + "name": "gcmoreira-by-hand", + "datetime": "2024-02-19T14:37:00" }, "format": "6.1.0" } From 9b0915dc85470df78bc739148093bb72d659297c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 15:36:46 +1100 Subject: [PATCH 100/130] Add logging for unknown ELF types --- volatility3/framework/layers/elf.py | 4 ++ volatility3/framework/plugins/linux/elfs.py | 4 ++ .../framework/symbols/linux/extensions/elf.py | 66 ++++++++++++++++--- 3 files changed, 64 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index 6bd5c2d63..81f3c3634 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -55,6 +55,10 @@ class Elf64Layer(segmented.SegmentedLayer): try: ptype = phdr.p_type.description except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue if ( diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 43cd6bb8b..6820576dc 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -93,6 +93,10 @@ class Elfs(plugins.PluginInterface): if phdr.p_type.description != "PT_LOAD": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue start = phdr.p_vaddr diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 828370fe8..2cf5c3d4e 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -165,6 +165,10 @@ class elf(objects.StructType): if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue for dsec in phdr.dynamic_sections(): @@ -172,6 +176,10 @@ class elf(objects.StructType): if dsec.d_tag.description != "DT_PLTGOT": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF dynamic section type: {dsec.d_tag}", + ) continue got_start = dsec.d_ptr @@ -186,16 +194,27 @@ class elf(objects.StructType): layer_name=self.vol.layer_name, ) if not link_map_ptr: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid ELF link map pointer at 0x{link_map_addr:x}", + ) continue linkmap_symname = ( elf_symbol_table + constants.BANG + self._type_prefix + "LinkMap" ) - link_map = self._context.object( - object_type=linkmap_symname, - offset=link_map_ptr, - layer_name=self.vol.layer_name, - ) + try: + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map_ptr, + layer_name=self.vol.layer_name, + ) + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid ELF link map address at 0x{link_map_ptr:x}", + ) + continue while link_map and link_map.vol.offset != 0: if link_map.vol.offset in link_maps_seen: @@ -204,11 +223,18 @@ class elf(objects.StructType): yield link_map - link_map = self._context.object( - object_type=linkmap_symname, - offset=link_map.l_next, - layer_name=self.vol.layer_name, - ) + try: + link_map = self._context.object( + object_type=linkmap_symname, + offset=link_map.l_next, + layer_name=self.vol.layer_name, + ) + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + f"ELF link map linked list is corrupt at 0x{self.vol.offset:x}", + ) + break def _find_symbols(self): dt_strtab = None @@ -221,6 +247,10 @@ class elf(objects.StructType): if phdr.p_type.description != "PT_DYNAMIC": continue except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {phdr.p_type}", + ) continue # This section contains pointers to the strtab, symtab, and strent sections @@ -228,6 +258,10 @@ class elf(objects.StructType): try: dtag = dsec.d_tag.description except ValueError: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF dynamic section type: {dsec.d_tag}", + ) continue if dtag == "DT_STRTAB": @@ -353,6 +387,10 @@ class elf_phdr(objects.StructType): except ValueError: # Unknown ELF object file type. Anyway, if the ELF object file type is not a # shared object (ET_DYN), the virtual address is 'p_vaddr'. + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF object type: {self._parent_e_type}", + ) pass return offset @@ -365,6 +403,10 @@ class elf_phdr(objects.StructType): except ValueError: # If the value is outside the ones declared in the enumeration, an # exception is raised + vollog.log( + constants.LOGLEVEL_VVVV, + f"Skipping unknown ELF program header type: {self.p_type}", + ) return None # the buffer of array starts at elf_base + our virtual address ( offset ) @@ -398,6 +440,10 @@ class elf_linkmap(objects.StructType): buf = self._context.layers.read(self.vol.layer_name, self.l_name, 256) except exceptions.PagedInvalidAddressException: # Protection against memory smear + vollog.log( + constants.LOGLEVEL_VVVV, + f"Invalid l_name address for ELF link map at 0x{self.vol.offset:x}", + ) return None idx = buf.find(b"\x00") From 3c9af096e15402a403b41b29a79ab3c74cc617f1 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 16:17:47 +1100 Subject: [PATCH 101/130] Add missing PAGE_SHIFT replacement --- volatility3/framework/symbols/linux/extensions/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 1faafc267..04cfa099f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -611,7 +611,8 @@ class vm_area_struct(objects.StructType): def get_page_offset(self) -> int: if self.vm_file == 0: return 0 - return self.vm_pgoff << constants.linux.PAGE_SHIFT + parent_layer = self._context.layers[self.vol.layer_name] + return self.vm_pgoff << parent_layer.page_shift def get_name(self, context, task): if self.vm_file != 0: From b681438ddcfdc33defa34d15cf48bd9ce4c9ca58 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 29 Feb 2024 16:21:34 +1100 Subject: [PATCH 102/130] Remove unnecessary 'pass' statement. --- volatility3/framework/symbols/linux/extensions/elf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 2cf5c3d4e..eadcbbae0 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -391,7 +391,6 @@ class elf_phdr(objects.StructType): constants.LOGLEVEL_VVVV, f"Skipping unknown ELF object type: {self._parent_e_type}", ) - pass return offset From a8e10828273fb8cfa74743137ba70fd730e42619 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 29 Feb 2024 14:36:46 +0100 Subject: [PATCH 103/130] restore pslist, add queue_head_t type class --- volatility3/framework/plugins/mac/pslist.py | 8 ++------ volatility3/framework/symbols/mac/__init__.py | 4 +++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 9835644b8..9b570f3f9 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -49,9 +49,7 @@ class PsList(interfaces.plugins.PluginInterface): ] @classmethod - def get_list_tasks( - cls, method: str - ) -> Callable[ + def get_list_tasks(cls, method: str) -> Callable[ [interfaces.context.ContextInterface, str, Callable[[int], bool]], Iterable[interfaces.objects.ObjectInterface], ]: @@ -188,9 +186,7 @@ class PsList(interfaces.plugins.PluginInterface): kernel_layer = context.layers[kernel.layer_name] - queue_entry = kernel.object( - object_type="queue_entry", offset=kernel.get_symbol("tasks").address - ) + queue_entry = kernel.object_from_symbol(symbol_name="tasks") seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index 56ac96633..be4fe8cd1 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -21,12 +21,14 @@ class MacKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("vm_map_object", extensions.vm_map_object) self.set_type_class("socket", extensions.socket) self.set_type_class("inpcb", extensions.inpcb) - self.set_type_class("queue_entry", extensions.queue_entry) self.set_type_class("ifnet", extensions.ifnet) self.set_type_class("sockaddr_dl", extensions.sockaddr_dl) self.set_type_class("sockaddr", extensions.sockaddr) self.set_type_class("sysctl_oid", extensions.sysctl_oid) self.set_type_class("kauth_scope", extensions.kauth_scope) + # https://developer.apple.com/documentation/kernel/queue_head_t + self.set_type_class("queue_entry", extensions.queue_entry) + self.set_type_class("queue_head_t", extensions.queue_entry) class MacUtilities(interfaces.configuration.VersionableInterface): From d56ccbfd0b749d8f2ad69764658bf8f3856b8d21 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 14:18:30 +0100 Subject: [PATCH 104/130] allow specifying integers as 0x in ListRequirement --- volatility3/cli/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 3de7d8f8f..a67762a31 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,7 +827,11 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - additional["type"] = requirement.element_type + # Allow a list of integers, specified with convenient 0x hexadecimal format + if requirement.element_type == int: + additional["type"] = lambda x: int(x, 0) + else: + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 3040bc5fc21f31e27f75ac8c3ec359e43454d098 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 14:20:13 +0100 Subject: [PATCH 105/130] add --dump mechanism, similarly to linux.proc --- .../framework/plugins/mac/proc_maps.py | 173 +++++++++++++++++- 1 file changed, 169 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 781b3ed66..204d414e4 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -2,17 +2,23 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.plugins.mac import pslist +from typing import Callable, Generator, Type, Optional +import logging + +vollog = logging.getLogger(__name__) class Maps(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" _required_framework_version = (2, 0, 0) + _version = (1, 1, 0) + MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @classmethod def get_requirements(cls): @@ -31,14 +37,155 @@ class Maps(interfaces.plugins.PluginInterface): element_type=int, optional=True, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed memory segments", + default=False, + optional=True, + ), + requirements.ListRequirement( + name="address", + description="Process virtual memory addresses to include " + "(all other VMA sections are excluded). This can be any " + "virtual address within the VMA section. Virtual addresses " + "must be separated by a space.", + element_type=int, + optional=True, + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size for dumped VMA sections " + "(all the bigger sections will be ignored)", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), ] + @classmethod + def list_vmas( + cls, + task: interfaces.objects.ObjectInterface, + filter_func: Callable[ + [interfaces.objects.ObjectInterface], bool + ] = lambda _: True, + ) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Lists the Virtual Memory Areas of a specific process. + + Args: + task: task object from which to list the vma + filter_func: Function to take a vma and return False if it should be filtered out + + Returns: + Yields vmas based on the task and filtered based on the filter function + """ + for vma in task.get_map_iter(): + if filter_func(vma): + yield vma + else: + vollog.debug( + f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.p_pid} due to filter_func" + ) + + @classmethod + def vma_dump( + cls, + context: interfaces.context.ContextInterface, + task: interfaces.objects.ObjectInterface, + vm_start: int, + vm_end: int, + open_method: Type[interfaces.plugins.FileHandlerInterface], + maxsize: int = MAXSIZE_DEFAULT, + ) -> Optional[interfaces.plugins.FileHandlerInterface]: + """Extracts the complete data for VMA as a FileInterface. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + task: an task_struct instance + vm_start: The start virtual address from the vma to dump + vm_end: The end virtual address from the vma to dump + open_method: class to provide context manager for opening the file + maxsize: Max size of VMA section (default MAXSIZE_DEFAULT) + + Returns: + An open FileInterface object containing the complete data for the task or None in the case of failure + """ + pid = task.p_pid + + try: + proc_layer_name = task.add_process_layer() + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + pid, excp.invalid_address, excp.layer_name + ) + ) + return None + vm_size = vm_end - vm_start + + # check if vm_size is negative, this should never happen. + if vm_size < 0: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is negative." + ) + return None + # check if vm_size is larger than the maxsize limit, and therefore is not saved out. + if maxsize <= vm_size: + vollog.warning( + f"Skip virtual memory dump for pid {pid} between {vm_start:#x}-{vm_end:#x} as {vm_size} is larger than maxsize limit of {maxsize}" + ) + return None + proc_layer = context.layers[proc_layer_name] + file_name = f"pid.{pid}.vma.{vm_start:#x}-{vm_end:#x}.dmp" + try: + file_handle = open_method(file_name) + chunk_size = 1024 * 1024 * 10 + offset = vm_start + while offset < vm_start + vm_size: + to_read = min(chunk_size, vm_start + vm_size - offset) + data = proc_layer.read(offset, to_read, pad=True) + file_handle.write(data) + offset += to_read + except Exception as excp: + vollog.debug(f"Unable to dump virtual memory {file_name}: {excp}") + return None + return file_handle + def _generator(self, tasks): + address_list = self.config.get("address", None) + if not address_list: + # do not filter as no address_list was supplied + vma_filter_func = lambda _: True + else: + # filter for any vm_start that matches the supplied address config + def vma_filter_function(task: interfaces.objects.ObjectInterface) -> bool: + addrs_in_vma = [ + addr + for addr in address_list + if task.links.start <= addr <= task.links.end + ] + + # if any of the user supplied addresses would fall within this vma return true + if addrs_in_vma: + return True + else: + return False + + vma_filter_func = vma_filter_function + for task in tasks: process_name = utility.array_to_string(task.p_comm) process_pid = task.p_pid - for vma in task.get_map_iter(): + for vma in self.list_vmas(task, filter_func=vma_filter_func): + try: + vm_start = vma.links.start + vm_end = vma.links.end + except AttributeError: + vollog.debug( + f"Unable to find the vm_start and vm_end for vma at {vma.vol.offset:#x} for pid {process_pid}" + ) + continue + path = vma.get_path( self.context, self.context.modules[self.config["kernel"]].symbol_table_name, @@ -46,15 +193,32 @@ class Maps(interfaces.plugins.PluginInterface): if path == "": path = vma.get_special_path() + file_output = "Disabled" + if self.config["dump"]: + file_output = "Error outputting file" + file_handle = self.vma_dump( + self.context, + task, + vm_start, + vm_end, + self.open, + self.config["maxsize"], + ) + + if file_handle: + file_handle.close() + file_output = file_handle.preferred_filename + yield ( 0, ( process_pid, process_name, - format_hints.Hex(vma.links.start), - format_hints.Hex(vma.links.end), + format_hints.Hex(vm_start), + format_hints.Hex(vm_end), vma.get_perms(), path, + file_output, ), ) @@ -72,6 +236,7 @@ class Maps(interfaces.plugins.PluginInterface): ("End", format_hints.Hex), ("Protection", str), ("Map Name", str), + ("File output", str), ], self._generator( list_tasks(self.context, self.config["kernel"], filter_func=filter_func) From 965ddcb674d30528792a404b819635c60c207f8e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 19:30:46 +0100 Subject: [PATCH 106/130] make queue_head_t optional, as it might not exist in all ISF --- volatility3/framework/symbols/mac/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index be4fe8cd1..83aebb13b 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -28,7 +28,7 @@ class MacKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.set_type_class("kauth_scope", extensions.kauth_scope) # https://developer.apple.com/documentation/kernel/queue_head_t self.set_type_class("queue_entry", extensions.queue_entry) - self.set_type_class("queue_head_t", extensions.queue_entry) + self.optional_set_type_class("queue_head_t", extensions.queue_entry) class MacUtilities(interfaces.configuration.VersionableInterface): From 2e835099b58b54af2f988355ae04cbfeff9cd13e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Mar 2024 20:19:31 +0100 Subject: [PATCH 107/130] fix wrong list_head comparison + better naming --- .../symbols/mac/extensions/__init__.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index c89b527e6..0a6bfbb90 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -490,22 +490,24 @@ class queue_entry(objects.StructType): for attr in ["next", "prev"]: with contextlib.suppress(exceptions.InvalidAddressException): - n = getattr(self, attr).dereference().cast(type_name) - - while n is not None and n.vol.offset != list_head: - if n.vol.offset in seen: + queue_element = getattr(self, attr).dereference().cast(type_name) + while ( + queue_element is not None + and queue_element.vol.offset != list_head.vol.offset + ): + if queue_element.vol.offset in seen: break - yield n + yield queue_element - seen.add(n.vol.offset) + seen.add(queue_element.vol.offset) yielded = yielded + 1 if yielded == max_size: - return + return None - n = ( - getattr(n.member(attr=member_name), attr) + queue_element = ( + getattr(queue_element.member(attr=member_name), attr) .dereference() .cast(type_name) ) From caf108b604be6925a2e5e1a4afc627da1fa943a5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 17:17:48 +0100 Subject: [PATCH 108/130] macOS dmesg plugin support --- volatility3/framework/plugins/mac/dmesg.py | 79 ++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 volatility3/framework/plugins/mac/dmesg.py diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py new file mode 100644 index 000000000..a006ff854 --- /dev/null +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -0,0 +1,79 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.objects import utility + +vollog = logging.getLogger(__name__) + + +class Dmesg(interfaces.plugins.PluginInterface): + """Prints the kernel log buffer.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Kernel module for the OS", + architectures=["Intel32", "Intel64"], + ), + ] + + @classmethod + def get_kernel_log_buffer( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str + ): + """ + Online documentation : + - https://github.com/apple-open-source/macos/blob/master/xnu/bsd/sys/msgbuf.h + - https://github.com/apple-open-source/macos/blob/ea4cd5a06831aca49e33df829d2976d6de5316ec/xnu/bsd/kern/subr_log.c#L751 + Volatility 2 plugin : + - https://github.com/volatilityfoundation/volatility/blob/master/volatility/plugins/mac/dmesg.py + """ + + kernel = context.modules[kernel_module_name] + if not kernel.has_symbol("msgbufp"): + vollog.error( + 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' + ) + return [] + + msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") + msgbufp = msgbufp_ptr.dereference() + msg_size = msgbufp.msg_size # max buffer size + msg_bufx = msgbufp.msg_bufx # write pointer + msg_bufc = msgbufp.msg_bufc + # msg_bufc is circular, meaning that if its size exceeds msg_size, + # msg_bufx will point to the beginning of the buffer and start overwriting. + msg_bufc_data: str = utility.pointer_to_string(msg_bufc, msg_size) + # Avoid OOB reads + msg_bufx = msg_bufx if msg_bufx <= msg_size else 0 + # We directly take into account the case where the write buffer did a loop, + # as older messages will start at msg_bufx offset (not overwritten yet). + dmesg = msg_bufc_data[msg_bufx:] + dmesg += msg_bufc_data[:msg_bufx] + + # Yield each line + for dmesg_line in dmesg.splitlines(): + yield (dmesg_line.strip(),) + + def _generator(self): + for value in self.get_kernel_log_buffer( + context=self.context, kernel_module_name=self.config["kernel"] + ): + yield (0, value) + + def run(self): + return renderers.TreeGrid( + [ + ("line", str), + ], + self._generator(), + ) From 8d79e3211ba2e8ea751c657b5da9f99a8b3a90c2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 17:30:51 +0100 Subject: [PATCH 109/130] prefer TypeError to vollog.error --- volatility3/framework/plugins/mac/dmesg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index a006ff854..b837b9db6 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -40,10 +40,9 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): - vollog.error( + raise TypeError( 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) - return [] msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") msgbufp = msgbufp_ptr.dereference() From 2f0bb6b297ffe601c563d646f09c80b8b0a4864f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 2 Mar 2024 18:39:48 +0100 Subject: [PATCH 110/130] do not strip each line --- volatility3/framework/plugins/mac/dmesg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index b837b9db6..d4f2c869d 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -61,7 +61,7 @@ class Dmesg(interfaces.plugins.PluginInterface): # Yield each line for dmesg_line in dmesg.splitlines(): - yield (dmesg_line.strip(),) + yield (dmesg_line,) def _generator(self): for value in self.get_kernel_log_buffer( From cb6f8c507268dd399f24e762766b3ba42498799a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 4 Mar 2024 20:47:49 +1100 Subject: [PATCH 111/130] Rename methods to be private --- .../framework/plugins/linux/library_list.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py index ed5545347..062ed078e 100644 --- a/volatility3/framework/plugins/linux/library_list.py +++ b/volatility3/framework/plugins/linux/library_list.py @@ -43,7 +43,7 @@ class LibraryList(interfaces.plugins.PluginInterface): ), ] - def get_libdl_libraries( + def _get_libdl_libraries( self, proc_layer_name: str, vma_start: int ) -> interfaces.objects.ObjectInterface: """Get the ELF link map objects for the given VMA address @@ -81,7 +81,7 @@ class LibraryList(interfaces.plugins.PluginInterface): # Protection against memory smear in this VMA pass - def get_libdl_maps( + def _get_libdl_maps( self, task: interfaces.objects.ObjectInterface, proc_layer_name: str ) -> interfaces.objects.ObjectInterface: """Get the ELF link maps objects for a task @@ -96,14 +96,14 @@ class LibraryList(interfaces.plugins.PluginInterface): link_map_seen = set() for vma in task.mm.get_vma_iter(): - for link_map in self.get_libdl_libraries(proc_layer_name, vma.vm_start): + for link_map in self._get_libdl_libraries(proc_layer_name, vma.vm_start): if link_map.l_addr in link_map_seen: continue yield link_map link_map_seen.add(link_map.l_addr) - def get_task_libraries( + def _get_task_libraries( self, task: interfaces.objects.ObjectInterface ) -> Tuple[int, str]: """Get the task libraries from the ELF headers found within the memory maps @@ -118,13 +118,13 @@ class LibraryList(interfaces.plugins.PluginInterface): if not proc_layer_name: return - for elf_link_map in self.get_libdl_maps(task, proc_layer_name): + for elf_link_map in self._get_libdl_maps(task, proc_layer_name): name = elf_link_map.get_name() if not name: continue yield elf_link_map.l_addr, name - def get_tasks_libraries( + def _get_tasks_libraries( self, tasks: Iterable[interfaces.objects.ObjectInterface], ) -> Iterable[Tuple[str, int, int, str]]: @@ -139,7 +139,7 @@ class LibraryList(interfaces.plugins.PluginInterface): """ for task in tasks: task_name = utility.array_to_string(task.comm) - for linkmap_addr, linkmap_name in self.get_task_libraries(task): + for linkmap_addr, linkmap_name in self._get_task_libraries(task): yield task_name, task.tgid, linkmap_addr, linkmap_name def _format_fields(self, fields): @@ -149,7 +149,7 @@ class LibraryList(interfaces.plugins.PluginInterface): def _generator( self, tasks: Iterable[interfaces.objects.ObjectInterface] ) -> Iterable[Tuple[int, Tuple]]: - for fields in self.get_tasks_libraries(tasks): + for fields in self._get_tasks_libraries(tasks): yield 0, self._format_fields(fields) def run(self): From 373bf9dfaa466fdaba9635a78a6449679af502bf Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 4 Mar 2024 18:33:03 +0000 Subject: [PATCH 112/130] Windows: Fix driverirp black issue --- volatility3/framework/plugins/windows/driverirp.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index f7eca0359..433c61ca2 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -115,9 +115,17 @@ class DriverIrp(interfaces.plugins.PluginInterface): ) if not module_found: - yield (0, (format_hints.Hex(driver.vol.offset), driver_name, MAJOR_FUNCTIONS[i], - format_hints.Hex(address), renderers.NotAvailableValue(), renderers.NotAvailableValue())) - + yield ( + 0, + ( + format_hints.Hex(driver.vol.offset), + driver_name, + MAJOR_FUNCTIONS[i], + format_hints.Hex(address), + renderers.NotAvailableValue(), + renderers.NotAvailableValue(), + ), + ) def run(self): return renderers.TreeGrid( From 403804431ae0ae5118b13cb0f894edfb9cc03d73 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:11:17 +0100 Subject: [PATCH 113/130] prefer SymbolError to TypeError --- volatility3/framework/plugins/mac/dmesg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index d4f2c869d..12c241641 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -3,7 +3,7 @@ # import logging -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility @@ -40,7 +40,7 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): - raise TypeError( + raise exceptions.SymbolError( 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) From 1bf18dbd8a9f68381dfbdb46c4d76ab92cbe0a68 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:11:50 +0100 Subject: [PATCH 114/130] implicit pointer dereference --- volatility3/framework/plugins/mac/dmesg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index 12c241641..daa6617ad 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -44,8 +44,7 @@ class Dmesg(interfaces.plugins.PluginInterface): 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' ) - msgbufp_ptr = kernel.object_from_symbol(symbol_name="msgbufp") - msgbufp = msgbufp_ptr.dereference() + msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") msg_size = msgbufp.msg_size # max buffer size msg_bufx = msgbufp.msg_bufx # write pointer msg_bufc = msgbufp.msg_bufc From 98705e110df4691dc974210f0818621db2f27bbf Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:12:12 +0100 Subject: [PATCH 115/130] more specific msg_bufx comment --- volatility3/framework/plugins/mac/dmesg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index daa6617ad..7f817e605 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -46,7 +46,7 @@ class Dmesg(interfaces.plugins.PluginInterface): msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") msg_size = msgbufp.msg_size # max buffer size - msg_bufx = msgbufp.msg_bufx # write pointer + msg_bufx = msgbufp.msg_bufx # write index of the msg_bufc circular buffer msg_bufc = msgbufp.msg_bufc # msg_bufc is circular, meaning that if its size exceeds msg_size, # msg_bufx will point to the beginning of the buffer and start overwriting. From 69a6c7d5b7cbfb99e8ec70d05dac95a6dea71713 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:16:55 +0100 Subject: [PATCH 116/130] revert int as hex format commit --- volatility3/cli/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index a67762a31..3de7d8f8f 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,11 +827,7 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - # Allow a list of integers, specified with convenient 0x hexadecimal format - if requirement.element_type == int: - additional["type"] = lambda x: int(x, 0) - else: - additional["type"] = requirement.element_type + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 9914f339dfc1cdd511fc3fad676b92b06520243e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:19:15 +0100 Subject: [PATCH 117/130] simplify addrs_in_vma check --- volatility3/framework/plugins/mac/proc_maps.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 204d414e4..fe5179dfa 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -165,10 +165,7 @@ class Maps(interfaces.plugins.PluginInterface): ] # if any of the user supplied addresses would fall within this vma return true - if addrs_in_vma: - return True - else: - return False + return bool(addrs_in_vma) vma_filter_func = vma_filter_function From f4b12c6406e2f1accc147069403e363e7c6a01dc Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:26:18 +0100 Subject: [PATCH 118/130] allow ints in the 0x format in ListRequirement --- volatility3/cli/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 3de7d8f8f..35e977eba 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -827,7 +827,11 @@ class CommandLine: requirement, volatility3.framework.configuration.requirements.ListRequirement, ): - additional["type"] = requirement.element_type + # Allow a list of integers, specified with the convenient 0x hexadecimal format + if requirement.element_type == int: + additional["type"] = lambda x: int(x, 0) + else: + additional["type"] = requirement.element_type nargs = "*" if requirement.optional else "+" additional["nargs"] = nargs elif isinstance( From 1f9a983f492ff7983030b8fa27b509106594f735 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 4 Mar 2024 20:38:01 +0100 Subject: [PATCH 119/130] correct use of SymbolError --- volatility3/framework/plugins/mac/dmesg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/mac/dmesg.py b/volatility3/framework/plugins/mac/dmesg.py index 7f817e605..f9f06a666 100644 --- a/volatility3/framework/plugins/mac/dmesg.py +++ b/volatility3/framework/plugins/mac/dmesg.py @@ -41,7 +41,9 @@ class Dmesg(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_module_name] if not kernel.has_symbol("msgbufp"): raise exceptions.SymbolError( - 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.' + "msgbufp", + kernel.symbol_table_name, + 'The provided symbol table does not include the "msgbufp" symbol. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.', ) msgbufp = kernel.object_from_symbol(symbol_name="msgbufp") From 9edf33b7212d46682b48dbb40b744f198f8741a8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 11 Mar 2024 21:20:29 +0000 Subject: [PATCH 120/130] Layers: Improve logging on crashdump layer --- volatility3/framework/layers/crash.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 8efd4f7c7..8fd0abbcc 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -261,11 +261,15 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): progress_callback: constants.ProgressCallback = None, ) -> Optional[interfaces.layers.DataLayerInterface]: for layer in [WindowsCrashDump32Layer, WindowsCrashDump64Layer]: - with contextlib.suppress(WindowsCrashDumpFormatException): + try: layer.check_header(context.layers[layer_name]) new_name = context.layers.free_layer_name(layer.__name__) context.config[ interfaces.configuration.path_join(new_name, "base_layer") ] = layer_name return layer(context, new_name, new_name) + except (WindowsCrashDump32Layer, WindowsCrashDump64Layer) as excp: + vollog.log( + constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" + ) return None From 084ea38f84b36e8db2594fc31f1c2d61b3b7c6ce Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 12 Mar 2024 00:13:02 +0000 Subject: [PATCH 121/130] Layers: Fix up typo in recent crashdump patch --- volatility3/framework/layers/crash.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 8fd0abbcc..5598fc2e4 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -268,8 +268,8 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): interfaces.configuration.path_join(new_name, "base_layer") ] = layer_name return layer(context, new_name, new_name) - except (WindowsCrashDump32Layer, WindowsCrashDump64Layer) as excp: + except WindowsCrashDumpFormatException as excp: vollog.log( constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" - ) + )\ return None From 8dbc64f4a8678455adbac80ac716dfa62b3aecb2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 12 Mar 2024 00:14:30 +0000 Subject: [PATCH 122/130] Layers: Fix up typo in recent crashdump patch - take 2 --- volatility3/framework/layers/crash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 5598fc2e4..042b18ddc 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -271,5 +271,5 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): except WindowsCrashDumpFormatException as excp: vollog.log( constants.LOGLEVEL_VVVV, f"Exception reading crashdump: {excp}" - )\ + ) return None From f6495d3d986e01fb96789a88aa89cc2c8e40cb71 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 16 Mar 2024 08:42:53 +0000 Subject: [PATCH 123/130] Documentation: Improve logging level docstrings --- volatility3/framework/constants/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 921f602fd..3c5e0eb2f 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -59,14 +59,18 @@ PACKAGE_VERSION = ( AUTOMAGIC_CONFIG_PATH = "automagic" """The root section within the context configuration for automagic values""" +LOGLEVEL_INFO = 20 +"""Logging level for information data, showed when use the requests any logging: -v""" +LOGLEVEL_DEBUG = 10 +"""Logging level for debugging data, showed when the user requests more logging detail: -vv""" LOGLEVEL_V = 9 -"""Logging level for a single -v""" +"""Logging level for the lowest "extra" level of logging: -vvv""" LOGLEVEL_VV = 8 -"""Logging level for -vv""" +"""Logging level for two levels of detail: -vvvv""" LOGLEVEL_VVV = 7 -"""Logging level for -vvv""" +"""Logging level for three levels of detail: -vvvvv""" LOGLEVEL_VVVV = 6 -"""Logging level for -vvvv""" +"""Logging level for four levels of detail: -vvvvvv""" CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") """Default path to store cached data""" From 436d355ef301d9cd73883b5129f8b9f744c2ecb7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 18:44:09 +0100 Subject: [PATCH 124/130] incrementally order extra log levels --- volatility3/cli/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 35e977eba..ec9918a65 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -264,6 +264,17 @@ class CommandLine: file_logger.setFormatter(file_formatter) rootlog.addHandler(file_logger) vollog.info("Logging started") + + for level, level_value in enumerate( + [ + constants.LOGLEVEL_V, + constants.LOGLEVEL_VV, + constants.LOGLEVEL_VVV, + constants.LOGLEVEL_VVVV, + ] + ): + logging.addLevelName(level_value, f"DETAIL {level+1}") + if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None From 33d653e7a66b8b96350a3568c6d156eadb0cf878 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 18:44:19 +0100 Subject: [PATCH 125/130] use logging explicit constants --- volatility3/cli/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index ec9918a65..116cde114 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -278,9 +278,9 @@ class CommandLine: if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None - console.setLevel(30 - (partial_args.verbosity * 10)) + console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: - console.setLevel(10 - (partial_args.verbosity - 2)) + console.setLevel(logging.DEBUG - (partial_args.verbosity - 2)) for level, msg in delayed_logs: vollog.log(level, msg) From 745491c846314fb37231c892e2cfab9eed4275f2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:02 +0100 Subject: [PATCH 126/130] put extra log level ordering in a function --- volatility3/cli/__init__.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 116cde114..457a49311 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -265,16 +265,7 @@ class CommandLine: rootlog.addHandler(file_logger) vollog.info("Logging started") - for level, level_value in enumerate( - [ - constants.LOGLEVEL_V, - constants.LOGLEVEL_VV, - constants.LOGLEVEL_VVV, - constants.LOGLEVEL_VVVV, - ] - ): - logging.addLevelName(level_value, f"DETAIL {level+1}") - + self.order_extra_verbose_levels() if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None @@ -706,6 +697,17 @@ class CommandLine: ) context.config[extended_path] = value + def order_extra_verbose_levels(self): + for level, level_value in enumerate( + [ + constants.LOGLEVEL_V, + constants.LOGLEVEL_VV, + constants.LOGLEVEL_VVV, + constants.LOGLEVEL_VVVV, + ] + ): + logging.addLevelName(level_value, f"DETAIL {level+1}") + def file_handler_class_factory(self, direct=True): output_dir = self.output_dir From 7ff5f57e5d4485ad2e5f2ad4dc78017880442c05 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:28 +0100 Subject: [PATCH 127/130] use logging explicit constants --- volatility3/cli/volshell/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 9e74acfec..998c8245b 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -198,9 +198,9 @@ class VolShell(cli.CommandLine): vollog.info("Logging started") if partial_args.verbosity < 3: - console.setLevel(30 - (partial_args.verbosity * 10)) + console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: - console.setLevel(10 - (partial_args.verbosity - 2)) + console.setLevel(logging.DEBUG - (partial_args.verbosity - 2)) for level, msg in delayed_logs: vollog.log(level, msg) From de4a3359982b67b74a354001d380502ff41804d1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 16 Mar 2024 22:19:45 +0100 Subject: [PATCH 128/130] call extra verbose level ordering --- volatility3/cli/volshell/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 998c8245b..035ed9b2e 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -197,6 +197,7 @@ class VolShell(cli.CommandLine): vollog.addHandler(file_logger) vollog.info("Logging started") + self.order_extra_verbose_levels() if partial_args.verbosity < 3: console.setLevel(logging.WARNING - (partial_args.verbosity * 10)) else: From 47891c3477fbd7bd008e8b3b8f119936289104de Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 3 Apr 2024 21:17:42 +0100 Subject: [PATCH 129/130] Core: Develop nested requirement files --- requirements-dev.txt | 21 ++++----------------- requirements.txt | 4 ++-- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c9b615cd8..ae3482290 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,22 +1,9 @@ -# The following packages are required for core functionality. -pefile>=2023.2.7 - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 - -# This is required for several plugins that perform malware analysis and disassemble code. -# It can also improve accuracy of Windows 8 and later memory samples. -capstone>=3.0.5 - -# This is required by plugins that decrypt passwords, password hashes, etc. -pycryptodome +-r requirements.txt # This can improve error messages regarding improperly configured ISF files, # but is only recommended for development jsonschema>=2.3.0 -# This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0 +# Used to build executable file +pyinstaller>=6.5.0 +pyinstaller-hooks-contrib>=2024.3 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4d09ff82a..c63dc0b36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -# The following packages are required for core functionality. -pefile>=2023.2.7 +# Include the minimal requirements +-r requirements-minimal.txt # The following packages are optional. # If certain packages are not necessary, place a comment (#) at the start of the line. From c1f239b8d2171e83e331ad51bf7ff6e48ed04e52 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 7 Apr 2024 19:34:17 +0100 Subject: [PATCH 130/130] Linux: Improve debugging of pslist dump feature --- volatility3/framework/plugins/linux/elfs.py | 1 + volatility3/framework/plugins/linux/pslist.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 6820576dc..22e39d127 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -84,6 +84,7 @@ class Elfs(plugins.PluginInterface): ) if not elf_object.is_valid(): + vollog.debug("ELF object to be dumped is not valid") return None sections = {} diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 046ee43d8..1888bd7b8 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -142,6 +142,8 @@ class PsList(interfaces.plugins.PluginInterface): file_output = str(file_handle.preferred_filename) file_handle.close() break + else: + file_output = "VMA start matching task start_code not found" return file_output def _generator(