From 48e29ae530ecfb65fa3311a7547c16ea77f7f5cd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 May 2022 23:44:29 +0100 Subject: [PATCH 01/21] CLI: Add support for a configuration options file --- volatility3/cli/__init__.py | 58 ++++++++++++++++++++++------ volatility3/cli/volshell/__init__.py | 51 +++++++++++++++++------- 2 files changed, 83 insertions(+), 26 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 8851e2b18..62be115d3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,7 +19,7 @@ import os import sys import tempfile import traceback -from typing import Any, Dict, Type, Union +from typing import Any, Dict, List, Tuple, Type, Union from urllib import parse, request import volatility3.plugins @@ -92,6 +92,9 @@ class CommandLine: renderers = dict([(x.name.lower(), x) for x in framework.class_subclasses(text_renderer.CLIRenderer)]) + # Load up system defaults + delayed_logs, default_config = self.load_system_defaults('vol.json') + parser = volargparse.HelpfulArgParser(add_help = False, prog = self.CLI_NAME, description = "An open-source memory forensics framework") @@ -174,6 +177,8 @@ class CommandLine: default = False, action = 'store_true') + parser.set_defaults(**default_config) + # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h'] @@ -184,17 +189,7 @@ class CommandLine: banner_output = sys.stderr banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") - if partial_args.plugin_dirs: - volatility3.plugins.__path__ = [os.path.abspath(p) - for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH - - if partial_args.symbol_dirs: - volatility3.symbols.__path__ = [os.path.abspath(p) - for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS - - if partial_args.cache_path: - constants.CACHE_PATH = partial_args.cache_path - + ### Start up logging if partial_args.log: file_logger = logging.FileHandler(partial_args.log) file_logger.setLevel(1) @@ -210,6 +205,21 @@ class CommandLine: else: console.setLevel(10 - (partial_args.verbosity - 2)) + for level, msg in delayed_logs: + vollog.log(level, msg) + + ### Alter constants if necessary + if partial_args.plugin_dirs: + volatility3.plugins.__path__ = [os.path.abspath(p) + for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH + + if partial_args.symbol_dirs: + volatility3.symbols.__path__ = [os.path.abspath(p) + for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS + + if partial_args.cache_path: + constants.CACHE_PATH = partial_args.cache_path + vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") @@ -366,6 +376,30 @@ class CommandLine: raise ValueError(f"File does not exist: {filename}") return parse.urlunparse(single_location) + def load_system_defaults(self, filename: str) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: + """Modify the main configuration based on the default configuration override""" + # Build the config path + default_config_path = os.path.join(os.path.expanduser("~"), ".config", "volatility3", filename) + if sys.platform == 'win32': + default_config_path = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3", + filename) + + delayed_logs = [] + + # Process it if the files exist + if os.path.exists(default_config_path): + result = json.load(open(default_config_path, 'rb')) + if not isinstance(result, dict): + delayed_logs.append((logging.INFO, + f'Default configuration file {default_config_path} does not contain a dictionary')) + else: + delayed_logs.append( + (logging.INFO, f"Loading default configuration options from {default_config_path}")) + delayed_logs.append((logging.DEBUG, + f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}")) + return delayed_logs, result + return delayed_logs, {} + def process_exceptions(self, excp): """Provide useful feedback if an exception occurs during a run of a plugin.""" # Ensure there's nothing in the cache diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 769e958fd..647be8a1a 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -45,6 +45,9 @@ class VolShell(cli.CommandLine): framework.require_interface_version(2, 0, 0) + # Load up system defaults + delayed_logs, default_config = self.load_system_defaults('volshell.json') + parser = argparse.ArgumentParser(prog = self.CLI_NAME, description = "A tool for interactivate forensic analysis of memory images") parser.add_argument("-c", @@ -68,13 +71,16 @@ class VolShell(cli.CommandLine): default = "", type = str) parser.add_argument("-v", "--verbosity", help = "Increase output verbosity", default = 0, action = "count") + parser.add_argument("--log", + help = "Log output to a file as well as the console", + default = None, + type = str) parser.add_argument("-o", "--output-dir", help = "Directory in which to output any generated files", default = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')), type = str) parser.add_argument("-q", "--quiet", help = "Remove progress feedback", default = False, action = 'store_true') - parser.add_argument("--log", help = "Log output to a file as well as the console", default = None, type = str) parser.add_argument("-f", "--file", metavar = 'FILE', @@ -97,6 +103,10 @@ class VolShell(cli.CommandLine): help = f"Change the default path ({constants.CACHE_PATH}) used to store the cache", default = constants.CACHE_PATH, type = str) + parser.add_argument("--offline", + help = "Do not search online for additional JSON files", + default = False, + action = 'store_true') # Volshell specific flags os_specific = parser.add_mutually_exclusive_group(required = False) @@ -108,10 +118,33 @@ class VolShell(cli.CommandLine): os_specific.add_argument("-l", "--linux", default = False, action = "store_true", help = "Run a Linux volshell") os_specific.add_argument("-m", "--mac", default = False, action = "store_true", help = "Run a Mac volshell") + parser.set_defaults(**default_config) + # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h'] partial_args, _ = parser.parse_known_args(known_args) + + ### Start up logging + if partial_args.log: + file_logger = logging.FileHandler(partial_args.log) + file_logger.setLevel(1) + file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S', + fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + file_logger.setFormatter(file_formatter) + rootlog.addHandler(file_logger) + vollog.info("Logging started") + if partial_args.verbosity < 3: + if partial_args.verbosity < 1: + sys.tracebacklimit = None + console.setLevel(30 - (partial_args.verbosity * 10)) + else: + console.setLevel(10 - (partial_args.verbosity - 2)) + + for level, msg in delayed_logs: + vollog.log(level, msg) + + ### Alter constants if necessary if partial_args.plugin_dirs: volatility3.plugins.__path__ = [os.path.abspath(p) for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH @@ -126,23 +159,13 @@ class VolShell(cli.CommandLine): vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") - if partial_args.log: - file_logger = logging.FileHandler(partial_args.log) - file_logger.setLevel(0) - file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S', - fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s') - file_logger.setFormatter(file_formatter) - vollog.addHandler(file_logger) - vollog.info("Logging started") - - if partial_args.verbosity < 3: - console.setLevel(30 - (partial_args.verbosity * 10)) - else: - console.setLevel(10 - (partial_args.verbosity - 2)) if partial_args.clear_cache: framework.clear_cache() + if partial_args.offline: + constants.OFFLINE = partial_args.offline + # Do the initialization ctx = contexts.Context() # Construct a blank context failures = framework.import_files(volatility3.plugins, From 9ea1a960cdece1c6897735adc58f592f73c44e86 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 30 May 2022 01:34:19 +0100 Subject: [PATCH 02/21] CLI: Synchonize volshell code a little better courtesy of @digitalisx --- volatility3/cli/volshell/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 647be8a1a..e2e7bc394 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -15,12 +15,14 @@ from volatility3.cli.volshell import generic, linux, mac, windows from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins # Make sure we log everything + +rootlog = logging.getLogger() vollog = logging.getLogger() vollog.setLevel(0) -# Trim the console down by default console = logging.StreamHandler() console.setLevel(logging.WARNING) formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') +# Trim the console down by default console.setFormatter(formatter) vollog.addHandler(console) From 2e4c257e15e83b33fb40cd9369c90dac2c1786e9 Mon Sep 17 00:00:00 2001 From: ikelos Date: Mon, 30 May 2022 01:53:07 +0100 Subject: [PATCH 03/21] Update volatility3/cli/__init__.py Co-authored-by: Donghyun Kim --- volatility3/cli/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 62be115d3..6881311bf 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -388,7 +388,8 @@ class CommandLine: # Process it if the files exist if os.path.exists(default_config_path): - result = json.load(open(default_config_path, 'rb')) + with open(default_config_path, 'rb') as config_json: + result = json.load(config_json) if not isinstance(result, dict): delayed_logs.append((logging.INFO, f'Default configuration file {default_config_path} does not contain a dictionary')) From 604c23bcbc88679a48d1348ef3c8d2163726ab44 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Thu, 14 Dec 2023 14:50:17 +0100 Subject: [PATCH 04/21] Import Address Table Plugin --- volatility3/framework/plugins/windows/iat.py | 132 +++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 volatility3/framework/plugins/windows/iat.py diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py new file mode 100644 index 000000000..1cf51cfa2 --- /dev/null +++ b/volatility3/framework/plugins/windows/iat.py @@ -0,0 +1,132 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + +import logging +import io +from typing import Callable, List +from volatility3.framework.symbols import intermed +from volatility3.framework import renderers, interfaces, exceptions, constants +from volatility3.framework.configuration import requirements +from volatility3.plugins.windows import pslist +from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.symbols.windows.extensions import pe +import pefile + +vollog = logging.getLogger(__name__) + + +class IAT(interfaces.plugins.PluginInterface): + """Extract Import Address Table to list API (functions) used by a program contained in external libraries""" + + _required_framework_version = (2, 4, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process ID to include (all other processes are excluded)", + optional=True, + ), + ] + + def _generator(self, procs): + kernel = self.context.modules[self.config["kernel"]] + + for proc in procs: + try: + proc_id = proc.UniqueProcessId + proc_layer_name = proc.add_process_layer() + peb = self.context.object( + kernel.symbol_table_name + constants.BANG + "_PEB", + layer_name=proc_layer_name, + offset=proc.Peb, + ) + + if proc_layer_name is None: + raise TypeError("Layer must be a string not None") + + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "windows", + "pe", + class_types=pe.class_types, + ) + pe_data = io.BytesIO() + + dos_header = self.context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=peb.ImageBaseAddress, + layer_name=proc_layer_name, + ) + + for offset, data in dos_header.reconstruct(): + pe_data.seek(offset) + pe_data.write(data) + + pe_obj = pefile.PE(data=pe_data.getvalue(), fast_load=True) + pe_obj.parse_data_directories( + [pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_IMPORT"]] + ) + if hasattr(pe_obj, "DIRECTORY_ENTRY_IMPORT"): + for entry in pe_obj.DIRECTORY_ENTRY_IMPORT: + dll_entry = entry.dll + if dll_entry: + dll_entry = dll_entry.decode() + else: + dll_entry = renderers.NotAvailableValue + + # Iterate over imported functions + for imp in entry.imports: + import_name = imp.name + if import_name: + import_name = imp.name.decode() + else: + import_name = renderers.NotAvailableValue() + yield ( + 0, + ( + proc_id, + proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ), + dll_entry, + import_name, + ), + ) + except exceptions.InvalidAddressException as excp: + vollog.debug( + "Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name + ) + ) + continue + + def run(self): + kernel = self.context.modules[self.config["kernel"]] + + return renderers.TreeGrid( + [("PID", int), ("Process", str), ("Library", str), ("Function", str)], + self._generator( + pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + filter_func=pslist.PsList.create_pid_filter( + self.config.get("pid", None) + ), + ) + ), + ) From 059f2d012896d96fa0e2d20469d7771ec27b0b6d Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 16 Dec 2023 21:19:35 +0100 Subject: [PATCH 05/21] Adding function addr + bound info. Formatting code. --- volatility3/framework/plugins/windows/iat.py | 32 +++++++++++++++----- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 1cf51cfa2..11c273859 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,16 +1,13 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -import logging -import io -from typing import Callable, List +import logging, io, pefile from volatility3.framework.symbols import intermed from volatility3.framework import renderers, interfaces, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.plugins.windows import pslist -from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.renderers import format_hints from volatility3.framework.symbols.windows.extensions import pe -import pefile vollog = logging.getLogger(__name__) @@ -86,6 +83,12 @@ class IAT(interfaces.plugins.PluginInterface): else: dll_entry = renderers.NotAvailableValue + bound = True + # Initially set to 0 if not bound + time_date_stamp = entry.struct.TimeDateStamp + if not time_date_stamp: + bound = False + # Iterate over imported functions for imp in entry.imports: import_name = imp.name @@ -93,6 +96,12 @@ class IAT(interfaces.plugins.PluginInterface): import_name = imp.name.decode() else: import_name = renderers.NotAvailableValue() + function_address = ( + pe_obj.OPTIONAL_HEADER.ImageBase + imp.address + ) + if not function_address: + function_address = renderers.NotAvailableValue + yield ( 0, ( @@ -103,7 +112,9 @@ class IAT(interfaces.plugins.PluginInterface): errors="replace", ), dll_entry, + bound, import_name, + format_hints.Hex(function_address), ), ) except exceptions.InvalidAddressException as excp: @@ -118,7 +129,14 @@ class IAT(interfaces.plugins.PluginInterface): kernel = self.context.modules[self.config["kernel"]] return renderers.TreeGrid( - [("PID", int), ("Process", str), ("Library", str), ("Function", str)], + [ + ("PID", int), + ("Name", str), + ("Library", str), + ("Bound", bool), + ("Function", str), + ("Address", format_hints.Hex), + ], self._generator( pslist.PsList.list_processes( context=self.context, From 62366d4f6f6b57e6e2f6920df67a88125dac9484 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 14 Jan 2024 22:37:56 +0000 Subject: [PATCH 06/21] Symbols: Add support for metadata verification of symbol tables --- volatility3/framework/constants/__init__.py | 4 +- volatility3/framework/plugins/linux/kmsg.py | 11 +- volatility3/framework/symbols/__init__.py | 49 +- volatility3/framework/symbols/intermed.py | 9 + volatility3/framework/symbols/metadata.py | 42 +- volatility3/schemas/schema-6.3.0.json | 503 ++++++++++++++++++++ 6 files changed, 613 insertions(+), 5 deletions(-) create mode 100644 volatility3/schemas/schema-6.3.0.json diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09dded076..9aaafb933 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_MINOR = 6 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d1f17bf94..d79248256 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -10,10 +10,12 @@ from typing import Generator, Iterator, List, Tuple from volatility3.framework import ( class_subclasses, constants, + exceptions, interfaces, renderers, ) from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility vollog = logging.getLogger(__name__) @@ -495,7 +497,7 @@ class Kmsg_5_10_to_(ABCKmsg): class Kmsg(interfaces.plugins.PluginInterface): """Kernel log buffer reader""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 6, 0) _version = (1, 0, 2) @@ -514,6 +516,13 @@ class Kmsg(interfaces.plugins.PluginInterface): yield (0, values) def run(self): + if not self.context.symbol_space.verify_table_versions( + "dwarf2json", lambda version, _: (not version) or version > (0, 4, 1) + ): + raise exceptions.SymbolSpaceError( + "Invalid symbol table, please ensure the ISF table produced by dwarf2json was produced using a version > 0.4.1" + ) + return renderers.TreeGrid( [ ("facility", str), diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index 10cf39cf1..49ff94f4b 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -4,9 +4,20 @@ import collections import collections.abc +import datetime import enum import logging -from typing import Any, Dict, Iterable, Iterator, TypeVar, List +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + Optional, + Tuple, + TypeVar, + List, +) from volatility3.framework import constants, exceptions, interfaces, objects @@ -113,6 +124,42 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self._resolved = {} del self._dict[key] + def verify_table_versions( + self, + producer: str, + validator: Callable[[Optional[Tuple], Optional[datetime.datetime]], bool], + tables: List[str] = None, + ) -> bool: + """Verifies the producer metadata and version of tables + + Args: + producer: String name of a table producer to have validation performed + validator: callable that takes an optional version and an optional datetime that returns False if table is invalid + + Returns: + False if an invalid table was found or True if no invalid table was found + """ + if tables is None: + tables = self._dict.keys() + for table_name in tables: + table = self._dict[table_name] + if not table.producer: + vollog.debug( + f"Symbol table {table_name} could not be validated because no producer metadata was found" + ) + continue + if table.producer.name == producer: + # Run the verification + if not validator( + table.producer.version, + table.producer.datetime, + ): + vollog.debug(f"Symbol table {table_name} does not pass validator") + return False + else: + continue + return True + ### Resolution functions class UnresolvedTemplate(objects.templates.ReferenceTemplate): diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 24e00bbd8..8c40e466a 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -183,6 +183,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): types = _construct_delegate_function("types", True) enumerations = _construct_delegate_function("enumerations", True) metadata = _construct_delegate_function("metadata", True) + producer = _construct_delegate_function("producer", True) clear_symbol_cache = _construct_delegate_function("clear_symbol_cache") get_type = _construct_delegate_function("get_type") get_symbol = _construct_delegate_function("get_symbol") @@ -372,6 +373,14 @@ class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass=ABCMeta): table.""" return None + @property + def producer(self) -> Optional["metadata.ProducerMetadata"]: + """Returns a metadata object containing information about the symbol + table.""" + return metadata.ProducerMetadata( + self._json_object.get("metadata", {}).get("producer", {}) + ) + def clear_symbol_cache(self) -> None: """Clears the symbol cache of the symbol table.""" self._symbol_cache.clear() diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 61947be69..f765e43c9 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -2,9 +2,49 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import datetime +import logging from typing import Optional, Tuple, Union -from volatility3.framework import interfaces +from volatility3.framework import constants, interfaces + +vollog = logging.getLogger(__name__) + + +class ProducerMetadata(interfaces.symbols.MetadataInterface): + """Class to handle the Producer metadata from an ISF""" + + @property + def name(self) -> Optional[str]: + return self._json_data.get("name", None) + + @property + def version(self) -> Optional[Tuple[int]]: + """Returns the version of the ISF file producer""" + version = self._json_data.get("version", None) + if not version: + return None + if all([x in "0123456789." for x in version]): + return tuple([int(x) for x in version.split(".")]) + else: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Metadata version contains unexpected characters: '{version}'", + ) + + @property + def datetime(self) -> Optional[datetime.datetime]: + """Returns a timestamp for when the file was produced""" + if "datetime" not in self._json_data: + return None + try: + timestamp = datetime.datetime.strptime( + self._json_data["datetime"], "YYYY-MM-DD" + ) + except (TypeError, ValueError): + vollog.debug("Invalid timestamp in producer information of symbol table") + return None + return timestamp class WindowsMetadata(interfaces.symbols.MetadataInterface): diff --git a/volatility3/schemas/schema-6.3.0.json b/volatility3/schemas/schema-6.3.0.json new file mode 100644 index 000000000..2cd8ebbda --- /dev/null +++ b/volatility3/schemas/schema-6.3.0.json @@ -0,0 +1,503 @@ +{ + "$schema": "http://json-schema.org/schema#", + "id": "http://volatilityfoundation.org/intermediate-format/schema", + "title": "Symbol Container", + "type": "object", + "definitions": { + "metadata_producer": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+.[0-9]+.[0-9]+$" + }, + "datetime": { + "type": "string", + "format": "date-time" + } + }, + "required":[ + "name", + "version" + ] + }, + "metadata_windows_pe": { + "type": "object", + "properties": { + "major": { + "type": "integer" + }, + "minor": { + "type": "integer" + }, + "revision": { + "type": "integer" + }, + "build": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "major", + "minor", + "revision" + ] + }, + "metadata_windows_pdb": { + "type": "object", + "properties": { + "GUID": { + "type": "string" + }, + "age": { + "type": "integer" + }, + "database": { + "type": "string" + }, + "machine_type": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "GUID", + "age", + "database", + "machine_type" + ] + }, + "metadata_windows": { + "type": "object", + "properties": { + "pe": { + "$ref": "#/definitions/metadata_windows_pe" + }, + "pdb": { + "$ref": "#/definitions/metadata_windows_pdb" + } + }, + "additionalProperties": false + }, + "metadata_nix": { + "type": "object", + "properties": { + "symbols": { + "type": "array", + "items": { + "$ref": "#/definitions/metadata_nix_item" + } + }, + "types": { + "type": "array", + "items": { + "$ref": "#/definitions/metadata_nix_item" + } + } + }, + "additionalProperties": false + }, + "metadata_format": { + "type": "string", + "pattern": "^6.[0-9]+.[0-9]+$" + }, + "metadata_nix_item": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "pattern": "^(dwarf|symtab|system-map)$" + }, + "name": { + "type": "string" + }, + "hash_type": { + "type": "string", + "pattern": "^(sha256)$" + }, + "hash_value": { + "type": "string", + "pattern": "^[a-fA-F0-9]+$" + } + }, + "additionalProperties": false + }, + "element_metadata": { + "type": "object", + "oneOf": [ + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + } + }, + "required": [ + "format" + ], + "additionalProperties": false + }, + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + }, + "windows": { + "$ref": "#/definitions/metadata_windows" + } + }, + "required": [ + "format", + "windows" + ], + "additionalProperties": false + }, + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + }, + "linux": { + "$ref": "#/definitions/metadata_nix" + } + }, + "required": [ + "format", + "linux" + ], + "additionalProperties": false + }, + { + "properties": { + "format": { + "$ref": "#/definitions/metadata_format" + }, + "producer": { + "$ref": "#/definitions/metadata_producer" + }, + "mac": { + "$ref": "#/definitions/metadata_nix" + } + }, + "required": [ + "format", + "mac" + ], + "additionalProperties": false + } + ] + }, + "element_enum": { + "properties": { + "size": { + "type": "integer" + }, + "base": { + "type": "string" + }, + "constants": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + } + }, + "required": [ + "size", + "base", + "constants" + ], + "additionalProperties": false + }, + "element_symbol": { + "properties": { + "address": { + "type": "number" + }, + "linkage_name": { + "type": "string" + }, + "type": { + "$ref": "#/definitions/type_descriptor" + }, + "constant_data": { + "type": "string", + "media": { + "binaryEncoding": "base64", + "readOnly": true + } + } + }, + "required": [ + "address" + ], + "additionalProperties": false + }, + "element_base_type": { + "properties": { + "size": { + "type": "integer" + }, + "signed": { + "type": "boolean" + }, + "kind": { + "type": "string", + "pattern": "^(void|int|float|char|bool)$" + }, + "endian": { + "type": "string", + "pattern": "^(little|big)$" + } + }, + "required": [ + "size", + "kind", + "signed", + "endian" + ], + "additionalProperties": false + }, + "element_user_type": { + "properties": { + "kind": { + "type": "string", + "pattern": "^(struct|union|class)$" + }, + "size": { + "type": "integer" + }, + "fields": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/field" + } + } + }, + "required": [ + "kind", + "size", + "fields" + ], + "additionalProperties": false + }, + "field": { + "properties": { + "type": { + "$ref": "#/definitions/type_descriptor" + }, + "offset": { + "type": "integer" + }, + "anonymous": { + "type": "boolean" + } + }, + "required": [ + "type", + "offset" + ], + "additionalProperties": false + }, + "type_descriptor": { + "oneOf": [ + { + "$ref": "#/definitions/type_pointer" + }, + { + "$ref": "#/definitions/type_base" + }, + { + "$ref": "#/definitions/type_array" + }, + { + "$ref": "#/definitions/type_struct" + }, + { + "$ref": "#/definitions/type_enum" + }, + { + "$ref": "#/definitions/type_function" + }, + { + "$ref": "#/definitions/type_bitfield" + } + ] + }, + "type_pointer": { + "properties": { + "kind": { + "type": "string", + "pattern": "^pointer$" + }, + "base": { + "type": "string" + }, + "subtype": { + "$ref": "#/definitions/type_descriptor" + } + }, + "required": [ + "kind", + "subtype" + ], + "additionalProperties": false + }, + "type_base": { + "properties": { + "kind": { + "type": "string", + "pattern": "^base$" + }, + "name": { + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "additionalProperties": false + }, + "type_array": { + "properties": { + "kind": { + "type": "string", + "pattern": "^array$" + }, + "subtype": { + "$ref": "#/definitions/type_descriptor" + }, + "count": { + "type": "integer" + } + }, + "required": [ + "kind", + "subtype", + "count" + ], + "additionalProperties": false + }, + "type_struct": { + "properties": { + "kind": { + "type": "string", + "pattern": "^(struct|class|union)$" + }, + "name": { + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "additionalProperties": false + }, + "type_enum": { + "properties": { + "kind": { + "type": "string", + "pattern": "^enum$" + }, + "name": { + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "additionalProperties": false + }, + "type_function": { + "properties": { + "kind": { + "type": "string", + "pattern": "^function$" + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + "type_bitfield": { + "properties": { + "kind": { + "type": "string", + "pattern": "^bitfield$" + }, + "bit_position": { + "type": "integer" + }, + "bit_length": { + "type": "integer" + }, + "type": { + "oneOf": [ + { + "$ref": "#/definitions/type_base" + }, + { + "$ref": "#/definitions/type_enum" + } + ] + } + }, + "required": [ + "kind", + "bit_position", + "bit_length", + "type" + ], + "additionalProperties": false + } + }, + "properties": { + "metadata": { + "$ref": "#/definitions/element_metadata" + }, + "base_types": { + "additionalProperties": { + "$ref": "#/definitions/element_base_type" + } + }, + "user_types": { + "additionalProperties": { + "$ref": "#/definitions/element_user_type" + } + }, + "enums": { + "additionalProperties": { + "$ref": "#/definitions/element_enum" + } + }, + "symbols": { + "additionalProperties": { + "$ref": "#/definitions/element_symbol" + } + } + }, + "required": [ + "metadata", + "base_types", + "user_types", + "enums", + "symbols" + ], + "additionalProperties": false +} From 7c8f4ca4ac5c23da3afa1d0ec6404c3fada69a27 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 14 Jan 2024 22:44:29 +0000 Subject: [PATCH 07/21] Schemas: As of 6.3.0 we require a producer metadata, not just format --- volatility3/schemas/schema-6.3.0.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/schemas/schema-6.3.0.json b/volatility3/schemas/schema-6.3.0.json index 2cd8ebbda..ba6e970bb 100644 --- a/volatility3/schemas/schema-6.3.0.json +++ b/volatility3/schemas/schema-6.3.0.json @@ -139,7 +139,8 @@ } }, "required": [ - "format" + "format", + "producer" ], "additionalProperties": false }, @@ -157,6 +158,7 @@ }, "required": [ "format", + "producer", "windows" ], "additionalProperties": false @@ -175,6 +177,7 @@ }, "required": [ "format", + "producer", "linux" ], "additionalProperties": false @@ -193,6 +196,7 @@ }, "required": [ "format", + "producer", "mac" ], "additionalProperties": false From 269058021dc171c7b46a6a2cb20d78d646b9f8fd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 17 Jan 2024 20:34:39 +0000 Subject: [PATCH 08/21] Plugins: Fix up kmsg code issues --- volatility3/framework/plugins/linux/kmsg.py | 1 - volatility3/framework/symbols/metadata.py | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d79248256..dd707a7ff 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -15,7 +15,6 @@ from volatility3.framework import ( renderers, ) from volatility3.framework.configuration import requirements -from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility vollog = logging.getLogger(__name__) diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index f765e43c9..149829371 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -26,11 +26,11 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): return None if all([x in "0123456789." for x in version]): return tuple([int(x) for x in version.split(".")]) - else: - vollog.log( - constants.LOGLEVEL_VVVV, - f"Metadata version contains unexpected characters: '{version}'", - ) + vollog.log( + constants.LOGLEVEL_VVVV, + f"Metadata version contains unexpected characters: '{version}'", + ) + return None @property def datetime(self) -> Optional[datetime.datetime]: From 130621a2e7655015f220ac561de974e99f571fca Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 2 Feb 2024 20:04:36 +0100 Subject: [PATCH 09/21] Changing comment when Type error is occuring --- volatility3/framework/plugins/windows/iat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 11c273859..73976fb86 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -50,7 +50,7 @@ class IAT(interfaces.plugins.PluginInterface): ) if proc_layer_name is None: - raise TypeError("Layer must be a string not None") + raise TypeError("add_process_layer failed") pe_table_name = intermed.IntermediateSymbolTable.create( self.context, From ab8ed049a4c4bddbc4c2a2129cb425b9113b5d37 Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sat, 3 Feb 2024 11:32:00 +0300 Subject: [PATCH 10/21] Windows: add TrueCrypt plugin --- .../framework/plugins/windows/truecrypt.py | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 volatility3/framework/plugins/windows/truecrypt.py diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py new file mode 100644 index 000000000..988c900ea --- /dev/null +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -0,0 +1,141 @@ +from typing import Iterable, Generator, List, Tuple + +from volatility3.framework import constants, interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces.configuration import RequirementInterface +from volatility3.framework.interfaces.objects import ObjectInterface +from volatility3.framework.objects import Bytes, DataFormatInfo, Integer, StructType +from volatility3.framework.objects.templates import ObjectTemplate +from volatility3.framework.objects.utility import array_to_string +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import pe + +from volatility3.plugins.windows import modules + +class Passphrase(interfaces.plugins.PluginInterface): + """TrueCrypt Cached Passphrase Finder""" + + _version = (0, 1, 0) + _required_framework_version = (2, 5, 2) + + @classmethod + def get_requirements(cls) -> List[RequirementInterface]: + return [ + requirements.ModuleRequirement( + 'kernel', + description='Windows kernel', + architectures=['Intel32', 'Intel64'] + ), + requirements.VersionRequirement( + name='modules', + component=modules.Modules, + version=(1, 1, 0) + ), + requirements.IntRequirement( + name="min-length", + description="Minimum length of passphrases to identify", + default=5, + optional=True + ), + ] + + def scan_module(self, module_base: int, layer_name: str) -> Generator[Tuple[int, str], None, None]: + """Scans the TrueCrypt kernel module for cached passphrases. + + Args: + module_base: the module's DLL base + layer_name: the name of the layer in which the module resides + + Generates: + A tuple of the offset at which a password is found, and the password + """ + pe_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "windows", + "pe", + class_types=pe.class_types + ) + dos_header: pe.IMAGE_DOS_HEADER = self.context.object( + pe_table_name + constants.BANG + '_IMAGE_DOS_HEADER', + layer_name, + module_base, + ) + data_section: StructType = next( + sec for sec in dos_header.get_nt_header().get_sections() + if array_to_string(sec.Name) == '.data' + ) + base: int = data_section.VirtualAddress + module_base + size: int = data_section.Misc.VirtualSize + # Looking at `Length` in TrueCrypt/Common/Password.h::Password struct + DWORD_SIZE_BYTES: int = 4 + format = DataFormatInfo(length=DWORD_SIZE_BYTES, byteorder="little", signed=True) + int32 = ObjectTemplate( + Integer, + pe_table_name + constants.BANG + 'int', + data_format=format + ) + count, not_aligned = divmod(size, DWORD_SIZE_BYTES) + if not_aligned: + raise ValueError("PE data section not DWORD-aligned!") + lengths = self.context.object( + pe_table_name + constants.BANG + 'array', + layer_name, + base, + count=count, + subtype=int32, + ) + min_length = self.config.get('min-length') + for length in lengths: + # TrueCrypt maximum password length is 64 + # (see TrueCrypt/Common/Password.h) + if not min_length <= length <= 64: + continue + offset = length.vol['offset'] + DWORD_SIZE_BYTES + passphrase: Bytes = self.context.object( + pe_table_name + constants.BANG + 'bytes', + layer_name, + offset, + length=length, + ) + # TrueCrypt/Common/Password.c permits chars in the range + # [0x20, 0x7F). + if not all(0x20 <= c < 0x7F for c in passphrase): + continue + # TrueCrypt/Common/Password.h::Password struct is padded with + # 3 zero bytes to keep 64-byte alignment. + buf: Bytes = self.context.object( + pe_table_name + constants.BANG + 'bytes', + layer_name, + offset + length + 1, # +1 for '\0'-terminated password string + length=3 + ) + if any(buf): + continue + # Password found. + yield offset, passphrase.decode(encoding='ascii') + + def _generator(self): + kernel = self.context.modules[self.config["kernel"]] + mods: Iterable[ObjectInterface] = modules.Modules.list_modules( + self.context, + kernel.layer_name, + kernel.symbol_table_name + ) + truecrypt_module_base = next( + mod.DllBase for mod in mods + if mod.BaseDllName.get_string().lower() == 'truecrypt.sys' + ) + for offset, password in self.scan_module(truecrypt_module_base, kernel.layer_name): + yield (0, (format_hints.Hex(offset), len(password), password)) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Length", int), + ("Password", str), + ], + self._generator() + ) From 41cbfbfa340ef8949d20e1717aa5baaee02ef984 Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sat, 3 Feb 2024 11:58:25 +0300 Subject: [PATCH 11/21] add license --- volatility3/framework/plugins/windows/truecrypt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py index 988c900ea..feba56965 100644 --- a/volatility3/framework/plugins/windows/truecrypt.py +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -1,3 +1,7 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + from typing import Iterable, Generator, List, Tuple from volatility3.framework import constants, interfaces, renderers From cf9029fd85ae090034a253e3dcd819a0381b6442 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 4 Feb 2024 00:43:33 +0100 Subject: [PATCH 12/21] Fixing the year in the Copyright --- volatility3/framework/plugins/windows/iat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index 73976fb86..d2fdc0ad8 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,4 +1,4 @@ -# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 import logging, io, pefile From d44c0fef36dfe1f7a83cc25211073489a56820be Mon Sep 17 00:00:00 2001 From: Iyassou Shimels Date: Sun, 4 Feb 2024 11:21:14 +0300 Subject: [PATCH 13/21] run Black formatter --- .../framework/plugins/windows/truecrypt.py | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/volatility3/framework/plugins/windows/truecrypt.py b/volatility3/framework/plugins/windows/truecrypt.py index feba56965..81250a749 100644 --- a/volatility3/framework/plugins/windows/truecrypt.py +++ b/volatility3/framework/plugins/windows/truecrypt.py @@ -17,6 +17,7 @@ from volatility3.framework.symbols.windows.extensions import pe from volatility3.plugins.windows import modules + class Passphrase(interfaces.plugins.PluginInterface): """TrueCrypt Cached Passphrase Finder""" @@ -27,78 +28,75 @@ class Passphrase(interfaces.plugins.PluginInterface): def get_requirements(cls) -> List[RequirementInterface]: return [ requirements.ModuleRequirement( - 'kernel', - description='Windows kernel', - architectures=['Intel32', 'Intel64'] + "kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], ), requirements.VersionRequirement( - name='modules', - component=modules.Modules, - version=(1, 1, 0) + name="modules", component=modules.Modules, version=(1, 1, 0) ), requirements.IntRequirement( name="min-length", description="Minimum length of passphrases to identify", default=5, - optional=True + optional=True, ), ] - - def scan_module(self, module_base: int, layer_name: str) -> Generator[Tuple[int, str], None, None]: + + def scan_module( + self, module_base: int, layer_name: str + ) -> Generator[Tuple[int, str], None, None]: """Scans the TrueCrypt kernel module for cached passphrases. - + Args: module_base: the module's DLL base layer_name: the name of the layer in which the module resides - + Generates: A tuple of the offset at which a password is found, and the password """ pe_table_name = intermed.IntermediateSymbolTable.create( - self.context, - self.config_path, - "windows", - "pe", - class_types=pe.class_types + self.context, self.config_path, "windows", "pe", class_types=pe.class_types ) dos_header: pe.IMAGE_DOS_HEADER = self.context.object( - pe_table_name + constants.BANG + '_IMAGE_DOS_HEADER', + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", layer_name, module_base, ) data_section: StructType = next( - sec for sec in dos_header.get_nt_header().get_sections() - if array_to_string(sec.Name) == '.data' + sec + for sec in dos_header.get_nt_header().get_sections() + if array_to_string(sec.Name) == ".data" ) base: int = data_section.VirtualAddress + module_base size: int = data_section.Misc.VirtualSize # Looking at `Length` in TrueCrypt/Common/Password.h::Password struct DWORD_SIZE_BYTES: int = 4 - format = DataFormatInfo(length=DWORD_SIZE_BYTES, byteorder="little", signed=True) + format = DataFormatInfo( + length=DWORD_SIZE_BYTES, byteorder="little", signed=True + ) int32 = ObjectTemplate( - Integer, - pe_table_name + constants.BANG + 'int', - data_format=format + Integer, pe_table_name + constants.BANG + "int", data_format=format ) count, not_aligned = divmod(size, DWORD_SIZE_BYTES) if not_aligned: raise ValueError("PE data section not DWORD-aligned!") lengths = self.context.object( - pe_table_name + constants.BANG + 'array', + pe_table_name + constants.BANG + "array", layer_name, base, count=count, subtype=int32, ) - min_length = self.config.get('min-length') + min_length = self.config.get("min-length") for length in lengths: # TrueCrypt maximum password length is 64 # (see TrueCrypt/Common/Password.h) if not min_length <= length <= 64: continue - offset = length.vol['offset'] + DWORD_SIZE_BYTES + offset = length.vol["offset"] + DWORD_SIZE_BYTES passphrase: Bytes = self.context.object( - pe_table_name + constants.BANG + 'bytes', + pe_table_name + constants.BANG + "bytes", layer_name, offset, length=length, @@ -110,30 +108,31 @@ class Passphrase(interfaces.plugins.PluginInterface): # TrueCrypt/Common/Password.h::Password struct is padded with # 3 zero bytes to keep 64-byte alignment. buf: Bytes = self.context.object( - pe_table_name + constants.BANG + 'bytes', + pe_table_name + constants.BANG + "bytes", layer_name, - offset + length + 1, # +1 for '\0'-terminated password string - length=3 + offset + length + 1, # +1 for '\0'-terminated password string + length=3, ) if any(buf): continue # Password found. - yield offset, passphrase.decode(encoding='ascii') - + yield offset, passphrase.decode(encoding="ascii") + def _generator(self): kernel = self.context.modules[self.config["kernel"]] mods: Iterable[ObjectInterface] = modules.Modules.list_modules( - self.context, - kernel.layer_name, - kernel.symbol_table_name + self.context, kernel.layer_name, kernel.symbol_table_name ) truecrypt_module_base = next( - mod.DllBase for mod in mods - if mod.BaseDllName.get_string().lower() == 'truecrypt.sys' + mod.DllBase + for mod in mods + if mod.BaseDllName.get_string().lower() == "truecrypt.sys" ) - for offset, password in self.scan_module(truecrypt_module_base, kernel.layer_name): + for offset, password in self.scan_module( + truecrypt_module_base, kernel.layer_name + ): yield (0, (format_hints.Hex(offset), len(password), password)) - + def run(self) -> renderers.TreeGrid: return renderers.TreeGrid( [ @@ -141,5 +140,5 @@ class Passphrase(interfaces.plugins.PluginInterface): ("Length", int), ("Password", str), ], - self._generator() + self._generator(), ) From 6a384c197c7cc7429ea14417cec8703a5f759395 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 7 Feb 2024 09:04:55 +0000 Subject: [PATCH 14/21] Linux: update mm_struct extension so that _get_maple_tree_iter and _get_mmap_iter are more clearly identified, showing that get_vma_iter should be used instead --- .../symbols/linux/extensions/__init__.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index d73d0cfb9..b57803d20 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -447,12 +447,14 @@ class maple_tree(objects.StructType): class mm_struct(objects.StructType): - def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns an iterator for the mmap list member of an mm_struct.""" + def _get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns an iterator for the mmap list member of an mm_struct. Use this only if + required, get_vma_iter() will choose the correct _get_maple_tree_iter() or + _get_mmap_iter() automatically as required.""" if not self.has_member("mmap"): raise AttributeError( - "get_mmap_iter called on mm_struct where no mmap member exists." + "_get_mmap_iter called on mm_struct where no mmap member exists." ) if not self.mmap: return None @@ -466,12 +468,14 @@ class mm_struct(objects.StructType): seen.add(link.vol.offset) link = link.vm_next - def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns an iterator for the mm_mt member of an mm_struct.""" + def _get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """Returns an iterator for the mm_mt member of an mm_struct. Use this only if + required, get_vma_iter() will choose the correct _get_maple_tree_iter() or + get_mmap_iter() automatically as required.""" if not self.has_member("mm_mt"): raise AttributeError( - "get_maple_tree_iter called on mm_struct where no mm_mt member exists." + "_get_maple_tree_iter called on mm_struct where no mm_mt member exists." ) symbol_table_name = self.get_symbol_table_name() for vma_pointer in self.mm_mt.get_slot_iter(): @@ -487,9 +491,9 @@ class mm_struct(objects.StructType): """Returns an iterator for the VMAs in an mm_struct. Automatically choosing the mmap or mm_mt as required.""" if self.has_member("mmap"): - yield from self.get_mmap_iter() + yield from self._get_mmap_iter() elif self.has_member("mm_mt"): - yield from self.get_maple_tree_iter() + yield from self._get_maple_tree_iter() else: raise AttributeError("Unable to find mmap or mm_mt in mm_struct") From 18220498ae572fadf6e47c3e38ffe65c24447183 Mon Sep 17 00:00:00 2001 From: Eve Date: Wed, 7 Feb 2024 09:07:19 +0000 Subject: [PATCH 15/21] Linux: update linux.pslist plugin to use task.mm.get_vma_iter() so that it works correctly on newer kernels --- volatility3/framework/plugins/linux/pslist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 9afd13e5a..771040dc0 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -17,7 +17,7 @@ class PsList(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 2, 0) + _version = (2, 2, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -128,7 +128,7 @@ class PsList(interfaces.plugins.PluginInterface): else: # Find the vma that belongs to the main ELF of the process file_output = "Error outputting file" - for v in task.mm.get_mmap_iter(): + for v in task.mm.get_vma_iter(): if v.vm_start == task.mm.start_code: file_handle = elfs.Elfs.elf_dump( self.context, From 9127509db9a8bcfffc971e7b959d429ffcecea6e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:13:14 +0000 Subject: [PATCH 16/21] CLI: Reblack after updating PR --- volatility3/cli/__init__.py | 45 ++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 0e7ba41ee..9393c0805 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -106,7 +106,7 @@ class CommandLine: ) # Load up system defaults - delayed_logs, default_config = self.load_system_defaults('vol.json') + delayed_logs, default_config = self.load_system_defaults("vol.json") parser = volargparse.HelpfulArgParser( add_help=False, @@ -280,7 +280,7 @@ class CommandLine: if partial_args.cache_path: constants.CACHE_PATH = partial_args.cache_path - + vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") @@ -473,28 +473,47 @@ class CommandLine: ) return requirements.URIRequirement.location_from_file(filename) - def load_system_defaults(self, filename: str) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: + def load_system_defaults( + self, filename: str + ) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: """Modify the main configuration based on the default configuration override""" # Build the config path - default_config_path = os.path.join(os.path.expanduser("~"), ".config", "volatility3", filename) - if sys.platform == 'win32': - default_config_path = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3", - filename) + default_config_path = os.path.join( + os.path.expanduser("~"), ".config", "volatility3", filename + ) + if sys.platform == "win32": + default_config_path = os.path.join( + os.environ.get("APPDATA", os.path.expanduser("~")), + "volatility3", + filename, + ) delayed_logs = [] # Process it if the files exist if os.path.exists(default_config_path): - with open(default_config_path, 'rb') as config_json: + with open(default_config_path, "rb") as config_json: result = json.load(config_json) if not isinstance(result, dict): - delayed_logs.append((logging.INFO, - f'Default configuration file {default_config_path} does not contain a dictionary')) + delayed_logs.append( + ( + logging.INFO, + f"Default configuration file {default_config_path} does not contain a dictionary", + ) + ) else: delayed_logs.append( - (logging.INFO, f"Loading default configuration options from {default_config_path}")) - delayed_logs.append((logging.DEBUG, - f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}")) + ( + logging.INFO, + f"Loading default configuration options from {default_config_path}", + ) + ) + delayed_logs.append( + ( + logging.DEBUG, + f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}", + ) + ) return delayed_logs, result return delayed_logs, {} From a66a4461906e7661b957559634559a56ee6356f6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:16:35 +0000 Subject: [PATCH 17/21] CLI: Reblack after updating PR - take 2 --- volatility3/cli/volshell/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 40f42f3af..9e74acfec 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -56,7 +56,7 @@ class VolShell(cli.CommandLine): framework.require_interface_version(2, 0, 0) # Load up system defaults - delayed_logs, default_config = self.load_system_defaults('volshell.json') + delayed_logs, default_config = self.load_system_defaults("volshell.json") parser = argparse.ArgumentParser( prog=self.CLI_NAME, @@ -151,10 +151,12 @@ class VolShell(cli.CommandLine): default=constants.CACHE_PATH, type=str, ) - parser.add_argument("--offline", - help = "Do not search online for additional JSON files", - default = False, - action = 'store_true') + parser.add_argument( + "--offline", + help="Do not search online for additional JSON files", + default=False, + action="store_true", + ) # Volshell specific flags os_specific = parser.add_mutually_exclusive_group(required=False) From 7b42ef0a9f23ecb6b4fff2425c0ac33c85b30d04 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:59:50 +0000 Subject: [PATCH 18/21] Documentation: Add in more information about overriding configuration values --- doc/source/vol-cli.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index cc6f7fe6a..7b91e815d 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -143,3 +143,18 @@ Options `hivescan` would match `windows.registry.hivescan.HiveScan`, but `pslist` is ambiguous because it could match `windows.pslist` or `linux.pslist`. + +Overriding options +------------------ + +The default values for the command line interface are defined by constants within the code, +but can be overridden by creating a JSON file (`%APPDATA%/volatility3/vol.json` for Windows +systems, or `~/.config/volatility3/vol.json` or `volshell.json` for all others). + +The format of this file is a JSON dictionary, containing the options above and their value. +It should be noted that the ordering is (`<` means is overridden by): + +`in-built default value < config file value < command line parameter` + +It should also be noted that boolean flags (such as `offline`) that are overridden as true will +not be unset by not specifying the command line flag. From 683319bd7f96cef33e4b1cc4333d4c0835310113 Mon Sep 17 00:00:00 2001 From: Eve Date: Mon, 12 Feb 2024 08:47:02 +0000 Subject: [PATCH 19/21] Linux: add deprecation warning for get_maple_tree_iter and get_mmap_iter mm_struct extensions. --- .../symbols/linux/extensions/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b57803d20..f9db30498 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -447,6 +447,16 @@ class maple_tree(objects.StructType): class mm_struct(objects.StructType): + + def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """ + Deprecated: Use either get_vma_iter() or _get_mmap_iter(). + """ + vollog.warning( + "This method has been deprecated in favour of using the get_vma_iter() method." + ) + yield from self.get_vma_iter() + def _get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or @@ -468,6 +478,15 @@ class mm_struct(objects.StructType): seen.add(link.vol.offset) link = link.vm_next + def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: + """ + Deprecated: Use either get_vma_iter() or _get_maple_tree_iter(). + """ + vollog.warning( + "This method has been deprecated in favour of using the get_vma_iter() method." + ) + yield from self.get_vma_iter() + def _get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mm_mt member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or From 437a91375db004de5d0df26443d1bd71ee9f2b94 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 20 Feb 2024 06:39:42 +0000 Subject: [PATCH 20/21] Linux: add TODO for mm_struct for methods that should be removed when moving to version 3.0.0 --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f9db30498..27113965f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -448,6 +448,7 @@ class maple_tree(objects.StructType): class mm_struct(objects.StructType): + # TODO: As of version 3.0.0 this method should be removed def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """ Deprecated: Use either get_vma_iter() or _get_mmap_iter(). @@ -478,6 +479,7 @@ class mm_struct(objects.StructType): seen.add(link.vol.offset) link = link.vm_next + # TODO: As of version 3.0.0 this method should be removed def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """ Deprecated: Use either get_vma_iter() or _get_maple_tree_iter(). From a597ee17768b1513f8587642c0d00c9170007f46 Mon Sep 17 00:00:00 2001 From: Eve Date: Tue, 20 Feb 2024 06:48:16 +0000 Subject: [PATCH 21/21] Linux: update minor version number due to API changes with linux extension for mm_struct --- volatility3/framework/constants/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 09dded076..9aaafb933 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -44,8 +44,8 @@ BANG = "!" # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 2 # Number of changes that do not change the interface +VERSION_MINOR = 6 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature