From d5a0b93383fda59267bdd9b42e716b70ad66595c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:28:40 +0100 Subject: [PATCH 001/268] add TAINT_FLAGS constant --- .../framework/constants/linux/__init__.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 7c485d3c3..9f25c9225 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -347,3 +347,88 @@ class PT_FLAGS(Flag): MODULE_MAXIMUM_CORE_SIZE = 20000000 MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 MODULE_MINIMUM_SIZE = 4096 + + +TAINT_FLAGS = { + "P": { + "shift": 1 << 0, + "desc": "PROPRIETARY_MODULE", + "when_present": True, + "module": True, + }, + "G": { + "shift": 1 << 0, + "desc": "PROPRIETARY_MODULE", + "when_present": False, + "module": True, + }, + "F": { + "shift": 1 << 1, + "desc": "FORCED_MODULE", + "when_present": True, + "module": False, + }, + # CPU_OUT_OF_SPEC was TAINT_UNSAFE_SMP on < 3.15-rc1 : https://lore.kernel.org/linux-kernel//20140303080432.GA25489@localhost/t/#:~:text=liked%20your%20proposal%3A-,%3E%20Right,-%2C%20I%20was%20about + "S": { + "shift": 1 << 2, + "desc": "CPU_OUT_OF_SPEC", + "when_present": True, + "module": False, + }, + "R": { + "shift": 1 << 3, + "desc": "FORCED_RMMOD", + "when_present": True, + "module": False, + }, + "M": { + "shift": 1 << 4, + "desc": "MACHINE_CHECK", + "when_present": True, + "module": False, + }, + "B": {"shift": 1 << 5, "desc": "BAD_PAGE", "when_present": True, "module": False}, + "U": {"shift": 1 << 6, "desc": "USER", "when_present": True, "module": False}, + "D": {"shift": 1 << 7, "desc": "DIE", "when_present": True, "module": False}, + "A": { + "shift": 1 << 8, + "desc": "OVERRIDDEN_ACPI_TABLE", + "when_present": True, + "module": False, + }, + "W": {"shift": 1 << 9, "desc": "WARN", "when_present": True, "module": False}, + "C": {"shift": 1 << 10, "desc": "CRAP", "when_present": True, "module": True}, + "I": { + "shift": 1 << 11, + "desc": "FIRMWARE_WORKAROUND", + "when_present": True, + "module": False, + }, + "O": {"shift": 1 << 12, "desc": "OOT_MODULE", "when_present": True, "module": True}, + "E": { + "shift": 1 << 13, + "desc": "UNSIGNED_MODULE", + "when_present": True, + "module": True, + }, + "L": { + "shift": 1 << 14, + "desc": "SOFTLOCKUP", + "when_present": True, + "module": False, + }, + "K": {"shift": 1 << 15, "desc": "LIVEPATCH", "when_present": True, "module": True}, + "X": {"shift": 1 << 16, "desc": "AUX", "when_present": True, "module": True}, + "T": {"shift": 1 << 17, "desc": "RANDSTRUCT", "when_present": True, "module": True}, + "N": {"shift": 1 << 18, "desc": "TEST", "when_present": True, "module": True}, +} +"""Flags used to taint kernel and modules, for debugging purposes. + +Map based on 6.12-rc5. + +Documentation : + - https://www.kernel.org/doc/Documentation/admin-guide/sysctl/kernel.rst#:~:text=guide/sysrq.rst.-,tainted,-%3D%3D%3D%3D%3D%3D%3D%0A%0ANon%2Dzero%20if + - https://www.kernel.org/doc/Documentation/admin-guide/tainted-kernels.rst#:~:text=More%20detailed%20explanation%20for%20tainting + - taint_flag kernel struct + - taint_flags kernel constant +""" From e1b343a284436ee4d92a7b8a6daf0a98dd01fdeb Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:33:54 +0100 Subject: [PATCH 002/268] add module taints parsing apis --- .../symbols/linux/extensions/__init__.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index aa3e8c675..89e2cde27 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -279,6 +279,77 @@ class module(generic.GenericIntelProcess): return None + def _module_flags_taints_pre_4_10_rc1(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Returns: + The raw taints string. + """ + taints_string = "" + for char, infos in linux_constants.TAINT_FLAGS.items(): + if infos["module"] and self.taints_value & infos["shift"]: + taints_string += char + + return taints_string + + def _module_flags_taints_post_4_10_rc1(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self.taint_flags_list): + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taint_flag.module and (self.taints_value & (1 << i)): + taints_string += c_true + elif taint_flag.module and c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self.taint_flags_list: + return self._module_flags_taints_post_4_10_rc1() + return self._module_flags_taints_pre_4_10_rc1() + + def get_taints_parsed(self) -> List[str]: + """Convert the module's taints string to a 1-1 descriptor mapping. + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for c in self.get_taints_as_plain_string(): + infos = linux_constants.TAINT_FLAGS.get(c) + if not infos: + comprehensive_taints.append(f"") + elif infos["when_present"]: + comprehensive_taints.append(infos["desc"]) + + return comprehensive_taints + @property def section_symtab(self): if self.has_member("kallsyms"): @@ -307,6 +378,17 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("module -> strtab: Unable to get strtab") + @property + def taints_value(self) -> int: + return self.taints + + @property + def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: + kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + if kernel.has_symbol("taint_flags"): + return list(kernel.object_from_symbol("taint_flags")) + return None + class task_struct(generic.GenericIntelProcess): def add_process_layer( From c88ebe89270355d188770c74b39eb8acef1f3549 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:35:54 +0100 Subject: [PATCH 003/268] introduce modxview linux plugin --- .../framework/plugins/linux/modxview.py | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 volatility3/framework/plugins/linux/modxview.py diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py new file mode 100644 index 000000000..66b644164 --- /dev/null +++ b/volatility3/framework/plugins/linux/modxview.py @@ -0,0 +1,195 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import List, Dict, Set, Iterator +from volatility3.plugins.linux import lsmod, check_modules, hidden_modules +from volatility3.framework import interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.symbols.linux import extensions +from volatility3.framework.constants import architectures + +vollog = logging.getLogger(__name__) + + +class Modxview(interfaces.plugins.PluginInterface): + """Centralize lsmod, check_modules and hidden_modules results to efficiently + spot modules presence and taints.""" + + _version = (1, 0, 0) + _required_framework_version = (2, 11, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="check_modules", + plugin=check_modules.Check_modules, + version=(0, 0, 0), + ), + requirements.PluginRequirement( + name="hidden_modules", + plugin=hidden_modules.Hidden_modules, + version=(1, 0, 0), + ), + requirements.BooleanRequirement( + name="plain_taints", + description="Display the plain taints string for each module.", + optional=True, + default=False, + ), + ] + + @classmethod + def run_lsmod( + cls, context: interfaces.context.ContextInterface, kernel_name: str + ) -> List[extensions.module]: + """Wrapper for the lsmod plugin.""" + return list(lsmod.Lsmod.list_modules(context, kernel_name)) + + @classmethod + def run_check_modules( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + ) -> List[extensions.module]: + """Wrapper for the check_modules plugin. + Here, we extract the /sys/module/ list.""" + kernel = context.modules[kernel_name] + sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( + context, kernel_name + ) + + # Convert get_kset_modules() offsets back to module objects + return [ + kernel.object(object_type="module", offset=m_offset, absolute=True) + for m_offset in sysfs_modules.values() + ] + + @classmethod + def run_hidden_modules( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + known_modules_addresses: Set[int], + ) -> List[extensions.module]: + """Wrapper for the hidden_modules plugin.""" + modules_memory_boundaries = ( + hidden_modules.Hidden_modules.get_modules_memory_boundaries( + context, kernel_name + ) + ) + return list( + hidden_modules.Hidden_modules.get_hidden_modules( + context, + kernel_name, + known_modules_addresses, + modules_memory_boundaries, + ) + ) + + @classmethod + def flatten_run_modules_results( + cls, run_results: Dict[str, List[extensions.module]], deduplicate: bool = True + ) -> Iterator[extensions.module]: + """Flatten a dictionary mapping plugin names and modules list, to a single merged list. + This is useful to get a generic lookup list of all the detected modules. + + Args: + run_results: dictionary of plugin names mapping a list of detected modules + deduplicate: remove duplicate modules, based on their offsets + + Returns: + Iterator of modules objects + """ + seen_addresses = set() + for modules in run_results.values(): + for module in modules: + if deduplicate and module.vol.offset in seen_addresses: + continue + yield module + + @classmethod + def run_modules_scanners( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + run_hidden_modules: bool = True, + ) -> Dict[str, List[extensions.module]]: + """Run module scanning plugins and aggregate the results. + + Args: + run_hidden_modules: specify if the hidden_modules plugin should be run + Returns: + Dictionary mapping each plugin to its corresponding result + """ + + kernel = context.modules[kernel_name] + run_results = {} + run_results["lsmod"] = cls.run_lsmod(context, kernel_name) + run_results["check_modules"] = cls.run_check_modules(context, kernel_name) + if run_hidden_modules: + known_module_addresses = set( + context.layers[kernel.layer_name].canonicalize(module.vol.offset) + for module in run_results["lsmod"] + run_results["check_modules"] + ) + run_results["hidden_modules"] = cls.run_hidden_modules( + context, kernel_name, known_module_addresses + ) + + return run_results + + def _generator(self): + kernel_name = self.config["kernel"] + run_results = self.run_modules_scanners(self.context, kernel_name) + modules_offsets = {} + for key in ["lsmod", "check_modules", "hidden_modules"]: + modules_offsets[key] = set(module.vol.offset for module in run_results[key]) + + seen_addresses = set() + for modules_list in run_results.values(): + for module in modules_list: + if module.vol.offset in seen_addresses: + continue + seen_addresses.add(module.vol.offset) + + if self.config.get("plain_taints"): + taints = module.get_taints_as_plain_string() + else: + taints = ",".join(module.get_taints_parsed()) + + yield ( + 0, + ( + module.get_name() or NotAvailableValue(), + format_hints.Hex(module.vol.offset), + module.vol.offset in modules_offsets["lsmod"], + module.vol.offset in modules_offsets["check_modules"], + module.vol.offset in modules_offsets["hidden_modules"], + taints or NotAvailableValue(), + ), + ) + + def run(self): + columns = [ + ("Name", str), + ("Address", format_hints.Hex), + ("In /proc/modules", bool), + ("In /sys/module/", bool), + ("Hidden", bool), + ("Taints", str), + ] + + return TreeGrid( + columns, + self._generator(), + ) From 9440f53429a1f9c7d77d51eeb75c2b5938da040f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:05:25 +0100 Subject: [PATCH 004/268] use a dict of dataclasses for taint_flags --- .../framework/constants/linux/__init__.py | 112 +++++++----------- .../symbols/linux/extensions/__init__.py | 12 +- 2 files changed, 47 insertions(+), 77 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 9f25c9225..6cf8585f5 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -6,6 +6,7 @@ Linux-specific values that aren't found in debug symbols """ from enum import IntEnum, Flag +from dataclasses import dataclass KERNEL_NAME = "__kernel__" @@ -349,78 +350,47 @@ MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 MODULE_MINIMUM_SIZE = 4096 +@dataclass +class TaintFlag: + shift: int + desc: str + when_present: bool + module: bool + + TAINT_FLAGS = { - "P": { - "shift": 1 << 0, - "desc": "PROPRIETARY_MODULE", - "when_present": True, - "module": True, - }, - "G": { - "shift": 1 << 0, - "desc": "PROPRIETARY_MODULE", - "when_present": False, - "module": True, - }, - "F": { - "shift": 1 << 1, - "desc": "FORCED_MODULE", - "when_present": True, - "module": False, - }, - # CPU_OUT_OF_SPEC was TAINT_UNSAFE_SMP on < 3.15-rc1 : https://lore.kernel.org/linux-kernel//20140303080432.GA25489@localhost/t/#:~:text=liked%20your%20proposal%3A-,%3E%20Right,-%2C%20I%20was%20about - "S": { - "shift": 1 << 2, - "desc": "CPU_OUT_OF_SPEC", - "when_present": True, - "module": False, - }, - "R": { - "shift": 1 << 3, - "desc": "FORCED_RMMOD", - "when_present": True, - "module": False, - }, - "M": { - "shift": 1 << 4, - "desc": "MACHINE_CHECK", - "when_present": True, - "module": False, - }, - "B": {"shift": 1 << 5, "desc": "BAD_PAGE", "when_present": True, "module": False}, - "U": {"shift": 1 << 6, "desc": "USER", "when_present": True, "module": False}, - "D": {"shift": 1 << 7, "desc": "DIE", "when_present": True, "module": False}, - "A": { - "shift": 1 << 8, - "desc": "OVERRIDDEN_ACPI_TABLE", - "when_present": True, - "module": False, - }, - "W": {"shift": 1 << 9, "desc": "WARN", "when_present": True, "module": False}, - "C": {"shift": 1 << 10, "desc": "CRAP", "when_present": True, "module": True}, - "I": { - "shift": 1 << 11, - "desc": "FIRMWARE_WORKAROUND", - "when_present": True, - "module": False, - }, - "O": {"shift": 1 << 12, "desc": "OOT_MODULE", "when_present": True, "module": True}, - "E": { - "shift": 1 << 13, - "desc": "UNSIGNED_MODULE", - "when_present": True, - "module": True, - }, - "L": { - "shift": 1 << 14, - "desc": "SOFTLOCKUP", - "when_present": True, - "module": False, - }, - "K": {"shift": 1 << 15, "desc": "LIVEPATCH", "when_present": True, "module": True}, - "X": {"shift": 1 << 16, "desc": "AUX", "when_present": True, "module": True}, - "T": {"shift": 1 << 17, "desc": "RANDSTRUCT", "when_present": True, "module": True}, - "N": {"shift": 1 << 18, "desc": "TEST", "when_present": True, "module": True}, + "P": TaintFlag( + shift=1 << 0, desc="PROPRIETARY_MODULE", when_present=True, module=True + ), + "G": TaintFlag( + shift=1 << 0, desc="PROPRIETARY_MODULE", when_present=False, module=True + ), + "F": TaintFlag(shift=1 << 1, desc="FORCED_MODULE", when_present=True, module=False), + "S": TaintFlag( + shift=1 << 2, desc="CPU_OUT_OF_SPEC", when_present=True, module=False + ), + "R": TaintFlag(shift=1 << 3, desc="FORCED_RMMOD", when_present=True, module=False), + "M": TaintFlag(shift=1 << 4, desc="MACHINE_CHECK", when_present=True, module=False), + "B": TaintFlag(shift=1 << 5, desc="BAD_PAGE", when_present=True, module=False), + "U": TaintFlag(shift=1 << 6, desc="USER", when_present=True, module=False), + "D": TaintFlag(shift=1 << 7, desc="DIE", when_present=True, module=False), + "A": TaintFlag( + shift=1 << 8, desc="OVERRIDDEN_ACPI_TABLE", when_present=True, module=False + ), + "W": TaintFlag(shift=1 << 9, desc="WARN", when_present=True, module=False), + "C": TaintFlag(shift=1 << 10, desc="CRAP", when_present=True, module=True), + "I": TaintFlag( + shift=1 << 11, desc="FIRMWARE_WORKAROUND", when_present=True, module=False + ), + "O": TaintFlag(shift=1 << 12, desc="OOT_MODULE", when_present=True, module=True), + "E": TaintFlag( + shift=1 << 13, desc="UNSIGNED_MODULE", when_present=True, module=True + ), + "L": TaintFlag(shift=1 << 14, desc="SOFTLOCKUP", when_present=True, module=False), + "K": TaintFlag(shift=1 << 15, desc="LIVEPATCH", when_present=True, module=True), + "X": TaintFlag(shift=1 << 16, desc="AUX", when_present=True, module=True), + "T": TaintFlag(shift=1 << 17, desc="RANDSTRUCT", when_present=True, module=True), + "N": TaintFlag(shift=1 << 18, desc="TEST", when_present=True, module=True), } """Flags used to taint kernel and modules, for debugging purposes. diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 89e2cde27..42c1a470d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -287,8 +287,8 @@ class module(generic.GenericIntelProcess): The raw taints string. """ taints_string = "" - for char, infos in linux_constants.TAINT_FLAGS.items(): - if infos["module"] and self.taints_value & infos["shift"]: + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if taint_flag.module and self.taints_value & taint_flag.shift: taints_string += char return taints_string @@ -342,11 +342,11 @@ class module(generic.GenericIntelProcess): """ comprehensive_taints = [] for c in self.get_taints_as_plain_string(): - infos = linux_constants.TAINT_FLAGS.get(c) - if not infos: + taint_flag = linux_constants.TAINT_FLAGS.get(c) + if not taint_flag: comprehensive_taints.append(f"") - elif infos["when_present"]: - comprehensive_taints.append(infos["desc"]) + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) return comprehensive_taints From 9d08c4681ae1cf18ddf4bd53ff970f6a9bc26573 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:07:23 +0100 Subject: [PATCH 005/268] add module offset to seen_addresses --- volatility3/framework/plugins/linux/modxview.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 66b644164..f44984926 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -116,6 +116,7 @@ class Modxview(interfaces.plugins.PluginInterface): for module in modules: if deduplicate and module.vol.offset in seen_addresses: continue + seen_addresses.add(module.vol.offset) yield module @classmethod From b209ea36a284ae1a75ba18222cbe8db1c1eede4f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:12:52 +0100 Subject: [PATCH 006/268] remove slashes in columns --- volatility3/framework/plugins/linux/modxview.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index f44984926..d79f5e7a9 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -184,8 +184,8 @@ class Modxview(interfaces.plugins.PluginInterface): columns = [ ("Name", str), ("Address", format_hints.Hex), - ("In /proc/modules", bool), - ("In /sys/module/", bool), + ("In procfs", bool), + ("In sysfs", bool), ("Hidden", bool), ("Taints", str), ] From c7259037356fde4cf6120acddecbae4af16c9ea0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 7 Nov 2024 18:08:58 +0100 Subject: [PATCH 007/268] introduce scatter-gather scatterlists --- .../framework/symbols/linux/__init__.py | 1 + .../symbols/linux/extensions/__init__.py | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3289775b6..e8a3d5240 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -43,6 +43,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.optional_set_type_class("bpf_prog_aux", extensions.bpf_prog_aux) self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) + self.optional_set_type_class("scatterlist", extensions.scatterlist) # kernels >= 4.18 self.optional_set_type_class("timespec64", extensions.timespec64) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index aa3e8c675..0dd657372 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2410,3 +2410,107 @@ class rb_root(objects.StructType): """ yield from self._walk_nodes(root_node=self.rb_node) + + +class scatterlist(objects.StructType): + SG_CHAIN = 0x01 + SG_END = 0x02 + SG_PAGE_LINK_MASK = SG_CHAIN | SG_END + + def _sg_flags(self) -> int: + return self.page_link & self.SG_PAGE_LINK_MASK + + def _sg_is_chain(self) -> int: + return self._sg_flags() & self.SG_CHAIN + + def _sg_is_last(self) -> int: + return self._sg_flags() & self.SG_END + + def _sg_chain_ptr(self) -> int: + """Clears the last two bits basically.""" + return self.page_link & ~self.SG_PAGE_LINK_MASK + + def _sg_dma_len(self) -> int: + # Depends on CONFIG_NEED_SG_DMA_LENGTH + if self.has_member("dma_length"): + return self.dma_length + return self.length + + def _get_sg_max_single_alloc(self) -> int: + """Based on kernel's SG_MAX_SINGLE_ALLOC. + + Doc. from kernel source : + * Maximum number of entries that will be allocated in one piece, if + * a list larger than this is required then chaining will be utilized. + """ + return self._context.layers[self.vol.layer_name].page_size // self.vol.size + + def _sg_next(self) -> interfaces.objects.ObjectInterface: + """Get the next scatterlist struct from the list. + Based on kernel's sg_next. + + Doc. from kernel source : + * Notes on SG table design. + * + * We use the unsigned long page_link field in the scatterlist struct to place + * the page pointer AND encode information about the sg table as well. The two + * lower bits are reserved for this information. + * + * If bit 0 is set, then the page_link contains a pointer to the next sg + * table list. Otherwise the next entry is at sg + 1. + * + * If bit 1 is set, then this sg entry is the last element in a list. + """ + if self._sg_is_last(): + return None + + if self._sg_is_chain(): + next_address = self._sg_chain_ptr() + else: + next_address = self.vol.offset + self.vol.size + + sg = self._context.object( + self.get_symbol_table_name() + constants.BANG + "scatterlist", + self.vol.layer_name, + next_address, + ) + return sg + + def for_each_sg(self) -> Iterator[interfaces.objects.ObjectInterface]: + """Iterate over each struct in the scatterlist.""" + sg = self + sg_max_single_alloc = self._get_sg_max_single_alloc() + + # Empty scatterlists protection + if sg.page_link == 0 and sg._sg_dma_len() == 0 and sg.dma_address == 0: + return None + else: + # Yield itself first + yield sg + + entries_count = 1 + # entries_count <= sg_max_single_alloc should always be true if the + # scatterlists were correctly chained. + while entries_count <= sg_max_single_alloc: + sg = sg._sg_next() + if sg is None: + break + # Points to a new scatterlist + elif sg._sg_is_chain(): + entries_count = 0 + else: + entries_count += 1 + yield sg + + def get_content( + self, + ) -> Iterator[bytes]: + """Traverse a scatterlist to gather content located at each + dma_address position. + + Returns: + An iterator of bytes + """ + physical_layer = self._context.layers["memory_layer"] + for sg in self.for_each_sg(): + yield from physical_layer.read(sg.dma_address, sg._sg_dma_len()) From 8f33aaf5b4ee859b12ca35347144597bec59ee1b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Nov 2024 17:45:53 +0100 Subject: [PATCH 008/268] Optional type hints --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0dd657372..e5a074a49 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2445,7 +2445,7 @@ class scatterlist(objects.StructType): """ return self._context.layers[self.vol.layer_name].page_size // self.vol.size - def _sg_next(self) -> interfaces.objects.ObjectInterface: + def _sg_next(self) -> Optional[interfaces.objects.ObjectInterface]: """Get the next scatterlist struct from the list. Based on kernel's sg_next. @@ -2476,7 +2476,7 @@ class scatterlist(objects.StructType): ) return sg - def for_each_sg(self) -> Iterator[interfaces.objects.ObjectInterface]: + def for_each_sg(self) -> Optional[Iterator[interfaces.objects.ObjectInterface]]: """Iterate over each struct in the scatterlist.""" sg = self sg_max_single_alloc = self._get_sg_max_single_alloc() @@ -2504,7 +2504,7 @@ class scatterlist(objects.StructType): def get_content( self, - ) -> Iterator[bytes]: + ) -> Optional[Iterator[bytes]]: """Traverse a scatterlist to gather content located at each dma_address position. From 485ef894e113cf68eb1acaef3c679b675639281d Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Nov 2024 17:49:08 +0100 Subject: [PATCH 009/268] remove taints_value overload attr --- .../framework/symbols/linux/extensions/__init__.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 42c1a470d..f9f72c161 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -288,7 +288,7 @@ class module(generic.GenericIntelProcess): """ taints_string = "" for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if taint_flag.module and self.taints_value & taint_flag.shift: + if taint_flag.module and self.taints & taint_flag.shift: taints_string += char return taints_string @@ -310,7 +310,7 @@ class module(generic.GenericIntelProcess): for i, taint_flag in enumerate(self.taint_flags_list): c_true = chr(taint_flag.c_true) c_false = chr(taint_flag.c_false) - if taint_flag.module and (self.taints_value & (1 << i)): + if taint_flag.module and (self.taints & (1 << i)): taints_string += c_true elif taint_flag.module and c_false != " ": taints_string += c_false @@ -378,10 +378,6 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("module -> strtab: Unable to get strtab") - @property - def taints_value(self) -> int: - return self.taints - @property def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) From dd3542b127b751ad083c91be4e8ffd373a1c74f7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Nov 2024 17:51:48 +0100 Subject: [PATCH 010/268] explicit loop iterator --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f9f72c161..402f8c9c6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -341,10 +341,10 @@ class module(generic.GenericIntelProcess): - module_flags_taint kernel function """ comprehensive_taints = [] - for c in self.get_taints_as_plain_string(): - taint_flag = linux_constants.TAINT_FLAGS.get(c) + for character in self.get_taints_as_plain_string(): + taint_flag = linux_constants.TAINT_FLAGS.get(character) if not taint_flag: - comprehensive_taints.append(f"") + comprehensive_taints.append(f"") elif taint_flag.when_present: comprehensive_taints.append(taint_flag.desc) From 9f145ddcf0ddb84cf7ae45585ce7e9da0d204a17 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:04:33 +0100 Subject: [PATCH 011/268] pillow dependency --- requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requirements.txt b/requirements.txt index e0d366391..21c8f9a76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,9 @@ leechcorepyc>=2.4.0; sys_platform != 'darwin' # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage gcsfs>=2023.1.0 s3fs>=2023.1.0 + +# This is required by plugins that manipulate pixels and images. +# https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst +# 10.0.0 dropped support for Python3.7 +# 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 +pillow>=10.0.0,<11.0.0 \ No newline at end of file From 36a18f405b8ba7605ca957ca47d540a5b7c520d7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:05:23 +0100 Subject: [PATCH 012/268] fourcc code converter helper --- volatility3/framework/symbols/linux/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3289775b6..573bc66d8 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -483,6 +483,22 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return kernel + @classmethod + def convert_fourcc_code(cls, code: int) -> str: + """Convert a fourcc integer back to its fourcc string representation. + + Args: + code: the numerical representation of the fourcc + + Returns: + The fourcc code string. + """ + + code_bytes_length = (code.bit_length() + 7) // 8 + return "".join( + [chr((code >> (i * 8)) & 0xFF) for i in range(code_bytes_length)] + ) + class IDStorage(ABC): """Abstraction to support both XArray and RadixTree""" From a7620cd6f659dfa685a445536a240182225a778c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:09:21 +0100 Subject: [PATCH 013/268] linux fbdev subsystem api plugin --- .../framework/plugins/linux/graphics/fbdev.py | 314 ++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 volatility3/framework/plugins/linux/graphics/fbdev.py diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py new file mode 100644 index 000000000..4bde6f62f --- /dev/null +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -0,0 +1,314 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +import io + +# Image manipulation functions are kept in the plugin, +# to prevent a general exit on missing PIL (pillow) dependency. +from PIL import Image +from dataclasses import dataclass +from typing import Type, List, Dict, Tuple +from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.objects import utility +from volatility3.framework.constants import architectures +from volatility3.framework.symbols import linux + +vollog = logging.getLogger(__name__) + + +@dataclass +class Framebuffer: + """Framebuffer object internal representation. This is useful to unify an framebuffer with precalculated + properties and pass it through functions conveniently.""" + + id: str + xres_virtual: int + yres_virtual: int + line_length: int + bpp: int + """Bits Per Pixel""" + size: int + color_fields: Dict[str, Tuple[int, int, int]] + fb_info: interfaces.objects.ObjectInterface + + +class Fbdev(interfaces.plugins.PluginInterface): + """Extract framebuffers from the fbdev graphics subsystem""" + + _version = (1, 0, 0) + _required_framework_version = (2, 11, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.BooleanRequirement( + name="dump", + description="Dump framebuffers", + default=False, + optional=True, + ), + ] + + @classmethod + def parse_fb_pixel_bitfields( + cls, fb_var_screeninfo: interfaces.objects.ObjectInterface + ) -> Dict[str, Tuple[int, int, int]]: + """Organize a framebuffer pixel format into a dictionary. + This is needed to know the position and bitlength of a color inside + a pixel. + + Args: + fb_var_screeninfo: a fb_var_screeninfo kernel object instance + + Returns: + The color fields mappings + + Documentation: + include/uapi/linux/fb.h: + struct fb_bitfield { + __u32 offset; /* beginning of bitfield */ + __u32 length; /* length of bitfield */ + __u32 msb_right; /* != 0 : Most significant bit is right */ + }; + """ + # Naturally order by RGBA + color_mappings = [ + ("R", fb_var_screeninfo.red), + ("G", fb_var_screeninfo.green), + ("B", fb_var_screeninfo.blue), + ("A", fb_var_screeninfo.transp), + ] + color_fields = {} + for color_code, fb_bitfield in color_mappings: + color_fields[color_code] = ( + int(fb_bitfield.offset), + int(fb_bitfield.length), + int(fb_bitfield.msb_right), + ) + return color_fields + + @classmethod + def convert_fb_raw_buffer_to_image( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + fb: Framebuffer, + ) -> Image.Image: + """Convert raw framebuffer pixels to an image. + + Args: + fb: the relevant Framebuffer object + + Returns: + A PIL Image object + + Documentation: + include/uapi/linux/fb.h: + /* Interpretation of offset for color fields: All offsets are from the right, + * inside a "pixel" value, which is exactly 'bits_per_pixel' wide (means: you + * can use the offset as right argument to <<). A pixel afterwards is a bit + * stream and is written to video memory as that unmodified. + """ + kernel = context.modules[kernel_name] + kernel_layer = context.layers[kernel.layer_name] + + raw_pixels = io.BytesIO(kernel_layer.read(fb.fb_info.screen_base, fb.size)) + bytes_per_pixel = fb.bpp // 8 + image = Image.new("RGBA", (fb.xres_virtual, fb.yres_virtual)) + + # This is not designed to be extremely fast (numpy isn't available), + # but convenient and dynamic for any color field layout. + for y in range(fb.yres_virtual): + for x in range(fb.xres_virtual): + raw_pixel = int.from_bytes(raw_pixels.read(bytes_per_pixel), "little") + pixel = [0, 0, 0, 255] + # The framebuffer is expected to have been correctly constructed, + # especially by parse_fb_pixel_bitfields, to get the needed RGBA mappings. + for i, color_code in enumerate(["R", "G", "B", "A"]): + offset, length, msb_right = fb.color_fields[color_code] + if length == 0: + continue + color_value = (raw_pixel >> offset) & (2**length - 1) + if msb_right: + # Reverse bit order + color_value = int( + "{:0{length}b}".format(color_value, length=length)[::-1], 2 + ) + pixel[i] = color_value + image.putpixel((x, y), tuple(pixel)) + + return image + + @classmethod + def dump_fb( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + open_method: Type[interfaces.plugins.FileHandlerInterface], + fb: Framebuffer, + convert_to_image: bool, + image_format: str = "PNG", + ) -> str: + """Dump a Framebuffer raw buffer to disk. + + Args: + fb: the relevant Framebuffer object + convert_to_image: a boolean specifying if the buffer should be converted to an image + image_format: the target PIL image format (defaults to PNG) + + Returns: + The filename of the dumped buffer. + """ + kernel = context.modules[kernel_name] + kernel_layer = context.layers[kernel.layer_name] + base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" + if convert_to_image: + image = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) + output = io.BytesIO() + image.save(output, image_format) + file_handle = open_method(f"{base_filename}.{image_format.lower()}") + file_handle.write(output.getvalue()) + else: + raw_pixels = kernel_layer.read(fb.fb_info.screen_base, fb.size) + file_handle = open_method(f"{base_filename}.raw") + file_handle.write(raw_pixels) + + file_handle.close() + return file_handle.preferred_filename + + @classmethod + def parse_fb_info( + cls, + fb_info: interfaces.objects.ObjectInterface, + ) -> Framebuffer: + """Parse an fb_info struct + Args: + fb_info: an fb_info kernel object live instance + + Returns: + A Framebuffer object + + Documentation: + https://docs.kernel.org/fb/api.html: + - struct fb_fix_screeninfo stores device independent unchangeable information about the frame buffer device and the current format. + Those information can't be directly modified by applications, but can be changed by the driver when an application modifies the format. + - struct fb_var_screeninfo stores device independent changeable information about a frame buffer device, its current format and video mode, + as well as other miscellaneous parameters. + """ + # NotAvailableValue() messes with the filename output on disk + id = utility.array_to_string(fb_info.fix.id) or "N-A" + color_fields = None + + # 0 = color, 1 = grayscale, >1 = FOURCC + if fb_info.var.grayscale in [0, 1]: + color_fields = cls.parse_fb_pixel_bitfields(fb_info.var) + + # There a lot of tricky pixel formats used by drivers and vendors in include/uapi/linux/videodev2.h. + # As Volatility3 is not a video format converter, it is best to play it safe and let the user parse + # the raw data manually (with ffmpeg for example). + elif fb_info.var.grayscale > 1: + fourcc = linux.LinuxUtilities.convert_fourcc_code(fb_info.var.grayscale) + warn_msg = f"""Framebuffer "{id}" uses a FOURCC pixel format "{fourcc}" that isn't natively supported. +You can try using ffmpeg to decode the raw buffer. Example usage: +"ffmpeg -pix_fmts" to list supported formats, then +"ffmpeg -f rawvideo -video_size {fb_info.var.xres_virtual}x{fb_info.var.yres_virtual} -i .raw -pix_fmt output.png".""" + vollog.warning(warn_msg) + + # Prefer using the virtual resolution, instead of the visible one. + # This prevents missing non-visible data stored in the framebuffer. + fb = Framebuffer( + id, + xres_virtual=fb_info.var.xres_virtual, + yres_virtual=fb_info.var.yres_virtual, + line_length=fb_info.fix.line_length, + bpp=fb_info.var.bits_per_pixel, + size=fb_info.var.yres_virtual * fb_info.fix.line_length, + color_fields=color_fields, + fb_info=fb_info, + ) + + return fb + + def _generator(self): + kernel_name = self.config["kernel"] + kernel = self.context.modules[kernel_name] + + if not kernel.has_symbol("num_registered_fb"): + raise exceptions.SymbolError( + "num_registered_fb", + kernel.symbol_table_name, + "The provided symbol does not exist in the symbol table. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt.", + ) + + num_registered_fb = kernel.object_from_symbol("num_registered_fb") + if num_registered_fb < 1: + vollog.info("No registered framebuffer in the fbdev API.") + return None + + registered_fb = kernel.object_from_symbol("registered_fb") + fb_info_list = utility.array_of_pointers( + registered_fb, + num_registered_fb, + kernel.symbol_table_name + constants.BANG + "fb_info", + self.context, + ) + + for fb_info in fb_info_list: + fb = self.parse_fb_info(fb_info) + file_output = "Disabled" + if self.config["dump"]: + try: + file_output = self.dump_fb( + self.context, kernel_name, self.open, fb, bool(fb.color_fields) + ) + except exceptions.InvalidAddressException as excp: + vollog.error( + f'Layer {excp.layer_name} failed to read address {hex(excp.invalid_address)} when dumping framebuffer "{fb.id}".' + ) + file_output = "Error" + + try: + fb_device_name = utility.pointer_to_string( + fb.fb_info.dev.kobj.name, 256 + ) + except exceptions.InvalidAddressException: + fb_device_name = NotAvailableValue() + + yield ( + 0, + ( + format_hints.Hex(fb.fb_info.screen_base), + fb_device_name, + fb.id, + fb.size, + f"{fb.xres_virtual}x{fb.yres_virtual}", + fb.bpp, + "RUNNING" if fb.fb_info.state == 0 else "SUSPENDED", + str(file_output), + ), + ) + + def run(self): + columns = [ + ("Address", format_hints.Hex), + ("Device", str), + ("ID", str), + ("Size", int), + ("Virtual resolution", str), + ("BPP", int), + ("State", str), + ("Filename", str), + ] + + return TreeGrid( + columns, + self._generator(), + ) From f192437a944f56ec3b8eae186d61f3049e691e80 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 15 Nov 2024 17:11:11 +0100 Subject: [PATCH 014/268] typo --- volatility3/framework/plugins/linux/graphics/fbdev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 4bde6f62f..60e00d033 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -21,7 +21,7 @@ vollog = logging.getLogger(__name__) @dataclass class Framebuffer: - """Framebuffer object internal representation. This is useful to unify an framebuffer with precalculated + """Framebuffer object internal representation. This is useful to unify a framebuffer with precalculated properties and pass it through functions conveniently.""" id: str From 20f15d3591a5e6340ecd2d3628406c01aef71924 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 2 Dec 2024 11:34:49 +0100 Subject: [PATCH 015/268] modular physical_layer access --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e5a074a49..2d77b8562 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2511,6 +2511,10 @@ class scatterlist(objects.StructType): Returns: An iterator of bytes """ - physical_layer = self._context.layers["memory_layer"] + # Either "physical" is layer-1 because this is a module layer, either "physical" is the current layer + physical_layer_name = self._context.layers[self.vol.layer_name].config.get( + "memory_layer", self.vol.layer_name + ) + physical_layer = self._context.layers[physical_layer_name] for sg in self.for_each_sg(): yield from physical_layer.read(sg.dma_address, sg._sg_dma_len()) From 1cde9ae06ee855e084fe5631eb37f2ffa979ef5c Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 09:46:28 +0000 Subject: [PATCH 016/268] Slightly modify volshell.rst --- doc/source/volshell.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 3c4f4ce5d..c95456dda 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -36,7 +36,7 @@ operating system mode for volshell, and the current layer available for use. (primary) >>> -Volshell itself in essentially a plugin, but an interactive one. As such, most values are accessed through `self` +Volshell itself is essentially a plugin, but an interactive one. As such, most values are accessed through `self` although there is also a `context` object whenever a context must be provided. The prompt for the tool will indicate the name of the current layer (which can be accessed as `self.current_layer` @@ -92,7 +92,7 @@ It can also be provided with an object and will interpret the data for each in t 0x2e8 : UniqueProcessId symbol_table_name1!pointer 4 ... -These values can be accessed directory as attributes +These values can be accessed directly as attributes :: @@ -180,7 +180,7 @@ used: layer = cc(mynewlayer.MyNewLayer, on_top_of = 'primary', other_parameter = 'important') with open('output.dmp', 'wb') as fp: - for i in range(0, 1073741824, 0x1000): + for i in range(0, 0x4000000, 0x1000): data = layer.read(i, 0x1000, pad = True) fp.write(data) From fdd49d0921a8ca22ea388573dec33ba93a1ef485 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 10:11:26 +0000 Subject: [PATCH 017/268] Slightly modify documentation --- doc/source/glossary.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 66dabfafe..c4a93f908 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -145,9 +145,9 @@ Struct, Structure Symbol This is used in many different contexts, as a short term for many things. Within Volatility, a symbol is a - construct that usually encompasses a specific type :ref:`type` at a specific :ref:`offset`, + construct that usually encompasses a specific :ref:`type` at a specific :ref:`offset`, representing a particular instance of that type within the memory of a compiled and running program. An example - would be the location in memory of a list of active tcp endpoints maintained by the networking stack + would be the location in memory of a list of active TCP endpoints maintained by the networking stack within an operating system. T From 4bccf116292e6269f0ecc306b8dba0973af55697 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:31:16 +0000 Subject: [PATCH 018/268] Remove redundant part of if statement Also reorder imports. --- volatility3/cli/volshell/generic.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 82c470e1a..534546dcd 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -11,11 +11,6 @@ import sys from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union from urllib import parse, request -from volatility3.cli import text_renderer, volshell -from volatility3.framework import exceptions, interfaces, objects, plugins, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.layers import intel, physical, resources, scanners - try: import capstone @@ -23,6 +18,11 @@ try: except ImportError: has_capstone = False +from volatility3.cli import text_renderer, volshell +from volatility3.framework import exceptions, interfaces, objects, plugins, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import intel, physical, resources, scanners + class Volshell(interfaces.plugins.PluginInterface): """Shell environment to directly interact with a memory image.""" @@ -553,12 +553,11 @@ class Volshell(interfaces.plugins.PluginInterface): if argname in kwargs: del kwargs[argname] - for keyword in kwargs: - val = kwargs[keyword] + for keyword, val in kwargs.items(): if not isinstance( val, interfaces.configuration.BasicTypes ) and not isinstance(val, list): - if not isinstance(val, list) or all( + if all( isinstance(x, interfaces.configuration.BasicTypes) for x in val ): raise TypeError( From faa6cab797da8719305f49e1e824448a159509eb Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:16:42 +0000 Subject: [PATCH 019/268] Remove redundant part of if statement Also reorder imports. --- volatility3/cli/volshell/generic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 534546dcd..b1a61fcff 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -555,8 +555,8 @@ class Volshell(interfaces.plugins.PluginInterface): for keyword, val in kwargs.items(): if not isinstance( - val, interfaces.configuration.BasicTypes - ) and not isinstance(val, list): + val, (interfaces.configuration.BasicTypes, list) + ): if all( isinstance(x, interfaces.configuration.BasicTypes) for x in val ): From 0b2f4fdeb772ccb097aaf310ff3169ae37279f13 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:26:39 +0000 Subject: [PATCH 020/268] Remove redundant part of if statement Also reorder imports. --- volatility3/cli/volshell/generic.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index b1a61fcff..08132608b 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -554,12 +554,8 @@ class Volshell(interfaces.plugins.PluginInterface): del kwargs[argname] for keyword, val in kwargs.items(): - if not isinstance( - val, (interfaces.configuration.BasicTypes, list) - ): - if all( - isinstance(x, interfaces.configuration.BasicTypes) for x in val - ): + if not isinstance(val, (interfaces.configuration.BasicTypes, list)): + if all(isinstance(x, interfaces.configuration.BasicTypes) for x in val): raise TypeError( "Configurable values must be simple types (int, bool, str, bytes)" ) From 39cb75accae827436d8d923592f1d62b34cf9ede Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:02:48 +0000 Subject: [PATCH 021/268] Slightly modify documentation Include regex_scan, new functionality of volshell. --- doc/source/volshell.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index c95456dda..73b000763 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -187,8 +187,22 @@ used: As this demonstrates, all of the python is accessible, as are the volshell built in functions (such as `cc` which creates a constructable, like a layer or a symbol table). +User Convenience +---------------- + +There are functions available that make often-done tasks easiers, and generally provide a shell-like experience. These can be listed using `help()` which, as already mentioned, is mentioned when volshell starts. + Loading files -------------- +^^^^^^^^^^^^^ Files can be loaded as physical layers using the `load_file` or `lf` command, which takes a filename or a URI. This will be added to `context.layers` and can be accessed by the name returned by `lf`. + +Regex +^^^^^ + +It is easy to scan for some bytes or a pattern using `regex_scan` or `rx`. + +An optional size can be given for the displayed results as with the other fuctions (db, dw, dd, dq, etc). + +You can of course specify a different layer name as well. From fc33fd912787d0420cd9c4c5effbfba0ea89a933 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:14:59 +0000 Subject: [PATCH 022/268] Slightly modify documentation Include regex_scan, new functionality of volshell. --- doc/source/volshell.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 73b000763..b46647dc3 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -190,7 +190,7 @@ creates a constructable, like a layer or a symbol table). User Convenience ---------------- -There are functions available that make often-done tasks easiers, and generally provide a shell-like experience. These can be listed using `help()` which, as already mentioned, is mentioned when volshell starts. +There are functions available that make often-done tasks easier, and generally provide a shell-like experience. These can be listed using `help()` which, as already mentioned, is advertised when volshell starts. Loading files ^^^^^^^^^^^^^ From 3d260f3829e5f7bd4611db726358169498bb6ae8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:42:20 +0000 Subject: [PATCH 023/268] Slightly modify documentation Include regex_scan, new functionality of volshell. --- doc/source/volshell.rst | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index b46647dc3..47ea2e905 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -203,6 +203,45 @@ Regex It is easy to scan for some bytes or a pattern using `regex_scan` or `rx`. +:: + + (layer_name) >>> rx(rb"(Linux version|Darwin Kernel Version) [0-9]+\.[0-9]+\.[0-9]+") + 0x880001400070 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0x880001400080 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0x880001400090 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0x8800014000a0 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0x8800014000b0 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0x8800014000c0 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0x8800014000d0 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0x8800014000e0 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0x880001769027 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0x880001769037 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0x880001769047 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0x880001769057 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0x880001769067 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0x880001769077 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0x880001769087 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0x880001769097 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0xffff81400070 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0xffff81400080 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0xffff81400090 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0xffff814000a0 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0xffff814000b0 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0xffff814000c0 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0xffff814000d0 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0xffff814000e0 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0xffff81769027 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0xffff81769037 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0xffff81769047 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0xffff81769057 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0xffff81769067 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0xffff81769077 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0xffff81769087 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0xffff81769097 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + An optional size can be given for the displayed results as with the other fuctions (db, dw, dd, dq, etc). -You can of course specify a different layer name as well. +You can, of course, specify a different layer name as well. From 58a9c3d6dae0759e0dbf590d53e14d7553f30a28 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 12 Dec 2024 07:20:30 +0000 Subject: [PATCH 024/268] Slightly modify documentation Include regex_scan, new functionality of volshell. Add Intermediate Symbol File (ISF) to glossary. --- doc/source/glossary.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index c4a93f908..a9460b1a2 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -23,7 +23,7 @@ Alignment .. _Array: Array - This represents a list of items, which can be access by an index, which is zero-based (meaning the first + This represents a list of items, which can be accessed by an index, which is zero-based (meaning the first element has index 0). Items in arrays are almost always the same size (it is not a generic list, as in python) even if they are :ref:`pointers` to different sized objects. @@ -43,7 +43,14 @@ Dereference .. _Domain: Domain - This the grouping for input values for a mapping or mathematical function. + The set of input values for a mapping or mathematical function. + +I +- +.. _Intermediate Symbol File (ISF): + +Intermediate Symbol File (ISF) + They contain kernel structures and specific offsets formatted as JSON. For macOS and Linux analysis, the kernel needs to be added as an ISF file to the volatility 3 symbols directory. For Windows, the required ISF file can often be generated from PDB files automatically downloaded from Microsoft servers, and therefore does not require manual intervention. M - @@ -55,7 +62,7 @@ Map, mapping attempts to use mathematical functional notation where possible. Within volatility a mapping is most often used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). For further information, please see - `Function (mathematics) in wikipedia https://en.wikipedia.org/wiki/Function_(mathematics)` + `https://en.wikipedia.org/wiki/Function_(mathematics)`. .. _Member: @@ -69,7 +76,7 @@ O .. _Object: Object - This has a specific meaning within computer programming (as in Object Oriented Programming), but within the world + This has a specific meaning within computer programming (as in object-oriented programming), but within the world of Volatility it is used to refer to a type that has been associated with a chunk of data, or a specific instance of a type. See also :ref:`Type`. From c45beb3ebe7feaec42567eb8ef3dd665e15db3ae Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:12:19 +0000 Subject: [PATCH 025/268] Automagic: Fixes #1417 --- volatility3/framework/automagic/symbol_cache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index e38771f79..2c9883c7d 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -299,6 +299,13 @@ class SqliteCache(CacheManagerInterface): This also updates remote locations based on a cache timeout. """ + if progress_callback is None: + + def dummy_progress(*args, **kargs) -> None: + return None + + progress_callback = dummy_progress + on_disk_locations = set( [ filename From f0f3bb65581e433cc7b44c2e877a1e95c927e17e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:31:41 +0000 Subject: [PATCH 026/268] Core: Start to fix up the typing in ModuleCollection Fixes #1418 --- volatility3/framework/contexts/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 6961d9328..1a55656b3 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -11,7 +11,7 @@ without them interfering with each other. import functools import hashlib import logging -from typing import Callable, Iterable, List, Optional, Set, Tuple, Union +from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, interfaces, symbols, exceptions from volatility3.framework.objects import templates @@ -386,10 +386,9 @@ class ModuleCollection(interfaces.context.ModuleContainer): """Class to contain a collection of SizedModules and reason about their contents.""" - def __init__( - self, modules: Optional[List[interfaces.context.ModuleInterface]] = None - ) -> None: + def __init__(self, modules: Optional[List[SizedModule]] = None) -> None: self._prefix_count = {} + self._modules: Dict[str, SizedModule] = {} super().__init__(modules) def deduplicate(self) -> "ModuleCollection": @@ -402,9 +401,9 @@ class ModuleCollection(interfaces.context.ModuleContainer): new_modules = [] seen: Set[str] = set() for mod in self._modules: - if mod.hash not in seen or mod.size == 0: + if self._modules[mod].hash not in seen or self._modules[mod].size == 0: new_modules.append(mod) - seen.add(mod.hash) # type: ignore # FIXME: mypy #5107 + seen.add(self._modules[mod].hash) return ModuleCollection(new_modules) def free_module_name(self, prefix: str = "module") -> str: From a0b169cf6b1f0a5d8ced886ff855e7ad7dd791c0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:34:19 +0000 Subject: [PATCH 027/268] This PR does not strictly change any interfaces, just the inner workings of a function. --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 55ef19e4b..2ea034176 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 12 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From 1b8f831fda1fc0d47eaf81144dec81354dca8490 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:37:39 +0000 Subject: [PATCH 028/268] Core: Also fix up the interface to match the concrete classes --- volatility3/framework/interfaces/context.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 8b5e816e8..e85429732 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -295,6 +295,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): def has_enumeration(self, name: str) -> bool: """Determines whether an enumeration is present in the module's symbol table.""" + @property def symbols(self) -> List: """Lists the symbols contained in the symbol table for this module""" From d7f678879d8982b1222e6f5676caed4b0e5a9f70 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Wed, 18 Dec 2024 15:17:26 +0800 Subject: [PATCH 029/268] Minor improvements for `mypy` --- pyproject.toml | 3 ++- volatility3/cli/__init__.py | 6 ++--- volatility3/cli/text_filter.py | 2 +- volatility3/cli/volshell/generic.py | 10 ++++---- volatility3/cli/volshell/linux.py | 6 ++--- volatility3/cli/volshell/mac.py | 6 ++--- volatility3/cli/volshell/windows.py | 6 ++--- volatility3/framework/__init__.py | 7 +++--- volatility3/framework/automagic/stacker.py | 4 +++- .../framework/automagic/symbol_cache.py | 9 +++++++ .../framework/configuration/requirements.py | 24 +++++++++---------- volatility3/framework/contexts/__init__.py | 2 +- volatility3/framework/interfaces/automagic.py | 2 +- .../framework/interfaces/configuration.py | 7 +++--- volatility3/framework/interfaces/context.py | 14 +++++++++-- volatility3/framework/interfaces/layers.py | 2 +- volatility3/framework/interfaces/objects.py | 1 + volatility3/framework/interfaces/renderers.py | 4 ++-- volatility3/framework/interfaces/symbols.py | 1 + .../framework/layers/scanners/__init__.py | 2 +- volatility3/framework/objects/__init__.py | 6 ++--- volatility3/framework/plugins/linux/pslist.py | 6 +++-- volatility3/framework/plugins/mac/pslist.py | 6 +++-- volatility3/framework/plugins/timeliner.py | 4 +++- .../framework/plugins/windows/modules.py | 4 ++-- .../framework/plugins/windows/pedump.py | 2 +- .../framework/plugins/windows/poolscanner.py | 2 +- .../framework/plugins/windows/pslist.py | 6 ++--- .../framework/plugins/windows/psscan.py | 2 +- .../plugins/windows/registry/printkey.py | 10 ++++---- .../plugins/windows/scheduled_tasks.py | 1 - volatility3/framework/renderers/__init__.py | 2 +- volatility3/framework/symbols/__init__.py | 8 +++---- .../framework/symbols/generic/__init__.py | 6 ++--- volatility3/framework/symbols/intermed.py | 4 ++-- .../symbols/linux/extensions/__init__.py | 2 +- volatility3/framework/symbols/mac/__init__.py | 4 ++-- .../symbols/mac/extensions/__init__.py | 2 +- volatility3/framework/symbols/metadata.py | 2 +- .../symbols/windows/extensions/__init__.py | 4 +++- .../symbols/windows/extensions/pool.py | 2 +- .../framework/symbols/windows/pdbconv.py | 4 +++- .../framework/symbols/windows/pdbutil.py | 14 +++++------ 43 files changed, 127 insertions(+), 94 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7035f7a15..cc09922e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dev = [ "jsonschema>=4.23.0,<5", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", + "types-jsonschema>=4.23.0,<5", ] test = [ @@ -68,7 +69,7 @@ include = ["volatility3*"] mypy_path = "./stubs" show_traceback = true -[tool.mypy.overrides] +[[tool.mypy.overrides]] ignore_missing_imports = true [tool.ruff] diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index da046de57..6172a17f3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,7 +19,7 @@ import os import sys import tempfile import traceback -from typing import Any, Dict, List, Tuple, Type, Union +from typing import Any, Dict, List, Optional, Tuple, Type, Union from urllib import parse, request try: @@ -64,7 +64,7 @@ class PrintedProgress: def __init__(self): self._max_message_len = 0 - def __call__(self, progress: Union[int, float], description: str = None): + def __call__(self, progress: Union[int, float], description: Optional[str] = None): """A simple function for providing text-based feedback. .. warning:: Only for development use. @@ -81,7 +81,7 @@ class PrintedProgress: class MuteProgress(PrintedProgress): """A dummy progress handler that produces no output when called.""" - def __call__(self, progress: Union[int, float], description: str = None): + def __call__(self, progress: Union[int, float], description: Optional[str] = None): pass diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 955d647f5..6bd6878a5 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -74,7 +74,7 @@ class ColumnFilter: """Identifies whether an item is found in the appropriate column""" try: if self.regex: - return re.search(self.pattern, f"{item}") + return bool(re.search(self.pattern, f"{item}")) return self.pattern in f"{item}" except OSError: return False diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 93a75ca19..12f5499f6 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -240,7 +240,7 @@ class Volshell(interfaces.plugins.PluginInterface): return None return self.context.modules[self.current_kernel_name] - def change_layer(self, layer_name: str = None): + def change_layer(self, layer_name: Optional[str] = None): """Changes the current default layer""" if not layer_name: layer_name = self.current_layer @@ -250,7 +250,7 @@ class Volshell(interfaces.plugins.PluginInterface): self.__current_layer = layer_name sys.ps1 = f"({self.current_layer}) >>> " - def change_symbol_table(self, symbol_table_name: str = None): + def change_symbol_table(self, symbol_table_name: Optional[str] = None): """Changes the current_symbol_table""" if not symbol_table_name: print("No symbol table provided, not changing current symbol table") @@ -262,7 +262,7 @@ class Volshell(interfaces.plugins.PluginInterface): self.__current_symbol_table = symbol_table_name print(f"Current Symbol Table: {self.current_symbol_table}") - def change_kernel(self, kernel_name: str = None): + def change_kernel(self, kernel_name: Optional[str] = None): if not kernel_name: print("No kernel module name provided, not changing current kernel") if kernel_name not in self.context.modules: @@ -347,7 +347,7 @@ class Volshell(interfaces.plugins.PluginInterface): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if not isinstance( @@ -479,7 +479,7 @@ class Volshell(interfaces.plugins.PluginInterface): if treegrid is not None: self.render_treegrid(treegrid) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: print("No symbol table provided") diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index c5e555ec7..41b86f78b 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -61,7 +61,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -69,7 +69,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 2b32ad677..0ed35eb27 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -63,7 +63,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -71,7 +71,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 5c2190c02..303d4d5c3 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -60,7 +60,7 @@ class Volshell(generic.Volshell): object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -68,7 +68,7 @@ class Volshell(generic.Volshell): object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index c9a2c92ea..754939460 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -12,7 +12,7 @@ import inspect import logging import os import traceback -from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar +from typing import Any, Dict, Generator, List, Optional, Tuple, Type, TypeVar from volatility3.framework import constants, interfaces @@ -58,7 +58,7 @@ class NonInheritable: self.default_value = value self.cls = cls - def __get__(self, obj: Any, get_type: Type = None) -> Any: + def __get__(self, obj: Any, get_type: Type = Optional[None]) -> Any: if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) @@ -185,8 +185,7 @@ def _zipwalk(path: str): zip_results[os.path.join(path, os.path.dirname(file.filename))] = ( dirlist ) - for value in zip_results: - yield value, zip_results[value] + yield from zip_results.items() def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: diff --git a/volatility3/framework/automagic/stacker.py b/volatility3/framework/automagic/stacker.py index c251d3c46..596864264 100644 --- a/volatility3/framework/automagic/stacker.py +++ b/volatility3/framework/automagic/stacker.py @@ -166,7 +166,9 @@ class LayerStacker(interfaces.automagic.AutomagicInterface): cls, context: interfaces.context.ContextInterface, initial_layer: str, - stack_set: List[Type[interfaces.automagic.StackerLayerInterface]] = None, + stack_set: Optional[ + List[Type[interfaces.automagic.StackerLayerInterface]] + ] = None, progress_callback: constants.ProgressCallback = None, ): """Stacks as many possible layers on top of the initial layer as can be done. diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 9fad506ae..065eb6d43 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -104,9 +104,11 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): for subclazz in framework.class_subclasses(IdentifierProcessor): self._classifiers[subclazz.operating_system] = subclazz + @abstractmethod def add_identifier(self, location: str, operating_system: str, identifier: str): """Adds an identifier to the store""" + @abstractmethod def find_location( self, identifier: bytes, operating_system: Optional[str] ) -> Optional[str]: @@ -120,15 +122,18 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): The location of the symbols file that matches the identifier """ + @abstractmethod def get_local_locations(self) -> Iterable[str]: """Returns a list of all the local locations""" + @abstractmethod def update(self): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. """ + @abstractmethod def get_identifier_dictionary( self, operating_system: Optional[str] = None, local_only: bool = False ) -> Dict[bytes, str]: @@ -142,12 +147,15 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): A dictionary of identifiers mapped to a location """ + @abstractmethod def get_identifier(self, location: str) -> Optional[bytes]: """Returns an identifier based on a specific location or None""" + @abstractmethod def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: """Returns all identifiers for a particular operating system""" + @abstractmethod def get_location_statistics( self, location: str ) -> Optional[Tuple[int, int, int, int]]: @@ -157,6 +165,7 @@ class CacheManagerInterface(interfaces.configuration.VersionableInterface): A tuple of base_types, types, enums, symbols, or None is location not found """ + @abstractmethod def get_hash(self, location: str) -> Optional[str]: """Returns the hash of the JSON from within a location ISF""" diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 0cfaf5693..812b8ec59 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -11,7 +11,7 @@ expect to be in the context (such as particular layers or symboltables). import abc import logging import os -from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type +from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, Type from urllib import parse, request from volatility3.framework import constants, interfaces @@ -314,11 +314,11 @@ class TranslationLayerRequirement( def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: interfaces.configuration.ConfigSimpleType = None, optional: bool = False, - oses: List = None, - architectures: List = None, + oses: Optional[List] = None, + architectures: Optional[List[str]] = None, ) -> None: """Constructs a Translation Layer Requirement. @@ -526,18 +526,18 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): description: Optional[str] = None, default: bool = False, optional: bool = False, - component: Type[interfaces.configuration.VersionableInterface] = None, + component: Optional[Type[interfaces.configuration.VersionableInterface]] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: if version is None: raise TypeError("Version cannot be None") + if component is None: + raise TypeError("Component cannot be None") if description is None: description = f"Version {'.'.join(str(x) for x in version)} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) - if component is None: - raise TypeError("Component cannot be None") self._component: Type[interfaces.configuration.VersionableInterface] = component self._version = version @@ -546,7 +546,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): context: interfaces.context.ContextInterface, config_path: str, accumulator: Optional[ - List[interfaces.configuration.VersionableInterface] + Set[interfaces.configuration.VersionableInterface] ] = None, ) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type @@ -580,7 +580,7 @@ class VersionRequirement(interfaces.configuration.RequirementInterface): ) if result: - result.update({config_path: self}) + result[config_path] = self return result context.config[interfaces.configuration.path_join(config_path, self.name)] = ( @@ -604,10 +604,10 @@ class PluginRequirement(VersionRequirement): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, optional: bool = False, - plugin: Type[interfaces.plugins.PluginInterface] = None, + plugin: Optional[Type[interfaces.plugins.PluginInterface]] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: super().__init__( @@ -627,7 +627,7 @@ class ModuleRequirement( def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, architectures: Optional[List[str]] = None, optional: bool = False, diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 5111b168a..f527544c0 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -229,7 +229,7 @@ class Module(interfaces.context.ModuleInterface): def object( self, object_type: str, - offset: int = None, + offset: Optional[int] = None, native_layer_name: Optional[str] = None, absolute: bool = False, **kwargs, diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index 0867b1608..4ac386fc0 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -42,7 +42,7 @@ class AutomagicInterface( priority = 10 """An ordering to indicate how soon this automagic should be run""" - exclusion_list = [] + exclusion_list: List[str] = [] """A list of plugin categories (typically operating systems) which the plugin will not operate on""" def __init__( diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index cbbf7e342..2e4f580a7 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -82,7 +82,7 @@ class HierarchicalDict(collections.abc.Mapping): def __init__( self, - initial_dict: Dict[str, "SimpleTypeRequirement"] = None, + initial_dict: Optional[Dict[str, "SimpleTypeRequirement"]] = None, separator: str = CONFIG_SEPARATOR, ) -> None: """ @@ -328,7 +328,7 @@ class RequirementInterface(metaclass=ABCMeta): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: ConfigSimpleType = None, optional: bool = False, ) -> None: @@ -618,7 +618,7 @@ class ConstructableRequirementInterface(RequirementInterface): self, context: "interfaces.context.ContextInterface", config_path: str, - requirement_dict: Dict[str, object] = None, + requirement_dict: Optional[Dict[str, object]] = None, ) -> Optional["interfaces.objects.ObjectInterface"]: """Constructs the class, handing args and the subrequirements as parameters to __init__""" @@ -652,6 +652,7 @@ class ConstructableRequirementInterface(RequirementInterface): class ConfigurableRequirementInterface(RequirementInterface): """Simple Abstract class to provide build_required_config.""" + @abstractmethod def build_configuration( self, context: "interfaces.context.ContextInterface", diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 8b5e816e8..a87e0f1e8 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -85,7 +85,7 @@ class ContextInterface(metaclass=ABCMeta): object_type: Union[str, "interfaces.objects.Template"], layer_name: str, offset: int, - native_layer_name: str = None, + native_layer_name: Optional[str] = None, **arguments, ) -> "interfaces.objects.ObjectInterface": """Object factory, takes a context, symbol, offset and optional @@ -114,6 +114,7 @@ class ContextInterface(metaclass=ABCMeta): """ return copy.deepcopy(self) + @abstractmethod def module( self, module_name: str, @@ -232,7 +233,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): def object( self, object_type: str, - offset: int = None, + offset: Optional[int] = None, native_layer_name: Optional[str] = None, absolute: bool = False, **kwargs, @@ -277,27 +278,35 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): symbol = self.get_symbol(name) return self.offset + symbol.address + @abstractmethod def get_type(self, name: str) -> "interfaces.objects.Template": """Returns a type from the module's symbol table.""" + @abstractmethod def get_symbol(self, name: str) -> "interfaces.symbols.SymbolInterface": """Returns a symbol object from the module's symbol table.""" + @abstractmethod def get_enumeration(self, name: str) -> "interfaces.objects.Template": """Returns an enumeration from the module's symbol table.""" + @abstractmethod def has_type(self, name: str) -> bool: """Determines whether a type is present in the module's symbol table.""" + @abstractmethod def has_symbol(self, name: str) -> bool: """Determines whether a symbol is present in the module's symbol table.""" + @abstractmethod def has_enumeration(self, name: str) -> bool: """Determines whether an enumeration is present in the module's symbol table.""" + @abstractmethod def symbols(self) -> List: """Lists the symbols contained in the symbol table for this module""" + @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: """Returns the symbols within table_name (or this module if not specified) that live at the specified absolute offset provided.""" @@ -343,6 +352,7 @@ class ModuleContainer(collections.abc.Mapping): def __iter__(self): return iter(self._modules) + @abstractmethod def free_module_name(self, prefix: str = "module") -> str: """Returns an unused table name to ensure no collision occurs when inserting a symbol table.""" diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 56798aca9..a90a78667 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -210,7 +210,7 @@ class DataLayerInterface( context: interfaces.context.ContextInterface, scanner: ScannerInterface, progress_callback: constants.ProgressCallback = None, - sections: Iterable[Tuple[int, int]] = None, + sections: Optional[Iterable[Tuple[int, int]]] = None, ) -> Iterable[Any]: """Scans a Translation layer by chunk. diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 51d25510d..23c90b13b 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -374,6 +374,7 @@ class Template: f"{self.__class__.__name__} object has no attribute {attr}" ) + @abc.abstractmethod def __call__( self, context: "interfaces.context.ContextInterface", diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index 7105274c0..e26164ee7 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -183,7 +183,7 @@ class TreeGrid(metaclass=ABCMeta): @abstractmethod def populate( self, - function: VisitorSignature = None, + function: Optional[VisitorSignature] = None, initial_accumulator: Any = None, fail_on_errors: bool = True, ) -> Optional[Exception]: @@ -235,7 +235,7 @@ class TreeGrid(metaclass=ABCMeta): node: Optional[TreeNode], function: VisitorSignature, initial_accumulator: _Type, - sort_key: ColumnSortKey = None, + sort_key: Optional[ColumnSortKey] = None, ) -> None: """Visits all the nodes in a tree, calling function on each one. diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index ead91fb4d..b8712e38d 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -256,6 +256,7 @@ class SymbolSpaceInterface(collections.abc.Mapping): """An interface for the container that holds all the symbol-containing tables for use within a context.""" + @abstractmethod def free_table_name(self, prefix: str = "layer") -> str: """Returns an unused table name to ensure no collision occurs when inserting a symbol table.""" diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index f54b44ff4..be9f1c39a 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -72,7 +72,7 @@ class MultiStringScanner(layers.ScannerInterface): return None for char in value: - trie[char] = trie.get(char, {}) + trie.setdefault(char, {}) trie = trie[char] # Mark the end of a string diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 5846da070..869d4dae6 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -152,7 +152,7 @@ class PrimitiveObject(interfaces.objects.ObjectInterface): type_name: str, object_info: interfaces.objects.ObjectInformation, data_format: DataFormatInfo, - new_value: TUnion[int, float, bool, bytes, str] = None, + new_value: Optional[TUnion[int, float, bool, bytes, str]] = None, **kwargs, ) -> "PrimitiveObject": """Creates the appropriate class and returns it so that the native type @@ -601,7 +601,7 @@ class Enumeration(interfaces.objects.ObjectInterface, int): inverse_choices[v] = k return inverse_choices - def lookup(self, value: int = None) -> str: + def lookup(self, value: Optional[int] = None) -> str: """Looks up an individual value and returns the associated name. If multiple identifiers map to the same value, the first matching identifier will be returned @@ -690,7 +690,7 @@ class Array(interfaces.objects.ObjectInterface, collections.abc.Sequence): type_name: str, object_info: interfaces.objects.ObjectInformation, count: int = 0, - subtype: templates.ObjectTemplate = None, + subtype: Optional[templates.ObjectTemplate] = None, ) -> None: super().__init__(context=context, type_name=type_name, object_info=object_info) self._vol["count"] = count diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index a6d2e6538..82b8dcc67 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import datetime -from typing import Any, Callable, Iterable, List, Tuple +from typing import Any, Callable, Iterable, List, Optional, Tuple from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -58,7 +58,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ] @classmethod - def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[Any], bool]: + def create_pid_filter( + cls, pid_list: Optional[List[int]] = None + ) -> Callable[[Any], bool]: """Constructs a filter function for process IDs. Args: diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 74d044ba9..8c5e5c1a5 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Dict, Iterable, List +from typing import Callable, Dict, Iterable, List, Optional from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -82,7 +82,9 @@ class PsList(interfaces.plugins.PluginInterface): return list_tasks @classmethod - def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[int], bool]: + def create_pid_filter( + cls, pid_list: Optional[List[int]] = None + ) -> Callable[[int], bool]: def filter_func(_): return False diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 4e483922b..0f4064d79 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -54,7 +54,9 @@ class Timeliner(interfaces.plugins.PluginInterface): self.automagics: Optional[List[interfaces.automagic.AutomagicInterface]] = None @classmethod - def get_usable_plugins(cls, selected_list: List[str] = None) -> List[Type]: + def get_usable_plugins( + cls, selected_list: Optional[List[str]] = None + ) -> List[Type]: # Initialize for the run plugin_list = list(framework.class_subclasses(TimeLinerInterface)) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index a3677ad34..85eb474a8 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import Generator, Iterable, List +from typing import Generator, Iterable, List, Optional from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -133,7 +133,7 @@ class Modules(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - pids: List[int] = None, + pids: Optional[List[int]] = None, ) -> Generator[str, None, None]: """Build a cache of possible virtual layers, in priority starting with the primary/kernel layer. Then keep one layer per session by cycling diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 85d5d14d1..678652624 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -96,7 +96,7 @@ class PEDump(interfaces.plugins.PluginInterface): pe_table_name: str, ldr_entry: interfaces.objects.ObjectInterface, open_method: Type[interfaces.plugins.FileHandlerInterface], - layer_name: str = None, + layer_name: Optional[str] = None, prefix: str = "", ) -> Optional[str]: """Extracts the PE file referenced an LDR_DATA_TABLE_ENTRY (DLL, kernel module) instance diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 8c56d202d..efde09638 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -183,7 +183,7 @@ class PoolScanner(plugins.PluginInterface): @staticmethod def builtin_constraints( - symbol_table: str, tags_filter: List[bytes] = None + symbol_table: str, tags_filter: Optional[List[bytes]] = None ) -> List[PoolConstraint]: """Get built-in PoolConstraints given a list of pool tags. diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index f262aeae6..579a235d8 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Iterator, List, Type +from typing import Callable, Iterator, List, Optional, Type from volatility3.framework import renderers, interfaces, layers, exceptions, constants from volatility3.framework.configuration import requirements @@ -114,7 +114,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def create_pid_filter( - cls, pid_list: List[int] = None, exclude: bool = False + cls, pid_list: Optional[List[int]] = None, exclude: bool = False ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process IDs. @@ -171,7 +171,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): @classmethod def create_name_filter( - cls, name_list: List[str] = None, exclude: bool = False + cls, name_list: Optional[List[str]] = None, exclude: bool = False ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process names. diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 86eb47300..cdf344ee6 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -89,7 +89,7 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, layer_name: str, - offset: int = None, + offset: Optional[int] = None, physical: bool = True, exclude: bool = False, ) -> Callable[[interfaces.objects.ObjectInterface], bool]: diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 4fe3f97fb..ed926805b 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import List, Sequence, Iterable, Tuple, Union +from typing import List, Optional, Sequence, Iterable, Tuple, Union from volatility3.framework import objects, renderers, exceptions, interfaces, constants from volatility3.framework.configuration import requirements @@ -51,7 +51,7 @@ class PrintKey(interfaces.plugins.PluginInterface): def key_iterator( cls, hive: RegistryHive, - node_path: Sequence[objects.StructType] = None, + node_path: Optional[Sequence[objects.StructType]] = None, recurse: bool = False, ) -> Iterable[ Tuple[ @@ -121,7 +121,7 @@ class PrintKey(interfaces.plugins.PluginInterface): def _printkey_iterator( self, hive: RegistryHive, - node_path: Sequence[objects.StructType] = None, + node_path: Optional[Sequence[objects.StructType]] = None, recurse: bool = False, ): """Method that wraps the more generic key_iterator, to provide output @@ -242,8 +242,8 @@ class PrintKey(interfaces.plugins.PluginInterface): self, layer_name: str, symbol_table: str, - hive_offsets: List[int] = None, - key: str = None, + hive_offsets: Optional[List[int]] = None, + key: Optional[str] = None, recurse: bool = False, ): for hive in hivelist.HiveList.list_hives( diff --git a/volatility3/framework/plugins/windows/scheduled_tasks.py b/volatility3/framework/plugins/windows/scheduled_tasks.py index 277a0d856..6dd5613c4 100644 --- a/volatility3/framework/plugins/windows/scheduled_tasks.py +++ b/volatility3/framework/plugins/windows/scheduled_tasks.py @@ -270,7 +270,6 @@ class _ScheduledTasksReader(io.BytesIO): return val def read_aligned_bstring_expand_sz(self) -> Optional[str]: - # type: () -> Optional[str] sz = self.read_aligned_u4() if sz is None: return None diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 39ce1135d..112e93751 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -214,7 +214,7 @@ class TreeGrid(interfaces.renderers.TreeGrid): def populate( self, - function: interfaces.renderers.VisitorSignature = None, + function: Optional[interfaces.renderers.VisitorSignature] = None, initial_accumulator: Any = None, fail_on_errors: bool = True, ) -> Optional[Exception]: diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index a8753bd4d..87f2288d7 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -53,10 +53,10 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} - def clear_symbol_cache(self, table_name: str = None) -> None: + def clear_symbol_cache(self, table_name: Optional[str] = None) -> None: """Clears the symbol cache for the specified table name. If no table name is specified, the caches of all symbol tables are cleared.""" - table_list: List[interfaces.symbols.BaseSymbolTableInterface] = list() + table_list: List[interfaces.symbols.BaseSymbolTableInterface] = [] if table_name is None: table_list = list(self._dict.values()) else: @@ -81,7 +81,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): yield table + constants.BANG + symbol_name def get_symbols_by_location( - self, offset: int, size: int = 0, table_name: str = None + self, offset: int, size: int = 0, table_name: Optional[str] = None ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = ( @@ -128,7 +128,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): self, producer: str, validator: Callable[[Optional[Tuple], Optional[datetime.datetime]], bool], - tables: List[str] = None, + tables: Optional[List[str]] = None, ) -> bool: """Verifies the producer metadata and version of tables diff --git a/volatility3/framework/symbols/generic/__init__.py b/volatility3/framework/symbols/generic/__init__.py index 9d6da5aa4..7dd00fa75 100644 --- a/volatility3/framework/symbols/generic/__init__.py +++ b/volatility3/framework/symbols/generic/__init__.py @@ -4,7 +4,7 @@ import random import string -from typing import Union +from typing import Optional, Union from volatility3.framework import objects, interfaces @@ -14,8 +14,8 @@ class GenericIntelProcess(objects.StructType): self, context: interfaces.context.ContextInterface, dtb: Union[int, interfaces.objects.ObjectInterface], - config_prefix: str = None, - preferred_name: str = None, + config_prefix: Optional[str] = None, + preferred_name: Optional[str] = None, ) -> str: """Constructs a new layer based on the process's DirectoryTableBase.""" diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 8a28d732f..5b4aa22b8 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -86,7 +86,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): config_path: str, name: str, isf_url: str, - native_types: interfaces.symbols.NativeTableInterface = None, + native_types: Optional[interfaces.symbols.NativeTableInterface] = None, table_mapping: Optional[Dict[str, str]] = None, validate: bool = True, class_types: Optional[ @@ -319,7 +319,7 @@ class ISFormatTable(interfaces.symbols.SymbolTableInterface, metaclass=ABCMeta): config_path: str, name: str, json_object: Any, - native_types: interfaces.symbols.NativeTableInterface = None, + native_types: Optional[interfaces.symbols.NativeTableInterface] = None, table_mapping: Optional[Dict[str, str]] = None, ) -> None: self._json_object = json_object diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7b025450c..4e2e80bc6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -308,7 +308,7 @@ class module(generic.GenericIntelProcess): class task_struct(generic.GenericIntelProcess): def add_process_layer( - self, config_prefix: str = None, preferred_name: str = None + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: """Constructs a new layer based on the process's DTB. diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index ee6dd10a3..dc54a8371 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Iterator, Any, Iterable, List, Tuple, Set +from typing import Iterator, Any, Iterable, List, Optional, Tuple, Set from volatility3.framework import interfaces, objects, exceptions, constants from volatility3.framework.symbols import intermed @@ -97,7 +97,7 @@ class MacUtilities(interfaces.configuration.VersionableInterface): context: interfaces.context.ContextInterface, handlers: Iterator[Any], target_address, - kernel_module_name: str = None, + kernel_module_name: Optional[str] = None, ): mod_name = "UNKNOWN" symbol_name = "N/A" diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index d2573fb95..cc700f209 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -18,7 +18,7 @@ class proc(generic.GenericIntelProcess): return self.task.dereference().cast("task") def add_process_layer( - self, config_prefix: str = None, preferred_name: str = None + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: """Constructs a new layer based on the process's DTB. diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 7e069e518..ea635f1f1 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -25,7 +25,7 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface): return self._json_data.get("version", "") @property - def version(self) -> Optional[Tuple[int]]: + def version(self) -> Optional[Tuple[int, ...]]: """Returns the version of the ISF file producer""" version = self.version_string if not version: diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 600f3e23f..d63f138b6 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -692,7 +692,9 @@ class EPROCESS(generic.GenericIntelProcess, pool.ExecutiveObject): return True - def add_process_layer(self, config_prefix: str = None, preferred_name: str = None): + def add_process_layer( + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None + ): """Constructs a new layer based on the process's DirectoryTableBase.""" parent_layer = self._context.layers[self.vol.layer_name] diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index 5a7847986..de5c8271b 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -362,7 +362,7 @@ class OBJECT_HEADER(objects.StructType): return True def get_object_type( - self, type_map: Dict[int, str], cookie: int = None + self, type_map: Dict[int, str], cookie: Optional[int] = None ) -> Optional[str]: """Across all Windows versions, the _OBJECT_HEADER embeds details on the type of object (i.e. process, file) but the way its embedded diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index ea2884bb2..248ef7d0c 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -984,7 +984,9 @@ if __name__ == "__main__": def __init__(self): self._max_message_len = 0 - def __call__(self, progress: Union[int, float], description: str = None): + def __call__( + self, progress: Union[int, float], description: Optional[str] = None + ): """A simple function for providing text-based feedback. .. warning:: Only for development use. diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 1a8644fa8..b5e8ca70a 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -36,7 +36,7 @@ class PDBUtility(interfaces.configuration.VersionableInterface): layer_name: str, offset: int, symbol_table_class: str = "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path: str = None, + config_path: Optional[str] = None, progress_callback: constants.ProgressCallback = None, ) -> Optional[str]: """Produces the name of a symbol table loaded from the offset for an MZ header @@ -388,8 +388,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, ) -> str: """Creates symbol table for a module in the specified layer_name. @@ -418,8 +418,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, create_module: bool = False, ) -> Tuple[Optional[str], Optional[str]]: if module_offset is None: @@ -478,8 +478,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, ) -> str: """Creates a module in the specified layer_name based on a pdb name. From 7464a3b883a50ee35a2e3c607d1906dd6d8c1807 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 18 Dec 2024 10:57:33 -0600 Subject: [PATCH 030/268] Windows Extensions: Fix potential AttributeErrors If `self.get_owner()` returns `None`, and the chained call to `is_valid()` is executed, an `AttributeError` will occur. This fixes two instances of this bug by intializing a local variable with the result of the `get_owner()` call, checking for `None`, and returning if that's the case. Also adds type-hints for these methods. --- .../symbols/windows/extensions/network.py | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 9b7573c2e..cfa2a7a30 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -4,7 +4,7 @@ import logging import socket -from typing import Dict, Tuple, List, Union +from typing import Dict, Tuple, List, Union, Optional from volatility3.framework import exceptions from volatility3.framework import objects, interfaces @@ -86,19 +86,29 @@ class _TCP_LISTENER(objects.StructType): except exceptions.InvalidAddressException: return None - def get_owner_pid(self): - if self.get_owner().is_valid(): - if self.get_owner().has_valid_member("UniqueProcessId"): - return self.get_owner().UniqueProcessId + def get_owner_pid(self) -> Optional[int]: + owner = self.get_owner() + + if owner is None: + return None + + if owner.is_valid(): + if owner.has_valid_member("UniqueProcessId"): + return owner.UniqueProcessId return None - def get_owner_procname(self): - if self.get_owner().is_valid(): - if self.get_owner().has_valid_member("ImageFileName"): - return self.get_owner().ImageFileName.cast( + def get_owner_procname(self) -> Optional[str]: + owner = self.get_owner() + + if owner is None: + return None + + if owner.is_valid(): + if owner.has_valid_member("ImageFileName"): + return owner.ImageFileName.cast( "string", - max_length=self.get_owner().ImageFileName.vol.count, + max_length=owner.ImageFileName.vol.count, errors="replace", ) From c865f4892c88d346d010a4d08ed75ecdb10a44c5 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:36:05 +0000 Subject: [PATCH 031/268] Slightly modify documentation --- doc/source/glossary.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index a9460b1a2..33e56883a 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -123,6 +123,11 @@ Page Table possible to use them as a way to map a particular address within a (potentially larger, but sparsely populated) virtual space to a concrete (and usually contiguous) physical space, through the process of :ref:`mapping`. +.. _Plugin: + +Plugin + Plugins are the "functions" of the volatility framework. They carry out algorithms on data stored in layers using objects constructed from symbols. Broadly, plugins take in a number of TranslationLayers (the data, which is a representation of part of an image, in a specified type described by templates) and outputs a TreeGrid. + .. _Pointer: Pointer From 1e2900d587acb1b752a1c8352bcf4fc112a9ec7a Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:39:03 +0000 Subject: [PATCH 032/268] Slightly modify documentation --- doc/source/glossary.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 33e56883a..04f1fb090 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -61,9 +61,7 @@ Map, mapping of the :ref:`Range`). Mappings can be seen as a mathematical function, and therefore volatility 3 attempts to use mathematical functional notation where possible. Within volatility a mapping is most often used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). - For further information, please see - `https://en.wikipedia.org/wiki/Function_(mathematics)`. - + For further information, please see `[Function (mathematics) in Wikipedia](https://en.wikipedia.org/wiki/Function_(mathematics))`. .. _Member: From 7eca407b6d8fc6123486c79121f0d6db2bf7dc8b Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 18 Dec 2024 11:34:25 -0600 Subject: [PATCH 033/268] Windows PEDump: Revert overwritten changes When #1364 was merged, it may not have been rebased onto the changes introduced in #1422, and they ended up overwritten to the old version. This reverts those changes. --- .../framework/plugins/windows/pedump.py | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 85d5d14d1..275775ddb 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -64,30 +64,27 @@ class PEDump(interfaces.plugins.PluginInterface): """ Returns the filename of the dump file or None """ - try: - file_handle = open_method(file_name) + with open_method(file_name) as file_handle: + try: + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base, + layer_name=layer_name, + ) - dos_header = context.object( - pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset=base, - layer_name=layer_name, - ) + for offset, data in dos_header.reconstruct(): + file_handle.seek(offset) + file_handle.write(data) + except ( + OSError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: + vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") + return None - for offset, data in dos_header.reconstruct(): - file_handle.seek(offset) - file_handle.write(data) - except ( - OSError, - exceptions.VolatilityException, - OverflowError, - ValueError, - ) as excp: - vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") - return None - finally: - file_handle.close() - - return file_handle.preferred_filename + return file_handle.preferred_filename @classmethod def dump_ldr_entry( From a9417edd2810e682474966d501786bfb8e3206ec Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:49:56 +0000 Subject: [PATCH 034/268] Slightly modify documentation --- doc/source/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 04f1fb090..d3bc9613a 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -61,7 +61,7 @@ Map, mapping of the :ref:`Range`). Mappings can be seen as a mathematical function, and therefore volatility 3 attempts to use mathematical functional notation where possible. Within volatility a mapping is most often used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). - For further information, please see `[Function (mathematics) in Wikipedia](https://en.wikipedia.org/wiki/Function_(mathematics))`. + For further information, please see `Function (mathematics) in Wikipedia_`. .. _Member: From 3c26955e34d68150e5d009557a66581fd7188bb4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 19 Dec 2024 12:02:16 +1100 Subject: [PATCH 035/268] xen_layer: fix potential uninitialized variable issue #1434 --- volatility3/framework/layers/xen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index e7aa0ccec..c0a5e1a7d 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -54,6 +54,7 @@ class XenCoreDumpLayer(elf.Elf64Layer): segments = [] self._segment_headers = [] + segment_names = None for sindex in range(ehdr.e_shnum): shdr = self.context.object( From c82d432b10258136ff0777dfec1fbf5844316132 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Thu, 19 Dec 2024 12:16:55 +0800 Subject: [PATCH 036/268] Typing fix --- volatility3/framework/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 754939460..a1925faef 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,7 +5,6 @@ # Check the python version to ensure it's suitable import glob import sys -from volatility3.framework import check_python_version as check_python_version import zipfile import importlib import inspect @@ -58,7 +57,7 @@ class NonInheritable: self.default_value = value self.cls = cls - def __get__(self, obj: Any, get_type: Type = Optional[None]) -> Any: + def __get__(self, obj: Any, get_type: Optional[Type] = None) -> Any: if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) From 95e103d9ea6c556e93872f35cc1a83d8453ab94f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:35:34 +0000 Subject: [PATCH 037/268] Remove commented out import --- volatility3/framework/plugins/windows/shimcachemem.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index 59f33510d..b8e9b5bd7 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -17,8 +17,6 @@ from volatility3.framework.symbols.windows.extensions import pe, shimcache from volatility3.plugins import timeliner from volatility3.plugins.windows import modules, pslist, vadinfo -# from volatility3.plugins.windows import pslist, vadinfo, modules - vollog = logging.getLogger(__name__) From c56d9334f82f90e90d46e1026e43d5e209cf9466 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:00:27 +0000 Subject: [PATCH 038/268] Tweak comment --- volatility3/framework/plugins/windows/pe_symbols.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 002577241..21e657ab3 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -158,7 +158,7 @@ class PESymbolFinder: class PDBSymbolFinder(PESymbolFinder): """ - PESymbolFinder implementation for PDB modules + PESymbolFinder implementation for PDB modules """ def _do_get_address(self, name: str) -> Optional[int]: @@ -195,7 +195,7 @@ class PDBSymbolFinder(PESymbolFinder): class ExportSymbolFinder(PESymbolFinder): """ - PESymbolFinder implementation for PDB modules + PESymbolFinder implementation for PDB modules """ def _get_name(self, export: pefile.ExportData) -> Optional[str]: @@ -300,7 +300,7 @@ class PESymbols(interfaces.plugins.PluginInterface): base_address: int, ) -> Optional[pefile.PE]: """ - Attempts to pefile object from the bytes of the PE file + Attempts to create a pefile object from the bytes of the PE file Args: pe_table_name: name of the pe types table From 0e0c959cd0f3d4a7d14f5cd19a683609b64808f6 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:21:20 +0000 Subject: [PATCH 039/268] Swap two letters in a typo --- volatility3/framework/plugins/windows/psxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index e3ec216dd..b5ddd2ee5 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -25,7 +25,7 @@ class PsXView(plugins.PluginInterface): identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this plugin's output in a terminal.""" - # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality + # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the functionality # which the original plugin used to do it. # The sessions method is omitted because it begins with the list of processes found by Pslist anyway. From a11131b3d4767ec15619bab8084c4914ad528f75 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 14:59:58 +0000 Subject: [PATCH 040/268] Update how to write a simple plugin --- doc/source/simple-plugin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 39670a62d..84b921114 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -52,7 +52,7 @@ to be able to run properly. Any that are defined as optional need not necessari version = (2, 0, 0))] -This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how +This is a classmethod, so it can be called before the specific plugin object has been instantiated (in order to know how to instantiate the plugin). At the moment these requirements are fairly straightforward: :: From 4fe3db50df1a53d80d79ac100fdfaf577146fa86 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 16:32:30 +0000 Subject: [PATCH 041/268] Reorder requirements by type --- .../framework/plugins/windows/dlllist.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 57f19f620..35b9fb2dc 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -19,7 +19,7 @@ vollog = logging.getLogger(__name__) class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): - """Lists the loaded modules in a particular windows memory image.""" + """Lists the loaded DLLs in a particular windows memory image.""" _required_framework_version = (2, 0, 0) _version = (3, 0, 0) @@ -39,6 +39,9 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.VersionRequirement( name="psscan", component=psscan.PsScan, version=(1, 1, 0) ), + requirements.VersionRequirement( + name="pedump", component=pedump.PEDump, version=(1, 0, 0) + ), requirements.VersionRequirement( name="info", component=info.Info, version=(1, 0, 0) ), @@ -53,16 +56,16 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): description="Process offset in the physical address space", optional=True, ), - requirements.StringRequirement( - name="name", - description="Specify a regular expression to match dll name(s)", - optional=True, - ), requirements.IntRequirement( name="base", description="Specify a base virtual address in process memory", optional=True, ), + requirements.StringRequirement( + name="name", + description="Specify a regular expression to match dll name(s)", + optional=True, + ), requirements.BooleanRequirement( name="ignore-case", description="Specify case insensitivity for the regular expression name matching", @@ -75,9 +78,6 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): default=False, optional=True, ), - requirements.VersionRequirement( - name="pedump", component=pedump.PEDump, version=(1, 0, 0) - ), ] def _generator(self, procs): @@ -90,12 +90,15 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): kuser = info.Info.get_kuser_structure( self.context, kernel.layer_name, kernel.symbol_table_name ) + nt_major_version = int(kuser.NtMajorVersion) nt_minor_version = int(kuser.NtMinorVersion) + # LoadTime only applies to versions higher or equal to Window 7 (6.1 and higher) dll_load_time_field = (nt_major_version > 6) or ( nt_major_version == 6 and nt_minor_version >= 1 ) + for proc in procs: proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() @@ -114,7 +117,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mod_re = re.compile(self.config["name"], flags) except re.error: vollog.debug( - "Error parsing regular expression: %s", self.config["name"] + f"Error parsing regular expression: {self.config["name"]}" ) return None From 59a5e85b504c48a381fb65f8fc2b42a84bf1bc9a Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:36:42 +0000 Subject: [PATCH 042/268] Reorder requirements by type --- volatility3/framework/plugins/windows/dlllist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 35b9fb2dc..65e337dfc 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -117,7 +117,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mod_re = re.compile(self.config["name"], flags) except re.error: vollog.debug( - f"Error parsing regular expression: {self.config["name"]}" + f'Error parsing regular expression: {self.config["name"]}' ) return None @@ -138,7 +138,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if dll_load_time_field: # Versions prior to 6.1 won't have the LoadTime attribute - # and 32bit version shouldn't have the Quadpart according to MSDN + # and 32-bit version shouldn't have the Quadpart according to MSDN try: DllLoadTime = conversion.wintime_to_datetime( entry.LoadTime.QuadPart From 8b35031d0f443184953d80263f2689e4dd0a059f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 00:37:31 +0000 Subject: [PATCH 043/268] Volshell: Bump linux.pslist plugin requirement --- volatility3/cli/volshell/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index c5e555ec7..72193201c 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -20,7 +20,7 @@ class Volshell(generic.Volshell): name="kernel", description="Linux kernel module" ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.IntRequirement( name="pid", description="Process ID", optional=True From 658d40335a15bc40d2ecb5679198d759858492b5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 13:06:14 +1100 Subject: [PATCH 044/268] testcases: Add basic volshell testcases for each OS image --- .github/workflows/test.yaml | 5 ++++ test/test_volatility.py | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dfc42499d..e07673175 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -42,6 +42,11 @@ jobs: - name: Testing... run: | + # VolShell + pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_windows_volshell -v + pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_linux_volshell -v + + # Volatility pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_windows -v pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_linux -v diff --git a/test/test_volatility.py b/test/test_volatility.py index 5bce07481..8ef6d8b70 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -54,13 +54,56 @@ def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]) return runvol(args, volatility, python) +def runvolshell(img, volshell, python, volshellargs=[], globalargs=[]): + args = ( + globalargs + + [ + "--single-location", + img, + "-q", + ] + + volshellargs + ) + + return runvol(args, volshell, python) + + # # TESTS # + +def basic_volshell_test(image, volatility, python): + # Basic VolShell test to verify requirements and ensure VolShell runs without crashing + + # FIXME: When the minimum Python version includes 3.12, replace the following with: + # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... + fd, filename = tempfile.mkstemp(suffix=".txt") + try: + with os.fdopen(fd, "w") as f: + f.write("exit()") + + rc, out, _err = runvolshell( + img=image, + volshell=volatility, + python=python, + volshellargs=["--script", filename], + ) + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + assert rc == 0 + assert out.count(b"\n") >= 4 + + # WINDOWS +def test_windows_volshell(image, volatility, python): + basic_volshell_test(image, volatility, python) + + def test_windows_pslist(image, volatility, python): rc, out, _err = runvol_plugin("windows.pslist.PsList", image, volatility, python) out = out.lower() @@ -332,6 +375,10 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): # LINUX +def test_linux_volshell(image, volatility, python): + basic_volshell_test(image, volatility, python) + + def test_linux_pslist(image, volatility, python): rc, out, _err = runvol_plugin("linux.pslist.PsList", image, volatility, python) @@ -770,6 +817,10 @@ def test_linux_hidden_modules(image, volatility, python): # MAC +def test_mac_volshell(image, volatility, python): + basic_volshell_test(image, volatility, python) + + def test_mac_pslist(image, volatility, python): rc, out, _err = runvol_plugin("mac.pslist.PsList", image, volatility, python) out = out.lower() From 90766f466f581561e17b2cc5e2c7d8eca56c56e6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 13:21:24 +1100 Subject: [PATCH 045/268] testcases: exclude volshell test from the volatility set --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e07673175..ce2722457 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -47,8 +47,8 @@ jobs: pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_linux_volshell -v # Volatility - pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_windows -v - pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k test_linux -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k "test_windows and not test_windows_volshell" -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k "test_linux and not test_linux_volshell" -v - name: Clean up post-test run: | From e6e0754fa523f9c0edb6a6e449bdbf08a16fa712 Mon Sep 17 00:00:00 2001 From: TheMythologist Date: Fri, 20 Dec 2024 10:46:54 +0800 Subject: [PATCH 046/268] Remove mypy overrides section --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cc09922e9..d695a4eac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,9 +69,6 @@ include = ["volatility3*"] mypy_path = "./stubs" show_traceback = true -[[tool.mypy.overrides]] -ignore_missing_imports = true - [tool.ruff] line-length = 88 target-version = "py38" From c317a45eb56a5da26ee50e2edc0d2e8c6821d262 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 14:31:40 +1100 Subject: [PATCH 047/268] testcases: Add missing operating system argument --- test/test_volatility.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 8ef6d8b70..f8a32fef3 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -73,7 +73,7 @@ def runvolshell(img, volshell, python, volshellargs=[], globalargs=[]): # -def basic_volshell_test(image, volatility, python): +def basic_volshell_test(image, volatility, python, globalargs): # Basic VolShell test to verify requirements and ensure VolShell runs without crashing # FIXME: When the minimum Python version includes 3.12, replace the following with: @@ -88,6 +88,7 @@ def basic_volshell_test(image, volatility, python): volshell=volatility, python=python, volshellargs=["--script", filename], + globalargs=globalargs, ) finally: with contextlib.suppress(FileNotFoundError): @@ -101,7 +102,7 @@ def basic_volshell_test(image, volatility, python): def test_windows_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python) + basic_volshell_test(image, volatility, python, globalargs=["-w"]) def test_windows_pslist(image, volatility, python): @@ -376,7 +377,7 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): def test_linux_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python) + basic_volshell_test(image, volatility, python, globalargs=["-l"]) def test_linux_pslist(image, volatility, python): @@ -818,7 +819,7 @@ def test_linux_hidden_modules(image, volatility, python): def test_mac_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python) + basic_volshell_test(image, volatility, python, globalargs=["-m"]) def test_mac_pslist(image, volatility, python): From d546c983f3012f3060aca2b2721fedcc48321370 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 14:33:36 +1100 Subject: [PATCH 048/268] testcases: Fix runvol* default list arguments --- test/test_volatility.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index f8a32fef3..47ef9769f 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -39,7 +39,9 @@ def runvol(args, volatility, python): return p.returncode, stdout, stderr -def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]): +def runvol_plugin(plugin, img, volatility, python, pluginargs=None, globalargs=None): + pluginargs = pluginargs or [] + globalargs = globalargs or [] args = ( globalargs + [ @@ -54,7 +56,9 @@ def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]) return runvol(args, volatility, python) -def runvolshell(img, volshell, python, volshellargs=[], globalargs=[]): +def runvolshell(img, volshell, python, volshellargs=None, globalargs=None): + volshellargs = volshellargs or [] + globalargs = globalargs or [] args = ( globalargs + [ From ef977efe62af8e4cc5baa96e8a20fb713ee7494f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 20 Dec 2024 14:50:35 +1100 Subject: [PATCH 049/268] testcases: Improve volshell basic testcase calling ps() on each of them --- test/test_volatility.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index 47ef9769f..bb7c9a851 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -80,12 +80,18 @@ def runvolshell(img, volshell, python, volshellargs=None, globalargs=None): def basic_volshell_test(image, volatility, python, globalargs): # Basic VolShell test to verify requirements and ensure VolShell runs without crashing + volshell_commands = [ + "print(ps())", + "exit()", + ] + # FIXME: When the minimum Python version includes 3.12, replace the following with: # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... fd, filename = tempfile.mkstemp(suffix=".txt") try: + volshell_script = "\n".join(volshell_commands) with os.fdopen(fd, "w") as f: - f.write("exit()") + f.write(volshell_script) rc, out, _err = runvolshell( img=image, @@ -101,12 +107,15 @@ def basic_volshell_test(image, volatility, python, globalargs): assert rc == 0 assert out.count(b"\n") >= 4 + return out + # WINDOWS def test_windows_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python, globalargs=["-w"]) + out = basic_volshell_test(image, volatility, python, globalargs=["-w"]) + assert out.count(b" 40 def test_windows_pslist(image, volatility, python): @@ -381,7 +390,8 @@ def test_windows_vadyarascan_yara_string(image, volatility, python): def test_linux_volshell(image, volatility, python): - basic_volshell_test(image, volatility, python, globalargs=["-l"]) + out = basic_volshell_test(image, volatility, python, globalargs=["-l"]) + assert out.count(b" 100 def test_linux_pslist(image, volatility, python): From 89564cca66a14f9a1707a6d3e056ef56e605b4f0 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 20 Dec 2024 05:48:42 +0000 Subject: [PATCH 050/268] Revert one f-string As part of the ruff linting we did recently all vollog messages should have explicitly been reverted back to %-formatting. --- volatility3/framework/plugins/windows/dlllist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 65e337dfc..1dafb6bf5 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -117,7 +117,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): mod_re = re.compile(self.config["name"], flags) except re.error: vollog.debug( - f'Error parsing regular expression: {self.config["name"]}' + "Error parsing regular expression: %s", self.config["name"] ) return None From 15eb80b600e6a2114e232355a04cb1bbf5ce8972 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 20 Dec 2024 13:11:35 +0100 Subject: [PATCH 051/268] fix unbound page variable access --- volatility3/framework/plugins/windows/malfind.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 510719352..14362776b 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -120,8 +120,7 @@ class Malfind(interfaces.plugins.PluginInterface): vadinfo.winnt_protections, ) write_exec = "EXECUTE" in protection_string and "WRITE" in protection_string - dirty_page_check = False - + dirty_page = None if not write_exec: """ # Inspect "PAGE_EXECUTE_READ" VAD pages to detect @@ -135,12 +134,12 @@ class Malfind(interfaces.plugins.PluginInterface): try: # If we have a dirty page in a non writable "EXECUTE" region, it is suspicious. if proc_layer.is_dirty(page): - dirty_page_check = True + dirty_page = page break except exceptions.InvalidAddressException: # Abort as it is likely that other addresses in the same range will also fail. break - if not dirty_page_check: + if dirty_page is None: continue else: continue @@ -152,10 +151,10 @@ class Malfind(interfaces.plugins.PluginInterface): if cls.is_vad_empty(proc_layer, vad): continue - if dirty_page_check: + if dirty_page is not None: # Useful information to investigate the page content with volshell afterwards. vollog.warning( - f"[proc_id {proc_id}] Found suspicious DIRTY + {protection_string} page at {hex(page)}", + f"[proc_id {proc_id}] Found suspicious DIRTY + {protection_string} page at {hex(dirty_page)}", ) data = proc_layer.read(vad.get_start(), 64, pad=True) yield vad, data From 675ef6deea00cf4659bca50e0004dea3be43e8c1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:01:09 +0000 Subject: [PATCH 052/268] Generic: Fix up potential issue with isfinfo Fixes #1436 --- volatility3/framework/plugins/isfinfo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 78e78fb9e..1c2ac52e9 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -132,6 +132,7 @@ class IsfInfo(plugins.PluginInterface): valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning(f"Invalid ISF: {entry}") + continue yield ( 0, ( From f6f4c7b986c8c2adf7a73211f7aeb1375b00f255 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:11:42 +0000 Subject: [PATCH 053/268] Layers: Fix MSF page len on a possibly uninitialized variable Fixes #1441 --- volatility3/framework/layers/msf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 03e144e25..2b4fae963 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -194,7 +194,7 @@ class PdbMSFStream(linear.LinearlyMappedLayer): ) -> None: super().__init__(context, config_path, name, metadata) self._base_layer = self.config["base_layer"] - self._pages = self.config.get("pages", None) + self._pages = self.config.get("pages", []) self._pages_len = len(self._pages) if not self._pages: raise PDBFormatException(name, "Invalid/no pages specified") From eaca7b31bba1c9d8f934ad3e607fced26797abfd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 20 Dec 2024 15:16:17 +0000 Subject: [PATCH 054/268] Layers: Fix vmware layer without a suitable meta Fixes #1442 --- volatility3/framework/layers/vmware.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 622ff0250..39fb21b63 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -57,6 +57,10 @@ class VmwareLayer(segmented.SegmentedLayer): ) meta_layer = self.context.layers.get(self._meta_layer, None) + if meta_layer is None: + raise exceptions.LayerException( + self._meta_layer, "VMware: Meta layer not found" + ) header_size = struct.calcsize(self.header_structure) data = meta_layer.read(0, header_size) magic, unknown, groupCount = struct.unpack(self.header_structure, data) From cc8fbd5b6824231d5f7bfb53a66560cb2e3fe3b1 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:34:52 +0000 Subject: [PATCH 055/268] Tweak a comment --- volatility3/framework/plugins/windows/svclist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index ea73247ce..a5825e1fe 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -41,7 +41,7 @@ class SvcList(svcscan.SvcScan): @classmethod def _get_exe_range(cls, proc) -> Optional[Tuple[int, int]]: """ - Returns a tuple of starting,ending address for + Returns a tuple of starting address and size of the the VAD containing services.exe """ From bed03dfbc7024d97c289b0c26f98d0627e105eee Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 21 Dec 2024 06:15:35 +0000 Subject: [PATCH 056/268] Refactor check of BasicType The intention is either a BasicType or a list where each element is only a BasicType (and not a list). --- volatility3/cli/volshell/generic.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 08132608b..a4b141c2d 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -554,11 +554,15 @@ class Volshell(interfaces.plugins.PluginInterface): del kwargs[argname] for keyword, val in kwargs.items(): - if not isinstance(val, (interfaces.configuration.BasicTypes, list)): - if all(isinstance(x, interfaces.configuration.BasicTypes) for x in val): - raise TypeError( - "Configurable values must be simple types (int, bool, str, bytes)" - ) + BasicType_or_list_of_BasicType = False # excludes list of lists + if isinstance(val, interfaces.configuration.BasicTypes): + BasicType_or_list_of_BasicType = True + if all(isinstance(x, interfaces.configuration.BasicTypes) for x in val): + BasicType_or_list_of_BasicType = True + if not BasicType_or_list_of_BasicType: + raise TypeError( + "Configurable values must be simple types (int, bool, str, bytes)" + ) self.context.config[config_path + "." + keyword] = val constructed = clazz(self.context, config_path, **constructor_args) From 88eb2aa88648180439e4d5311c9bf49ef2ae9363 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 21 Dec 2024 13:22:04 +0100 Subject: [PATCH 057/268] switch pillow to pyproject.toml --- pyproject.toml | 78 ++++++++++++++++++++++++++++++++++++++++++++---- requirements.txt | 29 ------------------ 2 files changed, 72 insertions(+), 35 deletions(-) delete mode 100644 requirements.txt diff --git a/pyproject.toml b/pyproject.toml index 2e1636a43..c8b2971e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,13 +8,55 @@ authors = [ ] requires-python = ">=3.8.0" license = { text = "VSL" } -dynamic = ["dependencies", "optional-dependencies", "version"] +dynamic = ["version"] + +dependencies = [ + "pefile>=2024.8.26", +] + +[project.optional-dependencies] +full = [ + "yara-python>=4.5.1,<5", + "capstone>=5.0.3,<6", + "pycryptodome>=3.21.0,<4", + "leechcorepyc>=2.19.2,<3; sys_platform != 'darwin'", + # https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst + # 10.0.0 dropped support for Python3.7 + # 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 + "pillow>=10.0.0,<11.0.0", +] + +cloud = [ + "gcsfs>=2024.10.0", + "s3fs>=2024.10.0", +] + +dev = [ + "volatility3[full,cloud]", + "jsonschema>=4.23.0,<5", + "pyinstaller>=6.11.0,<7", + "pyinstaller-hooks-contrib>=2024.9", +] + +test = [ + "volatility3[dev]", + "pytest>=8.3.3,<9", + "capstone>=5.0.3,<6", + "yara-x>=0.10.0,<1", +] + +docs = [ + "volatility3[dev]", + "sphinx>=8.0.0,<7", + "sphinx-autodoc-typehints>=2.5.0,<3", + "sphinx-rtd-theme>=3.0.1,<4", +] [project.urls] -Homepage = "https://github.com/volatilityfoundation/volatility3/" -"Bug Tracker" = "https://github.com/volatilityfoundation/volatility3/issues" -Documentation = "https://volatility3.readthedocs.io/" -"Source Code" = "https://github.com/volatilityfoundation/volatility3" +homepage = "https://github.com/volatilityfoundation/volatility3/" +documentation = "https://volatility3.readthedocs.io/" +repository = "https://github.com/volatilityfoundation/volatility3" +issues = "https://github.com/volatilityfoundation/volatility3/issues" [project.scripts] vol = "volatility3.cli:main" @@ -22,11 +64,35 @@ volshell = "volatility3.cli.volshell:main" [tool.setuptools.dynamic] version = { attr = "volatility3.framework.constants._version.PACKAGE_VERSION" } -dependencies = { file = "requirements-minimal.txt" } [tool.setuptools.packages.find] include = ["volatility3*"] +[tool.mypy] +mypy_path = "./stubs" +show_traceback = true + +[tool.mypy.overrides] +ignore_missing_imports = true + +[tool.ruff] +line-length = 88 +target-version = "py38" + +[tool.ruff.lint] +select = [ + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "G", # flake8-logging-format + "PIE", # flake8-pie + "UP", # pyupgrade +] + +ignore = [ + "E501", # ignore due to conflict with formatter +] + [build-system] requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 21c8f9a76..000000000 --- a/requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# Include the minimal requirements --r requirements-minimal.txt - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 - -# This is required for several plugins that perform malware analysis and disassemble code. -# It can also improve accuracy of Windows 8 and later memory samples. -# FIXME: Version 6.0.0 is incompatible (#1336) so we'll need an adaptor at some point -capstone>=3.0.5,<6.0.0 - -# This is required by plugins that decrypt passwords, password hashes, etc. -pycryptodome - -# This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0; sys_platform != 'darwin' - -# This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage -gcsfs>=2023.1.0 -s3fs>=2023.1.0 - -# This is required by plugins that manipulate pixels and images. -# https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst -# 10.0.0 dropped support for Python3.7 -# 11.0.0 dropped support for Python3.8, which is still supported by Volatility3 -pillow>=10.0.0,<11.0.0 \ No newline at end of file From f6a54c5c48b6ba47a334956cb7994fecfcf14f0c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 21 Dec 2024 13:31:29 +0100 Subject: [PATCH 058/268] ruff fix --- volatility3/framework/plugins/linux/graphics/fbdev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 60e00d033..7144e081a 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -218,7 +218,7 @@ class Fbdev(interfaces.plugins.PluginInterface): fourcc = linux.LinuxUtilities.convert_fourcc_code(fb_info.var.grayscale) warn_msg = f"""Framebuffer "{id}" uses a FOURCC pixel format "{fourcc}" that isn't natively supported. You can try using ffmpeg to decode the raw buffer. Example usage: -"ffmpeg -pix_fmts" to list supported formats, then +"ffmpeg -pix_fmts" to list supported formats, then "ffmpeg -f rawvideo -video_size {fb_info.var.xres_virtual}x{fb_info.var.yres_virtual} -i .raw -pix_fmt output.png".""" vollog.warning(warn_msg) From a84d3611130b1ba42e1e3c317bc6f633adc669ab Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 21 Dec 2024 12:18:23 -0600 Subject: [PATCH 059/268] Add the suspended threads plugin from DEF CON 2024 --- .../plugins/windows/suspended_threads.py | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 volatility3/framework/plugins/windows/suspended_threads.py diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py new file mode 100644 index 000000000..2cc0673d7 --- /dev/null +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -0,0 +1,147 @@ +import logging + +from typing import Dict +from functools import partial + +from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +import volatility3.plugins.windows.pslist as pslist +import volatility3.plugins.windows.threads as threads +import volatility3.plugins.windows.pe_symbols as pe_symbols + +from volatility3.framework.objects import utility + +vollog = logging.getLogger(__name__) + + +class SuspendedThreads(interfaces.plugins.PluginInterface): + """Enumerates suspended threads.""" + + _required_framework_version = (2, 13, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="pslist", component=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="pe_symbols", component=pe_symbols.PESymbols, version=(1, 0, 0) + ), + requirements.VersionRequirement( + name="threads", component=threads.Threads, version=(1, 0, 0) + ), + ] + + def _generator(self): + """ + The goal of this plugin is to report on threads that are suspended + + Legitimate programs can start threads suspended but then will later resume them + + Subsets of malware techniques, such as EDR evasion and process hollowing, + create suspended threads and do not resume them. These are the threads that this + plugin is designed to catch. + + See the whitepaper from our DEF CON 2024 presentation for more details: + + https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + """ + kernel = self.context.modules[self.config["kernel"]] + + vads_cache: Dict[int, pe_symbols.PESymbols.ranges_type] = {} + + proc_modules = None + + # walk the threads of each process checking for suspended threads + for proc in pslist.PsList.list_processes( + context=self.context, + layer_name=kernel.layer_name, + symbol_table=kernel.symbol_table_name, + ): + for thread in threads.Threads.list_threads(kernel, proc): + try: + # we only care if the thread is suspended + if thread.Tcb.SuspendCount == 0: + continue + + # 4 == terminated + if thread.Tcb.State == 4: + continue + + owner_proc = thread.owning_process() + owner_proc_pid = thread.Cid.UniqueProcess + owner_proc_name = utility.array_to_string(owner_proc.ImageFileName) + thread_tid = thread.Cid.UniqueThread + thread_start_addr = thread.StartAddress + thread_win32_addr = thread.Win32StartAddress + except exceptions.InvalidAddressException: + continue + + # Nothing useful to report if a process doesn't have VADs.. Also a sign of smear/terminated + vads = pe_symbols.PESymbols.get_vads_for_process_cache(vads_cache, owner_proc) + if not vads: + continue + + # Only compute this if needed as its expensive and 99.9% of samples + # will not have suspended threads + if not proc_modules: + proc_modules = pe_symbols.PESymbols.get_process_modules( + self.context, kernel.layer_name, kernel.symbol_table_name, None + ) + + path_and_symbol = partial( + pe_symbols.PESymbols.path_and_symbol_for_address, + self.context, + self.config_path, + proc_modules, + ) + + start_file, start_sym = path_and_symbol(vads, thread_start_addr) + win32_file, win32_sym = path_and_symbol(vads, thread_win32_addr) + + # the only false positive found in mass scanning of samples + if start_file and start_file.endswith("\\WorkFoldersShell.dll"): + continue + + if win32_file and win32_file.endswith("\\WorkFoldersShell.dll"): + continue + + yield ( + 0, + ( + owner_proc_name, + owner_proc_pid, + thread_tid, + start_file or renderers.NotAvailableValue(), + start_sym or renderers.NotAvailableValue(), + format_hints.Hex(thread_start_addr), + win32_file or renderers.NotAvailableValue(), + win32_sym or renderers.NotAvailableValue(), + format_hints.Hex(thread_win32_addr), + ), + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("TID", int), + ("StartFile", str), + ("StartSymbol", str), + ("StartAddress", format_hints.Hex), + ("Win32StartFile", str), + ("Win32StartSymbol", str), + ("Win32StartAddress", format_hints.Hex), + ], + self._generator(), + ) + From d31ac276edb29721847f248c13953696c4a98a9c Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 21 Dec 2024 12:22:33 -0600 Subject: [PATCH 060/268] Add the suspended threads plugin from DEF CON 2024 --- volatility3/framework/plugins/windows/suspended_threads.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py index 2cc0673d7..6ddfecca7 100644 --- a/volatility3/framework/plugins/windows/suspended_threads.py +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -86,7 +86,9 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): continue # Nothing useful to report if a process doesn't have VADs.. Also a sign of smear/terminated - vads = pe_symbols.PESymbols.get_vads_for_process_cache(vads_cache, owner_proc) + vads = pe_symbols.PESymbols.get_vads_for_process_cache( + vads_cache, owner_proc + ) if not vads: continue @@ -144,4 +146,3 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): ], self._generator(), ) - From b9fa217d7980042aa7264efbdafee6ce4daa930f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 21 Dec 2024 18:48:12 +0000 Subject: [PATCH 061/268] Add a required framework version Added _required_framework_version and set it to the same value (2, 0, 0) as its plugin requirement of svcscan. Also tweaked one comment. --- volatility3/framework/plugins/windows/svclist.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index a5825e1fe..8a64084c5 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -18,6 +18,7 @@ vollog = logging.getLogger(__name__) class SvcList(svcscan.SvcScan): """Lists services contained with the services.exe doubly linked list of services""" + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) def __init__(self, *args, **kwargs): @@ -41,7 +42,7 @@ class SvcList(svcscan.SvcScan): @classmethod def _get_exe_range(cls, proc) -> Optional[Tuple[int, int]]: """ - Returns a tuple of starting address and size of the + Returns a tuple of starting address and size of the VAD containing services.exe """ From ea273fe878fd5724f1801fd709a805bfa92d7ce0 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 21 Dec 2024 23:23:37 +0000 Subject: [PATCH 062/268] Volshell: Fix up shuffled imports --- volatility3/cli/volshell/generic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 90d73b4ac..2321408fe 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -11,6 +11,11 @@ import sys from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union from urllib import parse, request +from volatility3.cli import text_renderer, volshell +from volatility3.framework import exceptions, interfaces, objects, plugins, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import intel, physical, resources, scanners + try: import capstone @@ -18,11 +23,6 @@ try: except ImportError: has_capstone = False -from volatility3.cli import text_renderer, volshell -from volatility3.framework import exceptions, interfaces, objects, plugins, renderers -from volatility3.framework.configuration import requirements -from volatility3.framework.layers import intel, physical, resources, scanners - class Volshell(interfaces.plugins.PluginInterface): """Shell environment to directly interact with a memory image.""" From 7caf8c572a4629a2a235a974e6dfff112ccabecd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:03:26 +0100 Subject: [PATCH 063/268] PIL import graceful exit --- .../framework/plugins/linux/graphics/fbdev.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 7144e081a..28cae8000 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -4,9 +4,6 @@ import logging import io -# Image manipulation functions are kept in the plugin, -# to prevent a general exit on missing PIL (pillow) dependency. -from PIL import Image from dataclasses import dataclass from typing import Type, List, Dict, Tuple from volatility3.framework import constants, exceptions, interfaces @@ -16,6 +13,15 @@ from volatility3.framework.objects import utility from volatility3.framework.constants import architectures from volatility3.framework.symbols import linux +# Image manipulation functions are kept in the plugin, +# to prevent a general exit on missing PIL (pillow) dependency. +try: + from PIL import Image + + has_pil = True +except ImportError: + has_pil = False + vollog = logging.getLogger(__name__) @@ -101,7 +107,7 @@ class Fbdev(interfaces.plugins.PluginInterface): context: interfaces.context.ContextInterface, kernel_name: str, fb: Framebuffer, - ) -> Image.Image: + ): """Convert raw framebuffer pixels to an image. Args: @@ -238,6 +244,13 @@ You can try using ffmpeg to decode the raw buffer. Example usage: return fb def _generator(self): + + if not has_pil: + vollog.error( + "PIL (pillow) module is required to use this plugin. Please install it manually or through pyproject.toml." + ) + return None + kernel_name = self.config["kernel"] kernel = self.context.modules[kernel_name] From 1fef570022eb0ae13924ed321e5c09a24fe72563 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:15:45 +0100 Subject: [PATCH 064/268] restrict output to PNG, unify file handling --- .../framework/plugins/linux/graphics/fbdev.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 28cae8000..e82827944 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -160,15 +160,13 @@ class Fbdev(interfaces.plugins.PluginInterface): kernel_name: str, open_method: Type[interfaces.plugins.FileHandlerInterface], fb: Framebuffer, - convert_to_image: bool, - image_format: str = "PNG", + convert_to_png_image: bool, ) -> str: - """Dump a Framebuffer raw buffer to disk. + """Dump a Framebuffer buffer to disk. Args: fb: the relevant Framebuffer object convert_to_image: a boolean specifying if the buffer should be converted to an image - image_format: the target PIL image format (defaults to PNG) Returns: The filename of the dumped buffer. @@ -176,19 +174,19 @@ class Fbdev(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_name] kernel_layer = context.layers[kernel.layer_name] base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" - if convert_to_image: - image = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) - output = io.BytesIO() - image.save(output, image_format) - file_handle = open_method(f"{base_filename}.{image_format.lower()}") - file_handle.write(output.getvalue()) + if convert_to_png_image: + image_object = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) + raw_io_output = io.BytesIO() + image_object.save(raw_io_output, "PNG") + final_fb_buffer = raw_io_output.getvalue() + filename = f"{base_filename}.png" else: - raw_pixels = kernel_layer.read(fb.fb_info.screen_base, fb.size) - file_handle = open_method(f"{base_filename}.raw") - file_handle.write(raw_pixels) + final_fb_buffer = kernel_layer.read(fb.fb_info.screen_base, fb.size) + filename = f"{base_filename}.raw" - file_handle.close() - return file_handle.preferred_filename + with open_method(filename) as f: + f.write(final_fb_buffer) + return f.preferred_filename @classmethod def parse_fb_info( From 8d213284e642c545f44502d0fab3f026bc4fa0ff Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 14:23:04 +0100 Subject: [PATCH 065/268] handle NotAvailableValue in filename --- volatility3/framework/plugins/linux/graphics/fbdev.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index e82827944..7f7deee4e 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -173,7 +173,8 @@ class Fbdev(interfaces.plugins.PluginInterface): """ kernel = context.modules[kernel_name] kernel_layer = context.layers[kernel.layer_name] - base_filename = f"{fb.id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" + id = "N-A" if isinstance(fb.id, NotAvailableValue) else fb.id + base_filename = f"{id}_{fb.xres_virtual}x{fb.yres_virtual}_{fb.bpp}bpp" if convert_to_png_image: image_object = cls.convert_fb_raw_buffer_to_image(context, kernel_name, fb) raw_io_output = io.BytesIO() @@ -207,8 +208,7 @@ class Fbdev(interfaces.plugins.PluginInterface): - struct fb_var_screeninfo stores device independent changeable information about a frame buffer device, its current format and video mode, as well as other miscellaneous parameters. """ - # NotAvailableValue() messes with the filename output on disk - id = utility.array_to_string(fb_info.fix.id) or "N-A" + id = utility.array_to_string(fb_info.fix.id) or NotAvailableValue() color_fields = None # 0 = color, 1 = grayscale, >1 = FOURCC From f3d7647433a727a5bb7bc8c91fa3803ad44a6bf4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 15:48:29 +0100 Subject: [PATCH 066/268] unify Tainting parsing capabilities --- .../framework/symbols/linux/__init__.py | 121 ++++++++++++++++++ .../symbols/linux/extensions/__init__.py | 74 ++--------- 2 files changed, 131 insertions(+), 64 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0230a9c48..832b1de9b 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -11,6 +11,7 @@ from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions +from volatility3.framework.constants import linux as linux_constants class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -830,3 +831,123 @@ class PageCache: page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page + + +class Tainting: + """Tainted kernel and modules parsing capabilities. + + Relevant kernel functions: + - modules: module_flags_taint + - kernel: print_tainted + """ + + def __init__( + self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ): + self.kernel = context.modules[kernel_module_name] + + @property + def kernel_taint_flags_list( + self, + ) -> Optional[List[interfaces.objects.ObjectInterface]]: + if self.kernel.has_symbol("taint_flags"): + return list(self.kernel.object_from_symbol("taint_flags")) + return None + + def _module_flags_taint_pre_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if is_module and is_module != taint_flag.module: + continue + + if taints & taint_flag.shift: + taints_string += char + + return taints_string + + def _module_flags_taint_post_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self.kernel_taint_flags_list): + if is_module and is_module != taint_flag.module: + continue + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taints & (1 << i): + taints_string += c_true + elif c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + """Convert the taints value to a 1-1 character mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + s + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self.kernel_taint_flags_list: + return self._module_flags_taint_post_4_10_rc1(taints, is_module) + return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + + def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + """Convert the taints string to a 1-1 descriptor mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for character in self.get_taints_as_plain_string(taints, is_module): + taint_flag = linux_constants.TAINT_FLAGS.get(character) + if not taint_flag: + comprehensive_taints.append(f"") + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) + + return comprehensive_taints diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0ecf731f4..075a83ae8 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -279,76 +279,29 @@ class module(generic.GenericIntelProcess): return None - def _module_flags_taints_pre_4_10_rc1(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on statically defined taints mappings in the framework. - - Returns: - The raw taints string. - """ - taints_string = "" - for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if taint_flag.module and self.taints & taint_flag.shift: - taints_string += char - - return taints_string - - def _module_flags_taints_post_4_10_rc1(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on kernel symbol embedded taints definitions. - - struct taint_flag { - char c_true; /* character printed when tainted */ - char c_false; /* character printed when not tainted */ - bool module; /* also show as a per-module taint flag */ - }; - - Returns: - The raw taints string. - """ - taints_string = "" - for i, taint_flag in enumerate(self.taint_flags_list): - c_true = chr(taint_flag.c_true) - c_false = chr(taint_flag.c_false) - if taint_flag.module and (self.taints & (1 << i)): - taints_string += c_true - elif taint_flag.module and c_false != " ": - taints_string += c_false - - return taints_string - def get_taints_as_plain_string(self) -> str: """Convert the module's taints value to a 1-1 character mapping. + Convenient wrapper around framework's Tainting capabilities. Returns: The raw taints string. - - Documentation: - - module_flags_taint kernel function """ - - if self.taint_flags_list: - return self._module_flags_taints_post_4_10_rc1() - return self._module_flags_taints_pre_4_10_rc1() + return linux.Tainting( + self._context, + linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, + ).get_taints_as_plain_string(self.taints, True) def get_taints_parsed(self) -> List[str]: """Convert the module's taints string to a 1-1 descriptor mapping. + Convenient wrapper around framework's Tainting capabilities. Returns: A comprehensive (user-friendly) taint descriptor list. - - Documentation: - - module_flags_taint kernel function """ - comprehensive_taints = [] - for character in self.get_taints_as_plain_string(): - taint_flag = linux_constants.TAINT_FLAGS.get(character) - if not taint_flag: - comprehensive_taints.append(f"") - elif taint_flag.when_present: - comprehensive_taints.append(taint_flag.desc) - - return comprehensive_taints + return linux.Tainting( + self._context, + linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, + ).get_taints_parsed(self.taints, True) @property def section_symtab(self): @@ -376,13 +329,6 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("Unable to get strtab") - @property - def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: - kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - if kernel.has_symbol("taint_flags"): - return list(kernel.object_from_symbol("taint_flags")) - return None - class task_struct(generic.GenericIntelProcess): def add_process_layer( From 9d4dd010a7eb6212effc43dd8d57f86e127684f2 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:28:40 +0000 Subject: [PATCH 067/268] Reformat how to write a simple plugin --- doc/source/simple-plugin.rst | 93 ++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 84b921114..aa8ec3a7e 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -41,15 +41,24 @@ to be able to run properly. Any that are defined as optional need not necessari @classmethod def get_requirements(cls): - return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), - requirements.ListRequirement(name = 'pid', - element_type = int, - description = "Process IDs to include (all other processes are excluded)", - optional = True), - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (2, 0, 0))] + return [ + requirements.ModuleRequirement( + name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"] + ), + requirements.ListRequirement( + name = 'pid', + element_type = int, + description = "Process IDs to include (all other processes are excluded)", + optional = True + ), + requirements.PluginRequirement( + name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0) + ), + ] This is a classmethod, so it can be called before the specific plugin object has been instantiated (in order to know how @@ -57,8 +66,11 @@ to instantiate the plugin). At the moment these requirements are fairly straigh :: - requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement( + name = 'kernel', + description = 'Windows kernel', + architectures = ["Intel32", "Intel64"] + ), This requirement specifies the need for a particular submodule. Each module requires a :py:class:`TranslationLayer ` and a @@ -85,9 +97,11 @@ not be requested directly from the user. :: - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.TranslationLayerRequirement( + name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"] + ), This requirement indicates that the plugin will operate on a single :py:class:`TranslationLayer `. The name of the @@ -110,8 +124,10 @@ not be requested directly from the user. :: - requirements.SymbolTableRequirement(name = "nt_symbols", - description = "Windows kernel symbols"), + requirements.SymbolTableRequirement( + name = "nt_symbols", + description = "Windows kernel symbols" + ), This requirement specifies the need for a particular :py:class:`SymbolTable ` @@ -127,10 +143,12 @@ not be requested directly from the user. :: - requirements.ListRequirement(name = 'pid', - description = 'Filter on specific process IDs', - element_type = int, - optional = True), + requirements.ListRequirement( + name = 'pid', + description = 'Filter on specific process IDs', + element_type = int, + optional = True + ), The next requirement is a List Requirement, populated by integers. The description will be presented to the user to describe what the value represents. The optional flag indicates that the plugin can function without the ``pid`` value @@ -138,9 +156,11 @@ being defined within the configuration tree at all. :: - requirements.PluginRequirement(name = 'pslist', - plugin = pslist.PsList, - version = (2, 0, 0))] + requirements.PluginRequirement( + name = 'pslist', + plugin = pslist.PsList, + version = (2, 0, 0) + ) This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements on that plugin. The version is specified in terms of Semantic Versioning meaning that, to be compatible, the major @@ -180,16 +200,21 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) kernel = self.context.modules[self.config['kernel']] - return renderers.TreeGrid([("PID", int), - ("Process", str), - ("Base", format_hints.Hex), - ("Size", format_hints.Hex), - ("Name", str), - ("Path", str)], - self._generator(pslist.PsList.list_processes(self.context, - kernel.layer_name, - kernel.symbol_table_name, - filter_func = filter_func))) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Base", format_hints.Hex), + ("Size", format_hints.Hex), + ("Name", str), + ("Path", str), + ], + self._generator( + pslist.PsList.list_processes( + self.context, kernel.layer_name, kernel.symbol_table_name, filter_func = filter_func + ) + ) + ) In this instance, the plugin constructs a filter (using the PsList plugin's *classmethod* for creating filters). It checks the plugin's configuration for the ``pid`` value, and passes it in as a list if it finds it, or None if @@ -281,5 +306,3 @@ such as ``!_UNICODE``) and the parameters to that type. Since the cast value must populate a string typed column, it had to be a Python string (such as being cast to the native type string) and could not have been a special Structure such as ``_UNICODE``. For the format hint columns, the format hint type must be used to ensure the error checking does not fail. - - From df37f0a909255410bd5df31cd4d16bdabb1aa9e8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:34:17 +0000 Subject: [PATCH 068/268] Reformat how to write a simple plugin --- doc/source/simple-plugin.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index aa8ec3a7e..07d9e1467 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -211,7 +211,10 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces. ], self._generator( pslist.PsList.list_processes( - self.context, kernel.layer_name, kernel.symbol_table_name, filter_func = filter_func + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func = filter_func ) ) ) From 0bb09191aae08b2a1b481fef4fbd565e07a7d91f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 23 Dec 2024 21:28:58 +0100 Subject: [PATCH 069/268] file output failure results in UnreadableValue --- .../framework/plugins/linux/graphics/fbdev.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index 7f7deee4e..ab4289cf1 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -8,7 +8,12 @@ from dataclasses import dataclass from typing import Type, List, Dict, Tuple from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements -from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.renderers import ( + format_hints, + TreeGrid, + NotAvailableValue, + UnreadableValue, +) from volatility3.framework.objects import utility from volatility3.framework.constants import architectures from volatility3.framework.symbols import linux @@ -280,11 +285,12 @@ You can try using ffmpeg to decode the raw buffer. Example usage: file_output = self.dump_fb( self.context, kernel_name, self.open, fb, bool(fb.color_fields) ) + file_output = str(file_output) except exceptions.InvalidAddressException as excp: vollog.error( f'Layer {excp.layer_name} failed to read address {hex(excp.invalid_address)} when dumping framebuffer "{fb.id}".' ) - file_output = "Error" + file_output = UnreadableValue() try: fb_device_name = utility.pointer_to_string( @@ -303,7 +309,7 @@ You can try using ffmpeg to decode the raw buffer. Example usage: f"{fb.xres_virtual}x{fb.yres_virtual}", fb.bpp, "RUNNING" if fb.fb_info.state == 0 else "SUSPENDED", - str(file_output), + file_output, ), ) From ea2757c06ce23d9a24e01e2ce7823e4980889ee8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 24 Dec 2024 11:45:23 +0100 Subject: [PATCH 070/268] minor version bump --- volatility3/framework/constants/_version.py | 2 +- volatility3/framework/plugins/linux/graphics/fbdev.py | 3 +++ volatility3/framework/symbols/linux/__init__.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 11edc07d8..9ca2d0a5b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 13 # Number of changes that only add to the interface +VERSION_MINOR = 14 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/plugins/linux/graphics/fbdev.py b/volatility3/framework/plugins/linux/graphics/fbdev.py index ab4289cf1..7b644eccf 100644 --- a/volatility3/framework/plugins/linux/graphics/fbdev.py +++ b/volatility3/framework/plugins/linux/graphics/fbdev.py @@ -60,6 +60,9 @@ class Fbdev(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=architectures.LINUX_ARCHS, ), + requirements.VersionRequirement( + name="linuxutils", component=linux.LinuxUtilities, version=(2, 2, 0) + ), requirements.BooleanRequirement( name="dump", description="Dump framebuffers", diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index ba223f979..5aa27b964 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -76,7 +76,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 1, 1) + _version = (2, 2, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) From 79fe1c50dfcda812cd9d4b307b271ddcc3c26bd9 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 24 Dec 2024 19:42:05 +0000 Subject: [PATCH 071/268] Tweak configuration.py Create a tuple directly and replace random.choice by random.choices. --- volatility3/framework/interfaces/configuration.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index 2e4f580a7..a376fa813 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -53,7 +53,7 @@ ConfigSimpleType = Optional[Union[SimpleTypes, List[SimpleTypes]]] def path_join(*args) -> str: """Joins configuration paths together.""" # If a path element (particularly the first) is empty, then remove it from the list - args = tuple([arg for arg in args if arg]) + args = tuple(arg for arg in args if arg) return CONFIG_SEPARATOR.join(args) @@ -772,8 +772,7 @@ class ConfigurableInterface(metaclass=ABCMeta): str: The newly generated full configuration path """ random_config_dict = "".join( - random.SystemRandom().choice(string.ascii_uppercase + string.digits) - for _ in range(8) + random.SystemRandom().choices(string.ascii_uppercase + string.digits, k=8) ) new_config_path = path_join(base_config_path, random_config_dict) # TODO: Check that the new_config_path is empty, although it's not critical if it's not since the values are merged in From 4c430d2ec464b3e1fdf8ddd9b51fdf4525078814 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Thu, 26 Dec 2024 07:27:45 +0000 Subject: [PATCH 072/268] Use BasicTypes variable This removes a "float", which should be excluded. --- volatility3/framework/interfaces/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index a376fa813..b6f4f889c 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -779,9 +779,9 @@ class ConfigurableInterface(metaclass=ABCMeta): # This should check that each k corresponds to a requirement and each v is of the appropriate type # This would require knowledge of the new configurable itself to verify, and they should do validation in the - # constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a simple type + # constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a basic type for k, v in kwargs.items(): - if not isinstance(v, (int, str, bool, float, bytes)): + if not isinstance(v, BasicTypes): raise TypeError( "Config values passed to make_subconfig can only be simple types" ) From 834b7d0f072984f232dfc7880c0214b40df424fb Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 28 Dec 2024 14:58:12 +0000 Subject: [PATCH 073/268] Make ETHREAD year check dynamic Change upper bound year check of ETHREAD to be a decade from now. Makes consistent with EPROCESS. --- volatility3/framework/symbols/windows/extensions/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d63f138b6..5ec84f95f 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -519,7 +519,8 @@ class ETHREAD(objects.StructType, pool.ExecutiveObject): if not isinstance(ctime, datetime.datetime): return False - if not (1998 < ctime.year < 2030): + current_year = datetime.datetime.now().year + if not (1998 < ctime.year < current_year + 10): return False except exceptions.InvalidAddressException: From 37873f9e1593fad055b0319085694d67a9568f4b Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sat, 28 Dec 2024 16:58:24 +0000 Subject: [PATCH 074/268] Remove superfluous spaces in intermed.py --- volatility3/framework/symbols/intermed.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 5b4aa22b8..6802af7d6 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -101,7 +101,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): Args: context: The volatility context for the symbol table config_path: The configuration path for the symbol table - name: The name for the symbol table (this is used in symbols e.g. table!symbol ) + name: The name for the symbol table (this is used in symbols e.g. table!symbol) isf_url: The URL pointing to the ISF file location native_types: The NativeSymbolTable that contains the native types for this symbol table table_mapping: A dictionary linking names referenced in the file with symbol tables in the context @@ -111,7 +111,7 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): """ # Check there are no obvious errors # Open the file and test the version - self._versions = dict([(x.version, x) for x in class_subclasses(ISFormatTable)]) + self._versions = dict((x.version, x) for x in class_subclasses(ISFormatTable)) with resources.ResourceAccessor().open(isf_url) as fp: reader = codecs.getreader("utf-8") json_object = json.load(reader(fp)) # type: ignore @@ -166,9 +166,9 @@ class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface): format. An interface version such as Major.Minor.Patch means that Major - of the provider must be equal to that of the consumer, and the + of the provider must be equal to that of the consumer, and the provider (the JSON in this instance) must have a greater minor - (indicating that only additive changes have been made) than + (indicating that only additive changes have been made) than the consumer (in this case, the file reader). """ major, minor, patch = (int(x) for x in version.split(".")) From 1bd031b9a86677f6b234e13f93ab2ad9feeb2cc8 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 28 Dec 2024 22:42:25 +0000 Subject: [PATCH 075/268] Prevent infinite loops in device enumeration extensions #1483 --- .../symbols/windows/extensions/__init__.py | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index d63f138b6..6dec08c38 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -405,11 +405,24 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): def get_attached_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the attached device's objects""" - device = self.AttachedDevice.dereference() - while device: - yield device - device = device.AttachedDevice.dereference() + seen = set() + try: + device = self.AttachedDevice.dereference() + except exceptions.InvalidAddressException: + return + + while device: + if device.vol.offset in seen: + break + seen.add(device.vol.offset) + + yield device + + try: + device = device.AttachedDevice.dereference() + except exceptions.InvalidAddressException: + return class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" @@ -421,10 +434,24 @@ class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): def get_devices(self) -> Generator[ObjectInterface, None, None]: """Enumerate the driver's device objects""" - device = self.DeviceObject.dereference() + seen = set() + + try: + device = self.DeviceObject.dereference() + except exceptions.InvalidAddressException: + return + while device: + if device.vol.offset in seen: + return + seen.add(device.vol.offset) + yield device - device = device.NextDevice.dereference() + + try: + device = device.NextDevice.dereference() + except exceptions.InvalidAddressException: + return def is_valid(self) -> bool: """Determine if the object is valid.""" From bf7f1ca91ed88bf482b19bd2558d79aa70bb5c0e Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 28 Dec 2024 22:43:56 +0000 Subject: [PATCH 076/268] Prevent infinite loops in device enumeration extensions #1483 --- volatility3/framework/symbols/windows/extensions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index 6dec08c38..c38d47f73 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -424,6 +424,7 @@ class DEVICE_OBJECT(objects.StructType, pool.ExecutiveObject): except exceptions.InvalidAddressException: return + class DRIVER_OBJECT(objects.StructType, pool.ExecutiveObject): """A class for kernel driver objects.""" From e64af61efa1a37f6d5c91e34d5375219b1544ee3 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:13:18 +0000 Subject: [PATCH 077/268] Do not analyze processes without VADs #1470 --- volatility3/framework/plugins/windows/direct_system_calls.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index b0c162f46..183e4095c 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -433,6 +433,8 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] vads = self.get_vad_maps(proc) + if not vads: + continue # for each valid process, look for malicious syscall invocations for address, vad_path in self._get_rule_hits( From 33855cf920a8ef76d2bfa414779b7cf96c8c8def Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:20:02 +0000 Subject: [PATCH 078/268] Significantly improve the smear/error handling in the netstat plugin --- .../framework/plugins/windows/netstat.py | 157 +++++++++++++----- .../symbols/windows/extensions/network.py | 8 +- 2 files changed, 120 insertions(+), 45 deletions(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index a1521a8c6..3408c0a3a 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -111,8 +111,21 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): The list of indices at which a 1 was found. """ ret = [] + # This value is broken in many samples and was causing essentially infinite loops + # Testing showed that 8192 is the current size across all Windows versions + # We give some leeway in case it increases in later versions, while still keeping it sane + # The problematic samples had values that looked like addresses, so in the billions + if bitmap_size_in_byte > 8192 * 10: + return ret + for idx in range(bitmap_size_in_byte): - current_byte = context.layers[layer_name].read(bitmap_offset + idx, 1)[0] + try: + current_byte = context.layers[layer_name].read(bitmap_offset + idx, 1)[ + 0 + ] + except exceptions.InvalidAddressException: + continue + current_offs = idx * 8 for bit in range(8): if current_byte & (1 << bit) != 0: @@ -154,32 +167,37 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) else: # invalid argument. - return None + return vollog.debug(f"Current Port: {port}") # the given port serves as a shifted index into the port pool lists list_index = port >> 8 truncated_port = port & 0xFF - # constructing port_pool object here so callers don't have to - port_pool = context.object( - net_symbol_table + constants.BANG + "_INET_PORT_POOL", - layer_name=layer_name, - offset=port_pool_addr, - ) + try: + # constructing port_pool object here so callers don't have to + port_pool = context.object( + net_symbol_table + constants.BANG + "_INET_PORT_POOL", + layer_name=layer_name, + offset=port_pool_addr, + ) + # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) + inpa = port_pool.PortAssignments[list_index] - # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) - inpa = port_pool.PortAssignments[list_index] - - # then parse the port assignment list (`_PORT_ASSIGNMENT_LIST`) and grab the correct entry - assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] + # then parse the port assignment list (`_PORT_ASSIGNMENT_LIST`) and grab the correct entry + assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] + except exceptions.InvalidAddressException: + return if not assignment: - return None + return # the value within assignment.Entry is a) masked and b) points inside of the network object # first decode the pointer - netw_inside = cls._decode_pointer(assignment.Entry) + try: + netw_inside = cls._decode_pointer(assignment.Entry) + except exceptions.InvalidAddressException: + return if netw_inside: # if the value is valid, calculate the actual object address by subtracting the offset @@ -188,16 +206,30 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ) yield curr_obj + try: + next_obj_address = cls._decode_pointer(curr_obj.Next) + except exceptions.InvalidAddressException: + return + # if the same port is used on different interfaces multiple objects are created # those can be found by following the pointer within the object's `Next` field until it is empty - while curr_obj.Next: - curr_obj = context.object( - obj_name, - layer_name=layer_name, - offset=cls._decode_pointer(curr_obj.Next) - ptr_offset, - ) + while next_obj_address: + try: + curr_obj = context.object( + obj_name, + layer_name=layer_name, + offset=next_obj_address - ptr_offset, + ) + except exceptions.InvalidAddressException: + return + yield curr_obj + try: + next_obj_address = cls._decode_pointer(curr_obj.Next) + except exceptions.InvalidAddressException: + return + @classmethod def get_tcpip_module( cls, @@ -243,16 +275,25 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): The hash table entries which are _not_ empty """ # we are looking for entries whose values are not their own address + # smear sanity check from mass testing + if ht_length > 4096: + return + for index in range(ht_length): current_addr = ht_offset + index * alignment - current_pointer = context.object( - net_symbol_table + constants.BANG + "pointer", - layer_name=layer_name, - offset=current_addr, - ) + try: + current_pointer = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=current_addr, + ) + except exceptions.InvalidAddressException: + continue + # check if addr of pointer is equal to the value pointed to if current_pointer.vol.offset == current_pointer: continue + yield current_pointer @classmethod @@ -292,11 +333,15 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): tcpip_symbol_table + constants.BANG + "PartitionCount" ).address - part_table_addr = context.object( - net_symbol_table + constants.BANG + "pointer", - layer_name=layer_name, - offset=tcpip_module_offset + part_table_symbol, - ) + try: + part_table_addr = context.object( + net_symbol_table + constants.BANG + "pointer", + layer_name=layer_name, + offset=tcpip_module_offset + part_table_symbol, + ) + except exceptions.InvalidAddressException: + vollog.debug(f"`PartitionTable` not present in memory.") + return # part_table is the actual partition table offset and consists out of a dynamic amount of _PARTITION objects part_table = context.object( @@ -304,10 +349,18 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): layer_name=layer_name, offset=part_table_addr, ) - part_count = int.from_bytes( - context.layers[layer_name].read(tcpip_module_offset + part_count_symbol, 1), - "little", - ) + + try: + part_count = int.from_bytes( + context.layers[layer_name].read( + tcpip_module_offset + part_count_symbol, 1 + ), + "little", + ) + except exceptions.InvalidAddressException: + vollog.debug(f"`PartitionCount` not present in memory.") + return + part_table.Partitions.count = part_count vollog.debug( @@ -316,9 +369,21 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): entry_offset = context.symbol_space.get_type(obj_name).relative_child_offset( "ListEntry" ) - for ctr, partition in enumerate(part_table.Partitions): + + try: + partitions = part_table.Partitions + except exceptions.InvalidAddressException: + vollog.debug("Partitions member not present in memory") + return + + for ctr, partition in enumerate(partitions): vollog.debug(f"Parsing partition {ctr}") - if partition.Endpoints.NumEntries > 0: + try: + num_entries = partition.Endpoints.NumEntries + except exceptions.InvalidAddressException: + continue + + if num_entries > 0: for endpoint_entry in cls.parse_hashtable( context, layer_name, @@ -402,6 +467,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): upp_symbol = context.symbol_space.get_symbol( tcpip_symbol_table + constants.BANG + "UdpPortPool" ).address + upp_addr = context.object( net_symbol_table + constants.BANG + "pointer", layer_name=layer_name, @@ -498,13 +564,16 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # then, towards the UDP and TCP port pools # first, find their addresses - upp_addr, tpp_addr = cls.find_port_pools( - context, - layer_name, - net_symbol_table, - tcpip_symbol_table, - tcpip_module_offset, - ) + try: + upp_addr, tpp_addr = cls.find_port_pools( + context, + layer_name, + net_symbol_table, + tcpip_symbol_table, + tcpip_module_offset, + ) + except (exceptions.SymbolError, exceptions.InvalidAddressException): + vollog.debug("Unable to reconstruct port pools") # create port pool objects at the detected address and parse the port bitmap upp_obj = context.object( diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 478deab6b..e41ac6a05 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -219,7 +219,13 @@ class _TCP_ENDPOINT(_TCP_LISTENER): return None def is_valid(self): - if self.State not in self.State.choices.values(): + # netstat calls this before validating the object itself + try: + state = self.State + except exceptions.InvalidAddressException: + return False + + if state not in state.choices.values(): vollog.debug( f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid tcp state {self.State}" ) From 61d6a92f8f32e4fe81d951fc35fd7f36e80a2146 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:25:17 +0000 Subject: [PATCH 079/268] Significantly improve the smear/error handling in the netstat plugin --- volatility3/framework/plugins/windows/netstat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 3408c0a3a..902be5fc8 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -340,7 +340,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): offset=tcpip_module_offset + part_table_symbol, ) except exceptions.InvalidAddressException: - vollog.debug(f"`PartitionTable` not present in memory.") + vollog.debug("`PartitionTable` not present in memory.") return # part_table is the actual partition table offset and consists out of a dynamic amount of _PARTITION objects @@ -358,7 +358,7 @@ class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): "little", ) except exceptions.InvalidAddressException: - vollog.debug(f"`PartitionCount` not present in memory.") + vollog.debug("`PartitionCount` not present in memory.") return part_table.Partitions.count = part_count From 65f602965b14a76dba1596e35a71ab1762257ea7 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sun, 29 Dec 2024 02:47:48 +0000 Subject: [PATCH 080/268] Address feedback --- volatility3/framework/plugins/windows/suspended_threads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/suspended_threads.py b/volatility3/framework/plugins/windows/suspended_threads.py index 6ddfecca7..cec51ed37 100644 --- a/volatility3/framework/plugins/windows/suspended_threads.py +++ b/volatility3/framework/plugins/windows/suspended_threads.py @@ -1,7 +1,7 @@ import logging from typing import Dict -from functools import partial +import functools from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements @@ -99,7 +99,7 @@ class SuspendedThreads(interfaces.plugins.PluginInterface): self.context, kernel.layer_name, kernel.symbol_table_name, None ) - path_and_symbol = partial( + path_and_symbol = functools.partial( pe_symbols.PESymbols.path_and_symbol_for_address, self.context, self.config_path, From 52a643d5b7f6c57e67acf39eed7c1feb2a0e9dbe Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 29 Dec 2024 20:29:03 +0000 Subject: [PATCH 081/268] Use enumerate for readability --- volatility3/framework/renderers/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 112e93751..093edf8cc 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -83,8 +83,7 @@ class TreeNode(interfaces.renderers.TreeNode): raise TypeError( "Values must be a list of objects made up of simple types and number the same as the columns" ) - for index in range(len(self._treegrid.columns)): - column = self._treegrid.columns[index] + for index, column in enumerate(self._treegrid.columns): val = values[index] if not isinstance(val, (column.type, interfaces.renderers.BaseAbsentValue)): raise TypeError( @@ -413,8 +412,7 @@ class ColumnSortKey(interfaces.renderers.ColumnSortKey): _index = None self._type = None self.ascending = ascending - for i in range(len(treegrid.columns)): - column = treegrid.columns[i] + for i, column in enumerate(treegrid.columns): if column.name.lower() == column_name.lower(): _index = i self._type = column.type From 28ff910d6280edc0dfa21b3b0585a1ab07de9279 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 30 Dec 2024 10:58:38 +0000 Subject: [PATCH 082/268] Use rsplit instead of split Since we want to split rightmost only. --- volatility3/plugins/windows/registry/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/plugins/windows/registry/__init__.py b/volatility3/plugins/windows/registry/__init__.py index 8915cdfad..aeeaa87f2 100644 --- a/volatility3/plugins/windows/registry/__init__.py +++ b/volatility3/plugins/windows/registry/__init__.py @@ -15,5 +15,5 @@ import os import sys # This is necessary to ensure the core plugins are available, whilst still be overridable -parent_module, module_name = ".".join(__name__.split(".")[:-1]), __name__.split(".")[-1] +parent_module, module_name = __name__.rsplit(".", maxsplit=1) __path__ = [os.path.join(x, module_name) for x in sys.modules[parent_module].__path__] From e9d9345cef488067e7035aa485ff11ed665c4414 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 09:37:40 -0600 Subject: [PATCH 083/268] Windows Cachedump: Handle uncaught InvalidAddressException --- volatility3/framework/plugins/windows/cachedump.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index 6e667984a..6c730e6ae 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -8,7 +8,7 @@ from typing import Tuple from Crypto.Cipher import ARC4, AES from Crypto.Hash import HMAC -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions @@ -140,9 +140,14 @@ class Cachedump(interfaces.plugins.PluginInterface): if cache_item.Name == "NL$Control": continue - data = sechive.read(cache_item.Data + 4, cache_item.DataLength) - if data is None: + try: + data = sechive.read(cache_item.Data + 4, cache_item.DataLength) + except exceptions.InvalidAddressException: continue + + if not data: + continue + ( uname_len, domain_len, From 2153b742a1dbde57b369adb34fedc5e05e5eb40c Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 30 Dec 2024 18:06:04 +0000 Subject: [PATCH 084/268] Fix uncheck read() call and remove variable that would not be definied if exception triggers --- .../plugins/windows/skeleton_key_check.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index f5d7e1b3a..b103bc831 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -289,7 +289,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException as excp: vollog.debug( - f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" + f"Invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None, None @@ -431,15 +431,20 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): # we do not want to fail just because the count is not in memory # 16 was the size on samples I tested, so I chose it as the default + count = 16 + if target_address: - count = int.from_bytes( - self.context.layers[proc_layer_name].read( - target_address, 4 - ), - "little", - ) - else: - count = 16 + try: + count = int.from_bytes( + self.context.layers[proc_layer_name].read( + target_address, 4 + ), + "little", + ) + except exceptions.InvalidAddressException: + vollog.debug( + "Unable to read `cCsystems`. Defaulting to 16." + ) found_count = True From a3844e8bc54f9d597d459005830e733bd73d7256 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Mon, 30 Dec 2024 18:08:30 +0000 Subject: [PATCH 085/268] Fix uncheck read() call and remove variable that would not be definied if exception triggers --- volatility3/framework/plugins/windows/skeleton_key_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index b103bc831..6ae07381a 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -282,7 +282,6 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface): for proc in proc_list: try: - proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() return proc, proc_layer_name From 5af5363c461eab6ae1661665429a1794a2a712fb Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 13:50:05 -0600 Subject: [PATCH 086/268] Windows Handles: Work in fixes from @attrc These changes fix bugs encountered during regression testing related to virtual offset validation and string length checks. --- volatility3/framework/plugins/windows/handles.py | 8 ++++++++ .../framework/symbols/windows/extensions/pool.py | 11 ++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 62eceb973..e3845b376 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -226,6 +226,14 @@ class Handles(interfaces.plugins.PluginInterface): masked_offset = offset & layer_object.maximum_address for entry in table: + # This triggered a backtrace in many testing samples + # in the level == 0 path + # The code above this calls `is_valid` on the `offset` + # It is sent but then does not validate `entry` before + # sending it to `_get_item` + if not self.context.layers[virtual].is_valid(entry.vol.offset): + continue + if level > 0: yield from self._make_handle_array(entry, level - 1, depth) depth += 1 diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index de5c8271b..ff65acdeb 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -376,7 +376,16 @@ class OBJECT_HEADER(objects.StructType): try: # vista and earlier have a Type member - self._vol["object_header_object_type"] = self.Type.Name.String + length = self.Type.member("Name").Length + if length == 0 or length > 128: + string = None + else: + string = self.Type.Name.String + if len(string) == 0 or len(string) > 128: + string = None + + self._vol["object_header_object_type"] = string + except AttributeError: # windows 7 and later have a TypeIndex, but windows 10 # further encodes the index value with nt1!ObHeaderCookie From 3eeb10be2916bb7988d296c7a85785ffb5a7f25e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 10:10:03 -0600 Subject: [PATCH 087/268] Windows Registry: Handle uncaught exceptions A number of calls to `get_key` across multiple plugins are not made within a `try/except` block that handles `registry.RegistryFormatException` - the calls are either unprotected or only check for `KeyError`. This adds the required `try/except` blocks, or updates the existing ones as needed. --- volatility3/framework/plugins/windows/amcache.py | 10 +++++----- volatility3/framework/plugins/windows/hashdump.py | 2 +- volatility3/framework/plugins/windows/lsadump.py | 7 +++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/amcache.py b/volatility3/framework/plugins/windows/amcache.py index 1e918d61c..46a742233 100644 --- a/volatility3/framework/plugins/windows/amcache.py +++ b/volatility3/framework/plugins/windows/amcache.py @@ -543,7 +543,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\InventoryDriverBinary") # type: ignore ) ) - except KeyError: + except (KeyError, registry.RegistryFormatException): # Registry key not found pass @@ -554,7 +554,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\Programs") ) # type: ignore } - except KeyError: + except (KeyError, registry.RegistryFormatException): programs = {} try: @@ -564,7 +564,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), key=_entry_sort_key, ) - except KeyError: + except (KeyError, registry.RegistryFormatException): files = [] for program_id, file_entries in itertools.groupby( @@ -593,7 +593,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\InventoryApplication") # type: ignore ) ) - except KeyError: + except (KeyError, registry.RegistryFormatException): programs = {} try: @@ -603,7 +603,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), key=_entry_sort_key, ) - except KeyError: + except (KeyError, registry.RegistryFormatException): files = [] for program_id, file_entries in itertools.groupby( diff --git a/volatility3/framework/plugins/windows/hashdump.py b/volatility3/framework/plugins/windows/hashdump.py index 0c98ab8ca..621b0ae53 100644 --- a/volatility3/framework/plugins/windows/hashdump.py +++ b/volatility3/framework/plugins/windows/hashdump.py @@ -332,7 +332,7 @@ class Hashdump(interfaces.plugins.PluginInterface): try: if hive: result = hive.get_key(key) - except KeyError: + except (KeyError, registry.RegistryFormatException): vollog.info( f"Unable to load the required registry key {hive.get_name()}\\{key} from this memory image" ) diff --git a/volatility3/framework/plugins/windows/lsadump.py b/volatility3/framework/plugins/windows/lsadump.py index da8dee325..f3925f2a2 100644 --- a/volatility3/framework/plugins/windows/lsadump.py +++ b/volatility3/framework/plugins/windows/lsadump.py @@ -8,7 +8,7 @@ from typing import Optional from Crypto.Cipher import ARC4, DES, AES from Crypto.Hash import MD5, SHA256 -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions @@ -81,7 +81,10 @@ class Lsadump(interfaces.plugins.PluginInterface): if not enc_reg_value: return None - obf_lsa_key = sechive.read(enc_reg_value.Data + 4, enc_reg_value.DataLength) + try: + obf_lsa_key = sechive.read(enc_reg_value.Data + 4, enc_reg_value.DataLength) + except exceptions.InvalidAddressException: + return None if not obf_lsa_key: return None From f3294ef5f12a6b036989585b88105fde62634ce2 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 12:58:44 -0600 Subject: [PATCH 088/268] Windows Registry: Handle possible exception in get_node Encountered a `SwappedInvalidAddressException` within the call to `cast` due to an underlying call to `read`. --- volatility3/framework/layers/registry.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index cc364ad50..ee7286e1e 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -140,7 +140,14 @@ class RegistryHive(linear.LinearlyMappedLayer): """Returns the appropriate Node, interpreted from the Cell based on its Signature.""" cell = self.get_cell(cell_offset) - signature = cell.cast("string", max_length=2, encoding="latin-1") + try: + signature = cell.cast("string", max_length=2, encoding="latin-1") + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read cell signature for cell at {cell.vol.offset:x}" + ) + return cell + if signature == "nk": return cell.u.KeyNode elif signature == "sk": From 21077f909f6bda2dec6a3900c7e9ec53268b8213 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Mon, 30 Dec 2024 22:28:47 +0000 Subject: [PATCH 089/268] Sort imports and swap two assignments --- volatility3/cli/text_filter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 6bd6878a5..b6f019da9 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -1,7 +1,8 @@ import logging -from typing import Any, List, Optional -from volatility3.framework import constants, interfaces import re +from typing import Any, List, Optional + +from volatility3.framework import constants, interfaces vollog = logging.getLogger(__name__) @@ -67,8 +68,8 @@ class ColumnFilter: ) -> None: self.column_num = column_num self.pattern = pattern - self.exclude = exclude self.regex = regex + self.exclude = exclude def find(self, item) -> bool: """Identifies whether an item is found in the appropriate column""" From 3288ac971397500f61501b86a99678592cbd4128 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 14:01:40 -0600 Subject: [PATCH 090/268] Windows Handles: Handle possibly invalid memory accesses Any number of member accesses here can raise an `InvalidAddressException`; each is now checked, and `None` returned if any `InvalidAddressException` occurs. --- .../framework/plugins/windows/handles.py | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index e3845b376..85b16d2d4 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -68,7 +68,12 @@ class Handles(interfaces.plugins.PluginInterface): if not self.context.layers[virtual].is_valid(handle_table_entry.Object): return None fast_ref = handle_table_entry.Object.cast("_EX_FAST_REF") - object_header = fast_ref.dereference().cast("_OBJECT_HEADER") + + try: + object_header = fast_ref.dereference().cast("_OBJECT_HEADER") + except exceptions.InvalidAddressException: + return None + object_header.GrantedAccess = handle_table_entry.GrantedAccess except AttributeError: # starting with windows 8 @@ -77,16 +82,26 @@ class Handles(interfaces.plugins.PluginInterface): ) if is_64bit: - if handle_table_entry.ObjectPointerBits == 0: + try: + pointer_bits = handle_table_entry.ObjectPointerBits + except exceptions.InvalidAddressException: return None - offset = handle_table_entry.ObjectPointerBits << 4 + if pointer_bits == 0: + return None + + offset = pointer_bits << 4 else: - if handle_table_entry.InfoTable == 0: + try: + info_table = handle_table_entry.InfoTable + except exceptions.InvalidAddressException: return None - offset = handle_table_entry.InfoTable & ~7 + if info_table == 0: + return None + + offset = info_table & ~7 # print("LowValue: {0:#x} Magic: {1:#x} Offset: {2:#x}".format(handle_table_entry.InfoTable, magic, offset)) object_header = self.context.object( @@ -94,7 +109,10 @@ class Handles(interfaces.plugins.PluginInterface): virtual, offset=offset, ) - object_header.GrantedAccess = handle_table_entry.GrantedAccessBits + try: + object_header.GrantedAccess = handle_table_entry.GrantedAccessBits + except exceptions.InvalidAddressException: + return None object_header.HandleValue = handle_value return object_header From 9a5365e681e971f0e58b23ed42195df09dced6e3 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 16:59:40 -0600 Subject: [PATCH 091/268] Windows Handles: Fix unbound local in exception handler This fixes an unbound local used in a debug message; If the exception is raised during the dereference operation, the `objct` variable may be uninitialized. This uses the offset of `ptr` instead. --- volatility3/framework/plugins/windows/handles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 85b16d2d4..38ccfbfbc 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -178,7 +178,7 @@ class Handles(interfaces.plugins.PluginInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, - f"Cannot access _OBJECT_HEADER Name at {objt.vol.offset:#x}", + f"Cannot access _OBJECT_HEADER Name at {ptr.vol.offset:#x}", ) continue From 263c87611b51f1cb9710c2ec71d1f41eb98e9c77 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 31 Dec 2024 10:43:27 -0600 Subject: [PATCH 092/268] Windows Registry: Handle exceptions in read calls These calls to `.read()` can raise an `InvalidAddressException`. Instead of propagating this exception to the caller, this adds debug logging, and pads the data will null bytes. Also updates the docstring for `decode_data()` to indicate that it can raise `TypeError` and `ValueError`. --- .../symbols/windows/extensions/registry.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 9e2f8df3b..97dd7390d 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -276,7 +276,16 @@ class CM_KEY_VALUE(objects.StructType): return RegValueTypes(self.Type) def decode_data(self) -> Union[int, bytes]: - """Properly decodes the data associated with the value node""" + """ + Properly decodes the data associated with the value node. + + If an InvalidAddressException occurs when reading data from the + underlying RegistryHive layer, the data will be padded with null bytes + of the same length. + + Raises ValueError if the data cannot be read + Raises TypeError if the class was not instantiated on a RegistryHive layer + """ # Determine if the data is stored inline datalen = self.DataLength data = b"" @@ -310,14 +319,26 @@ class CM_KEY_VALUE(objects.StructType): and block_offset < layer.maximum_address ): amount = min(BIG_DATA_MAXLEN, datalen) - data += layer.read( - offset=layer.get_cell(block_offset).vol.offset, length=amount - ) + try: + data += layer.read( + offset=layer.get_cell(block_offset).vol.offset, + length=amount, + ) + except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {amount:x} bytes of data, padding with {amount:x}" + ) datalen -= amount else: # Suspect Data actually points to a Cell, # but the length at the start could be negative so just adding 4 to jump past it - data = layer.read(self.Data + 4, datalen) + try: + data = layer.read(self.Data + 4, datalen) + except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {datalen:x} bytes of data, returning {datalen:x} null bytes" + ) + data = b"\x00" * datalen if self.get_type() == RegValueTypes.REG_DWORD: if len(data) != struct.calcsize(" Date: Tue, 31 Dec 2024 11:11:57 -0600 Subject: [PATCH 093/268] Windows Registry: Update docstrings + exceptions This updates the docstrings on several methods to indicate that they may raise an exception. --- .../symbols/windows/extensions/registry.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 97dd7390d..e53338855 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -159,6 +159,11 @@ class CM_KEY_NODE(objects.StructType): """Extension to allow traversal of registry keys.""" def get_volatile(self) -> bool: + """ + Returns a bool indicating whether or not the key is volatile. + + Raises ValueError if the key was not instantiated on a RegistryHive layer + """ if not isinstance(self._context.layers[self.vol.layer_name], RegistryHive): raise ValueError( "Cannot determine volatility of registry key without an offset in a RegistryHive layer" @@ -166,7 +171,10 @@ class CM_KEY_NODE(objects.StructType): return bool(self.vol.offset & 0x80000000) def get_subkeys(self) -> Iterator["CM_KEY_NODE"]: - """Returns a list of the key nodes.""" + """Returns a list of the key nodes. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") @@ -222,7 +230,10 @@ class CM_KEY_NODE(objects.StructType): yield from self._get_subkeys_recursive(hive, subnode) def get_values(self) -> Iterator["CM_KEY_VALUE"]: - """Returns a list of the Value nodes for a key.""" + """Returns a list of the Value nodes for a key. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") @@ -251,6 +262,11 @@ class CM_KEY_NODE(objects.StructType): return self.Name.cast("string", max_length=namelength, encoding="latin-1") def get_key_path(self) -> str: + """ + Returns the full path to this registry key. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ reg = self._context.layers[self.vol.layer_name] if not isinstance(reg, RegistryHive): raise TypeError("Key was not instantiated on a RegistryHive layer") From fa67f10d3183cd743ca7e44b66d152141a34b2fb Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 31 Dec 2024 11:40:41 -0600 Subject: [PATCH 094/268] Windows Registry: Catch RegistryInvalidIndex refs #1484 This catches uncaught exceptions when casting the cell to a string in `get_node`. --- volatility3/framework/layers/registry.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index cc364ad50..c684ccd40 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -140,7 +140,13 @@ class RegistryHive(linear.LinearlyMappedLayer): """Returns the appropriate Node, interpreted from the Cell based on its Signature.""" cell = self.get_cell(cell_offset) - signature = cell.cast("string", max_length=2, encoding="latin-1") + try: + signature = cell.cast("string", max_length=2, encoding="latin-1") + except (RegistryInvalidIndex, exceptions.InvalidAddressException): + vollog.debug( + f"Failed to get cell signature for cell (0x{cell.vol.offset:x})" + ) + return cell if signature == "nk": return cell.u.KeyNode elif signature == "sk": From c6209800bdc810627f8757881f32e1cf0cfb6f17 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 13:23:02 +0000 Subject: [PATCH 095/268] Core: Fix up issues when resolving merge --- volatility3/framework/constants/_version.py | 2 +- volatility3/framework/interfaces/context.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 02402c5c9..694375538 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 15 # Number of changes that only add to the interface +VERSION_MINOR = 14 # Number of changes that only add to the interface VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 30840a5b9..0b2ae0cc9 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -306,6 +306,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): @abstractmethod def symbols(self) -> List: """Lists the symbols contained in the symbol table for this module""" + raise NotImplementedError("Symbols property has not been implemented.") @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: From 443f7afc9c95c5738cd9eec841ca834860914759 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 13:59:20 +0000 Subject: [PATCH 096/268] Core: Fix code scanning issue concerning equality --- volatility3/framework/contexts/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index a9ec4ac69..e1fb56d94 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -11,6 +11,7 @@ without them interfering with each other. import functools import hashlib import logging +import re from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, interfaces, symbols, exceptions @@ -387,7 +388,6 @@ class ModuleCollection(interfaces.context.ModuleContainer): contents.""" def __init__(self, modules: Optional[List[SizedModule]] = None) -> None: - self._prefix_count = {} self._modules: Dict[str, SizedModule] = {} super().__init__(modules) @@ -408,13 +408,12 @@ class ModuleCollection(interfaces.context.ModuleContainer): def free_module_name(self, prefix: str = "module") -> str: """Returns an unused module name""" - if prefix not in self._prefix_count: - self._prefix_count[prefix] = 1 + existing_names = [name for name in self if re.match(rf"^{prefix}[0-9]*$", name)] + if not existing_names: return prefix - count = self._prefix_count[prefix] + count = len(existing_names) while prefix + str(count) in self: count += 1 - self._prefix_count[prefix] = count return prefix + str(count) @property From 6be039a7e5ddf23f918f073428cbab7e3604f4e7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 14:05:02 +0000 Subject: [PATCH 097/268] Core: Fix black error --- volatility3/framework/interfaces/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 0b2ae0cc9..48c066e96 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -306,7 +306,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): @abstractmethod def symbols(self) -> List: """Lists the symbols contained in the symbol table for this module""" - raise NotImplementedError("Symbols property has not been implemented.") + raise NotImplementedError("Symbols property has not been implemented.") @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: From 9c02f0d12a13fb77db8fb326f6f68dc31fceec1f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 15:22:09 +0000 Subject: [PATCH 098/268] Linux: Fix kmsf f-strings Closes #1496 --- volatility3/framework/plugins/linux/kmsg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d66e3b9ca..c1d09aff8 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return f"{nsec / 1000000000:lu}.{(nsec % 1000000000) / 1000:06lu}" + return f"{nsec / 1000000000}.{(nsec % 1000000000) / 1000:06}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = f"{caller_name}({caller_id & ~0x80000000:u})" + caller = f"{caller_name}({int(caller_id & ~0x80000000)})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: From ac3e76665b7a44b6c5dbc18e633814bd2371ff75 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 15:29:07 +0000 Subject: [PATCH 099/268] Linux: Fix kmsg unguarded read of msg.len --- volatility3/framework/plugins/linux/kmsg.py | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d66e3b9ca..67114c087 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -317,23 +317,27 @@ class Kmsg_3_5_to_3_11(ABCKmsg): while cur_idx < end_idx: msg_offset = log_buf_ptr + cur_idx # type: ignore msg = self.vmlinux.object(object_type=log_struct_name, offset=msg_offset) - if msg.len == 0: - # As per kernel/printk.c: - # A length == 0 for the next message indicates a wrap-around to - # the beginning of the buffer. - cur_idx = 0 - end_idx = log_next_idx - else: - facility, level, timestamp, caller = self.get_prefix(msg) - level_txt = self.get_level_text(level) - facility_txt = self.get_facility_text(facility) + try: + if msg.len == 0: + # As per kernel/printk.c: + # A length == 0 for the next message indicates a wrap-around to + # the beginning of the buffer. + cur_idx = 0 + end_idx = log_next_idx + else: + facility, level, timestamp, caller = self.get_prefix(msg) + level_txt = self.get_level_text(level) + facility_txt = self.get_facility_text(facility) - for line in self.get_log_lines(msg): - yield facility_txt, level_txt, timestamp, caller, line - for line in self.get_dict_lines(msg): - yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_log_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_dict_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line - cur_idx += msg.len + cur_idx += msg.len + except exceptions.InvalidAddressException: + vollog.warning("Kmsg buffer msg length could not be read") + return class Kmsg_3_11_to_5_10(Kmsg_3_5_to_3_11): From c8e67e526a831dcd05b59fa0adeeda8937c4f81a Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 1 Jan 2025 22:33:25 -0600 Subject: [PATCH 100/268] Convert ValueError to TypeError All other methods in this class raise a `TypeError` if the hive was not instantiated on a registry layer; this changes makes this method consistent with the convention used in the others. All `except` blocks checking for `ValueError` have been audited to ensure that this doesn't break exception handling in existing code within the framework. This also includes a minor version bump because: 1. RegistryHives are currently only instantiated one way, which is through the `hivelist` plugin. `hivelist` uses the correct layers when instantiating the hives. 2. Because there is currently a single source for registry hives, and it's unlikely that a hive from that source will ever be created on the wrong layer, it's unlikely that the existing `ValueError` is being raised anywhere within the framework's code. 3. It seems unlikely that consumers of this framework would be instantiating registry hives independent of the `hivelist` plugin, given that they would effectively have to duplicate the `hivelist` code to do so. For these reasons, we're going to do a minor version bump, even though an argument can be made that this warrants a major version bump according to the SemVer rules. This is a one-off and does not indicate any change in the way that we typically update version numbers. --- volatility3/framework/constants/_version.py | 2 +- .../framework/symbols/windows/extensions/registry.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 9ca2d0a5b..2f0c53093 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 14 # Number of changes that only add to the interface +VERSION_MINOR = 15 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index e53338855..c9544a8ba 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -162,12 +162,10 @@ class CM_KEY_NODE(objects.StructType): """ Returns a bool indicating whether or not the key is volatile. - Raises ValueError if the key was not instantiated on a RegistryHive layer + Raises TypeError if the key was not instantiated on a RegistryHive layer """ if not isinstance(self._context.layers[self.vol.layer_name], RegistryHive): - raise ValueError( - "Cannot determine volatility of registry key without an offset in a RegistryHive layer" - ) + raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") return bool(self.vol.offset & 0x80000000) def get_subkeys(self) -> Iterator["CM_KEY_NODE"]: From 97b93abe438bf32b32067bd962a19e07d9917406 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 2 Jan 2025 11:26:21 +0000 Subject: [PATCH 101/268] Linux: Remove unnecessary int cast --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index c1d09aff8..894ca575f 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = f"{caller_name}({int(caller_id & ~0x80000000)})" + caller = f"{caller_name}({caller_id & ~0x80000000})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: From 7278bb244f58f36b346d254512e393cde6f63871 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 14:47:18 +0100 Subject: [PATCH 102/268] move get_flags_list at bottom --- .../framework/symbols/linux/extensions/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 34d0fcba9..9546fcf82 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2628,6 +2628,19 @@ class page(objects.StructType): page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) return page_data + def get_flags_list(self) -> List[str]: + """Returns a list of page flags + + Returns: + List of page flags + """ + flags = [] + for name, value in self.pageflags_enum.items(): + if self.flags & (1 << value) != 0: + flags.append(name) + + return flags + class IDR(objects.StructType): IDR_BITS = 8 From b61ba66223a866a29a119eb17f44fba250ecc01e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 14:51:23 +0100 Subject: [PATCH 103/268] multi-architecture vmemmap_start calculation --- .../symbols/linux/extensions/__init__.py | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9546fcf82..b02f80433 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -15,7 +15,7 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion from volatility3.framework.constants import linux as linux_constants -from volatility3.framework.layers import linear +from volatility3.framework.layers import linear, intel from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf @@ -2525,16 +2525,13 @@ class address_space(objects.StructType): class page(objects.StructType): - @property - @functools.lru_cache + @functools.cached_property def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values Returns: A dictionary with the pageflags enumeration key/values """ - # FIXME: It would be even better to use @functools.cached_property instead, - # however, this requires Python +3.8 try: pageflags_enum = self._context.symbol_space.get_enumeration( self.get_symbol_table_name() + constants.BANG + "pageflags" @@ -2548,24 +2545,12 @@ class page(objects.StructType): return pageflags_enum - def get_flags_list(self) -> List[str]: - """Returns a list of page flags + @functools.cached_property + def _intel_vmemmap_start(self) -> int: + """Determine the start of the struct page array, for Intel systems. Returns: - List of page flags - """ - flags = [] - for name, value in self.pageflags_enum.items(): - if self.flags & (1 << value) != 0: - flags.append(name) - - return flags - - def to_paddr(self) -> int: - """Converts a page's virtual address to its physical address using the current physical memory model. - - Returns: - int: page physical address + int: vmemmap_start address """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] @@ -2605,13 +2590,39 @@ class page(objects.StructType): "Something went wrong, we shouldn't be here" ) - page_type_size = vmlinux.get_type("page").size + return vmemmap_start + + def _intel_to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current Intel memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] pagec = vmlinux_layer.canonicalize(self.vol.offset) - pfn = (pagec - vmemmap_start) // page_type_size + pfn = (pagec - self._intel_vmemmap_start) // vmlinux.get_type("page").size page_paddr = pfn * vmlinux_layer.page_size return page_paddr + def to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current CPU memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + if isinstance(vmlinux_layer, intel.Intel): + page_paddr = self._intel_to_paddr() + else: + raise exceptions.LayerException( + f"Architecture {type(vmlinux_layer)} vmemmap_start calculation isn't currently supported." + ) + + return page_paddr + def get_content(self) -> Union[str, None]: """Returns the page content @@ -2620,7 +2631,10 @@ class page(objects.StructType): """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] - physical_layer = vmlinux.context.layers["memory_layer"] + physical_layer_name = self._context.layers[self.vol.layer_name].config.get( + "memory_layer", self.vol.layer_name + ) + physical_layer = self._context.layers[physical_layer_name] page_paddr = self.to_paddr() if not page_paddr: return None From dda104bd62b9f5f7b9c0208832c6d788c0ebd2ea Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:37:14 +0100 Subject: [PATCH 104/268] move out Tainting capabilities --- .../framework/symbols/linux/__init__.py | 121 ------------------ 1 file changed, 121 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 832b1de9b..0230a9c48 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -11,7 +11,6 @@ from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions -from volatility3.framework.constants import linux as linux_constants class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -831,123 +830,3 @@ class PageCache: page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page - - -class Tainting: - """Tainted kernel and modules parsing capabilities. - - Relevant kernel functions: - - modules: module_flags_taint - - kernel: print_tainted - """ - - def __init__( - self, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - ): - self.kernel = context.modules[kernel_module_name] - - @property - def kernel_taint_flags_list( - self, - ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self.kernel.has_symbol("taint_flags"): - return list(self.kernel.object_from_symbol("taint_flags")) - return None - - def _module_flags_taint_pre_4_10_rc1( - self, taints: int, is_module: bool = False - ) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on statically defined taints mappings in the framework. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - The raw taints string. - """ - taints_string = "" - for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if is_module and is_module != taint_flag.module: - continue - - if taints & taint_flag.shift: - taints_string += char - - return taints_string - - def _module_flags_taint_post_4_10_rc1( - self, taints: int, is_module: bool = False - ) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on kernel symbol embedded taints definitions. - - struct taint_flag { - char c_true; /* character printed when tainted */ - char c_false; /* character printed when not tainted */ - bool module; /* also show as a per-module taint flag */ - }; - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - The raw taints string. - """ - taints_string = "" - for i, taint_flag in enumerate(self.kernel_taint_flags_list): - if is_module and is_module != taint_flag.module: - continue - c_true = chr(taint_flag.c_true) - c_false = chr(taint_flag.c_false) - if taints & (1 << i): - taints_string += c_true - elif c_false != " ": - taints_string += c_false - - return taints_string - - def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: - """Convert the taints value to a 1-1 character mapping. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - s - Returns: - The raw taints string. - - Documentation: - - module_flags_taint kernel function - """ - - if self.kernel_taint_flags_list: - return self._module_flags_taint_post_4_10_rc1(taints, is_module) - return self._module_flags_taint_pre_4_10_rc1(taints, is_module) - - def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: - """Convert the taints string to a 1-1 descriptor mapping. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - A comprehensive (user-friendly) taint descriptor list. - - Documentation: - - module_flags_taint kernel function - """ - comprehensive_taints = [] - for character in self.get_taints_as_plain_string(taints, is_module): - taint_flag = linux_constants.TAINT_FLAGS.get(character) - if not taint_flag: - comprehensive_taints.append(f"") - elif taint_flag.when_present: - comprehensive_taints.append(taint_flag.desc) - - return comprehensive_taints From 2a5f38ebad48e0d729b3b22caac84bd4209f20a2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:39:07 +0100 Subject: [PATCH 105/268] introduce versioned Linux utilities --- .../framework/symbols/linux/utilities/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 volatility3/framework/symbols/linux/utilities/__init__.py diff --git a/volatility3/framework/symbols/linux/utilities/__init__.py b/volatility3/framework/symbols/linux/utilities/__init__.py new file mode 100644 index 000000000..4225d444b --- /dev/null +++ b/volatility3/framework/symbols/linux/utilities/__init__.py @@ -0,0 +1,11 @@ +from volatility3 import framework +from volatility3.framework import interfaces + + +class LinuxUtilityInterface(interfaces.configuration.VersionableInterface): + """Class with multiple useful Linux functions surrounding a specific piece of functionality.""" + + _version = (2, 1, 1) + _required_framework_version = (2, 0, 0) + + framework.require_interface_version(*_required_framework_version) From 8bc62598f4530bdf2fb99aeb725e5b8f3e0d8cd5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:39:57 +0100 Subject: [PATCH 106/268] initial tainting utilities --- .../symbols/linux/utilities/tainting.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 volatility3/framework/symbols/linux/utilities/tainting.py diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py new file mode 100644 index 000000000..e6d75a963 --- /dev/null +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -0,0 +1,130 @@ +from volatility3 import framework +from volatility3.framework import interfaces +from volatility3.framework.symbols.linux.utilities import LinuxUtilityInterface +from volatility3.framework.constants import linux as linux_constants +from typing import List, Optional + + +class Tainting(LinuxUtilityInterface): + """Tainted kernel and modules parsing capabilities. + + Relevant Linux kernel functions: + - modules: module_flags_taint + - kernel: print_tainted + """ + + _version = (1, 0, 0) + _required_framework_version = (2, 14, 0) + + framework.require_interface_version(*_required_framework_version) + + def __init__( + self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ): + self.kernel = context.modules[kernel_module_name] + + @property + def _kernel_taint_flags_list( + self, + ) -> Optional[List[interfaces.objects.ObjectInterface]]: + if self.kernel.has_symbol("taint_flags"): + return list(self.kernel.object_from_symbol("taint_flags")) + return None + + def _module_flags_taint_pre_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if is_module and is_module != taint_flag.module: + continue + + if taints & taint_flag.shift: + taints_string += char + + return taints_string + + def _module_flags_taint_post_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self._kernel_taint_flags_list): + if is_module and is_module != taint_flag.module: + continue + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taints & (1 << i): + taints_string += c_true + elif c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + """Convert the taints value to a 1-1 character mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + s + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self._kernel_taint_flags_list: + return self._module_flags_taint_post_4_10_rc1(taints, is_module) + return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + + def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + """Convert the taints string to a 1-1 descriptor mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for character in self.get_taints_as_plain_string(taints, is_module): + taint_flag = linux_constants.TAINT_FLAGS.get(character) + if not taint_flag: + comprehensive_taints.append(f"") + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) + + return comprehensive_taints From 3105a31964a1a36420281bd995d983a81b161e97 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:40:49 +0100 Subject: [PATCH 107/268] leverage Tainting from separated Linux utilities --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 075a83ae8..ac07d2def 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -19,7 +19,7 @@ from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf - +from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -286,7 +286,7 @@ class module(generic.GenericIntelProcess): Returns: The raw taints string. """ - return linux.Tainting( + return tainting.Tainting( self._context, linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, ).get_taints_as_plain_string(self.taints, True) @@ -298,7 +298,7 @@ class module(generic.GenericIntelProcess): Returns: A comprehensive (user-friendly) taint descriptor list. """ - return linux.Tainting( + return tainting.Tainting( self._context, linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, ).get_taints_parsed(self.taints, True) From 6e4213e321b96dd8f0b35df6c87aa8426698a42c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:41:37 +0100 Subject: [PATCH 108/268] update tainting requirements to new versioned utilities --- volatility3/framework/plugins/linux/modxview.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index d79f5e7a9..c97864a87 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -9,6 +9,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue from volatility3.framework.symbols.linux import extensions from volatility3.framework.constants import architectures +from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -18,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 11, 0) + _required_framework_version = (2, 14, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -28,6 +29,9 @@ class Modxview(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=architectures.LINUX_ARCHS, ), + requirements.VersionRequirement( + name="linux-tainting", component=tainting.Tainting, version=(1, 0, 0) + ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), From 0a3502697cca4dac2ac2f39896ecfaaef507ac9b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:42:16 +0100 Subject: [PATCH 109/268] 2.13.0 -> 2.14.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 11edc07d8..9ca2d0a5b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 13 # Number of changes that only add to the interface +VERSION_MINOR = 14 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 89b8da8c39fe14f699d711c95a8311ec1e21331e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:48:22 +0100 Subject: [PATCH 110/268] make self.kernel private and call parent __init__ --- .../framework/symbols/linux/utilities/tainting.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index e6d75a963..603215961 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -22,15 +22,18 @@ class Tainting(LinuxUtilityInterface): self, context: interfaces.context.ContextInterface, kernel_module_name: str, + *args, + **kwargs, ): - self.kernel = context.modules[kernel_module_name] + super().__init__(*args, **kwargs) + self._kernel = context.modules[kernel_module_name] @property def _kernel_taint_flags_list( self, ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self.kernel.has_symbol("taint_flags"): - return list(self.kernel.object_from_symbol("taint_flags")) + if self._kernel.has_symbol("taint_flags"): + return list(self._kernel.object_from_symbol("taint_flags")) return None def _module_flags_taint_pre_4_10_rc1( From d2bb5c9f31d7f01fe2e343867c0c7c1926b3ac50 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 3 Jan 2025 10:01:37 +1100 Subject: [PATCH 111/268] linux: fix kmsg fstring bug introduced in #1502 --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 894ca575f..30f67b319 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return f"{nsec / 1000000000}.{(nsec % 1000000000) / 1000:06}" + return f"{nsec // 1000000000}.{(nsec % 1000000000) // 1000:06}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info From 4a34b988d1e4cb02e33e555c3e2ed63d808e5028 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 3 Jan 2025 13:21:09 +0100 Subject: [PATCH 112/268] minor readability adjustments --- .../framework/symbols/linux/utilities/tainting.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 603215961..f7f6c83ec 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -51,7 +51,7 @@ class Tainting(LinuxUtilityInterface): """ taints_string = "" for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if is_module and is_module != taint_flag.module: + if is_module and not taint_flag.module: continue if taints & taint_flag.shift: @@ -79,12 +79,12 @@ class Tainting(LinuxUtilityInterface): The raw taints string. """ taints_string = "" - for i, taint_flag in enumerate(self._kernel_taint_flags_list): - if is_module and is_module != taint_flag.module: + for taint_bit, taint_flag in enumerate(self._kernel_taint_flags_list): + if is_module and not taint_flag.module: continue c_true = chr(taint_flag.c_true) c_false = chr(taint_flag.c_false) - if taints & (1 << i): + if taints & (1 << taint_bit): taints_string += c_true elif c_false != " ": taints_string += c_false @@ -97,7 +97,6 @@ class Tainting(LinuxUtilityInterface): Args: taints: The taints value, represented by an integer is_module: Indicates if the taints value is associated with a built-in/LKM module - s Returns: The raw taints string. From 38c5cc168f93a4d1a5cab2a6c9b071cf32e22fc2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 3 Jan 2025 13:23:27 +0100 Subject: [PATCH 113/268] bump framework req to 2.16.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- volatility3/framework/symbols/linux/utilities/tainting.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index c97864a87..042930740 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 14, 0) + _required_framework_version = (2, 16, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index f7f6c83ec..fc2f94109 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -14,7 +14,7 @@ class Tainting(LinuxUtilityInterface): """ _version = (1, 0, 0) - _required_framework_version = (2, 14, 0) + _required_framework_version = (2, 16, 0) framework.require_interface_version(*_required_framework_version) From 32ca62bbb1205b11be0338e741e3046d503153a8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:20:35 +0000 Subject: [PATCH 114/268] Make f-string slightly more readable --- .../framework/plugins/windows/shimcachemem.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index b8e9b5bd7..9d968c30a 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -305,14 +305,14 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf If a number of validity checks are passed, this method will return the `SHIM_CACHE_HEAD` object. Otherwise, `None` is returned. """ - # print("checking RTL_AVL_TABLE at offset %s" % hex(offset)) + # Check RTL_AVL_TABLE at offset rtl_avl_table = context.object( symbol_table + constants.BANG + "_RTL_AVL_TABLE", layer_name, offset ) if not rtl_avl_table.is_valid(mod_page_start, mod_page_end): return None - vollog.debug(f"Candidate RTL_AVL_TABLE found at offset {hex(offset)}") + vollog.debug(f"Candidate RTL_AVL_TABLE found at offset {offset:#x}") ersrc_size = context.symbol_space.get_type( kernel_symbol_table + constants.BANG + "_ERESOURCE" @@ -324,13 +324,13 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf # 0x20 if context.symbol_space.get_type("pointer").size == 8 else 0x10 ) vollog.debug( - f"ERESOURCE size: {hex(ersrc_size)}, ERESOURCE alignment: {hex(ersrc_alignment)}" + f"ERESOURCE size: {ersrc_size:#x}, ERESOURCE alignment: {ersrc_alignment:#x}" ) eresource_rel_off = ersrc_size + ((offset - ersrc_size) % ersrc_alignment) eresource_offset = offset - eresource_rel_off - vollog.debug(f"Constructing ERESOURCE at {hex(eresource_offset)}") + vollog.debug(f"Constructing ERESOURCE at {eresource_offset:#x}") eresource = context.object( kernel_symbol_table + constants.BANG + "_ERESOURCE", layer_name, @@ -408,8 +408,8 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf # iterate over ahcache kernel module's .data section in search of *two* SHIM handles shim_heads = [] - vollog.debug(f"PAGE offset: {hex(mod_page_offset)}") - vollog.debug(f".data offset: {hex(data_sec_offset)}") + vollog.debug(f"PAGE offset: {mod_page_offset:#x}") + vollog.debug(f".data offset: {data_sec_offset:#x}") handle_type = context.symbol_space.get_type( shimcache_symbol_table + constants.BANG + "SHIM_CACHE_HANDLE" @@ -419,7 +419,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf data_sec_offset + data_sec_size, 8 if symbols.symbol_table_is_64bit(context, nt_symbol_table) else 4, ): - vollog.debug(f"Building shim handle pointer at {hex(offset)}") + vollog.debug(f"Building shim handle pointer at {offset:#x}") shim_handle = context.object( object_type=shimcache_symbol_table + constants.BANG + "pointer", layer_name=kernel_layer_name, @@ -430,7 +430,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf if shim_handle.is_valid(mod_page_offset, mod_page_offset + mod_page_size): if shim_handle.head is not None: vollog.debug( - f"Found valid shim handle @ {hex(shim_handle.vol.offset)}" + f"Found valid shim handle @ {shim_handle.vol.offset:#x}" ) shim_heads.append(shim_handle.head) if len(shim_heads) == 2: @@ -440,7 +440,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf vollog.debug("Failed to identify two valid SHIM_CACHE_HANDLE structures") return - # On Windows 8 x64, the frist cache contains the shim cache + # On Windows 8 x64, the first cache contains the shim cache. # On Windows 8 x86, 8.1 x86/x64, and 10, the second cache contains the shim cache. if ( not symbols.symbol_table_is_64bit(context, nt_symbol_table) From 03049f789559af5c4cdb56f343460178b52220f9 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 3 Jan 2025 18:40:52 +0000 Subject: [PATCH 115/268] Add missing exception handling in env var recovery. Prevent backtraces --- volatility3/framework/plugins/linux/envars.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 8cdbfe493..04b75c8a8 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -5,7 +5,7 @@ import logging from typing import Iterable, Tuple -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -58,10 +58,16 @@ class Envars(plugins.PluginInterface): Tuples of (key, value) representing each environment variable. """ - task_name = utility.array_to_string(task.comm) + # This ensures the `task` is valid as well as its + # memory mapping structures + try: + task_name = utility.array_to_string(task.comm) + env_start = task.mm.env_start + env_end = task.mm.env_end + except exceptions.InvalidAddressException: + return None + task_pid = task.pid - env_start = task.mm.env_start - env_end = task.mm.env_end env_area_size = env_end - env_start if not (0 < env_area_size <= env_area_max_size): vollog.debug( From 8ba60a2aaddf86e4cbd065c95d2553ce221db183 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 4 Jan 2025 16:49:02 +0000 Subject: [PATCH 116/268] Change add_process_layer to return None instead of throwing an exception as it was meant to be designed --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b02f80433..df1c00e3d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -324,9 +324,11 @@ class task_struct(generic.GenericIntelProcess): raise TypeError( "Parent layer is not a translation layer, unable to construct process layer" ) - dtb, layer_name = parent_layer.translate(pgd) - if not dtb: + try: + dtb, layer_name = parent_layer.translate(pgd) + except exceptions.InvalidAddressException: return None + if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.pid}" # Add the constructed layer and return the name From 5f1d318c715311ed12d67bde5a87a8a78e0d3bf0 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 13:39:00 +0000 Subject: [PATCH 117/268] Tiny comment changes --- volatility3/framework/plugins/windows/cmdscan.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 9645ee507..3dc70d649 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -67,6 +67,7 @@ class CmdScan(interfaces.plugins.PluginInterface): Args: conhost_proc: the process object for conhost.exe + size_filter: filter (keep) vads less than this size (bytes) Returns: A list of tuples of: @@ -100,7 +101,7 @@ class CmdScan(interfaces.plugins.PluginInterface): kernel_symbol_table_name: The name of the table containing the kernel symbols config_path: The config path where to find symbol files procs: list of process objects - max_history: an initial set of CommandHistorySize values + max_history: An initial set of CommandHistorySize values Returns: The conhost process object, the command history structure, a dictionary of properties for @@ -227,7 +228,6 @@ class CmdScan(interfaces.plugins.PluginInterface): "data": command_history.CommandCountMax, } ) - command_history_properties.append( { "level": 1, @@ -236,6 +236,7 @@ class CmdScan(interfaces.plugins.PluginInterface): "data": "", } ) + for ( cmd_index, bucket_cmd, @@ -352,7 +353,7 @@ class CmdScan(interfaces.plugins.PluginInterface): def _conhost_proc_filter(self, proc: interfaces.objects.ObjectInterface): """ - Used to filter to only conhost.exe processes + Used to filter only conhost.exe processes """ process_name = utility.array_to_string(proc.ImageFileName) From ab60add9933ee3863c3f2329d2c99af314b5b453 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 17:14:21 +0000 Subject: [PATCH 118/268] Update case insensitive check Update link and use casefold() instead of lower(). --- volatility3/framework/layers/registry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index c684ccd40..6d85da982 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -192,9 +192,9 @@ class RegistryHive(linear.LinearlyMappedLayer): while key_array and node_key: subkeys = node_key[-1].get_subkeys() for subkey in subkeys: - # registry keys are not case sensitive so compare lowercase - # https://msdn.microsoft.com/en-us/library/windows/desktop/ms724946(v=vs.85).aspx - if subkey.get_name().lower() == key_array[0].lower(): + # registry keys are not case sensitive so compare likewise + # https://learn.microsoft.com/en-gb/windows/win32/sysinfo/structure-of-the-registry + if subkey.get_name().casefold() == key_array[0].casefold(): node_key = node_key + [subkey] found_key, key_array = found_key + [key_array[0]], key_array[1:] break From 8f4f576e93a7594666f0e58f8ae73cce5538902c Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 21:09:15 +0000 Subject: [PATCH 119/268] Update case insensitive check Update link and use casefold() instead of lower(). --- volatility3/framework/layers/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 6d85da982..21e1a938e 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -193,7 +193,7 @@ class RegistryHive(linear.LinearlyMappedLayer): subkeys = node_key[-1].get_subkeys() for subkey in subkeys: # registry keys are not case sensitive so compare likewise - # https://learn.microsoft.com/en-gb/windows/win32/sysinfo/structure-of-the-registry + # https://learn.microsoft.com/en-us/windows/win32/sysinfo/structure-of-the-registry if subkey.get_name().casefold() == key_array[0].casefold(): node_key = node_key + [subkey] found_key, key_array = found_key + [key_array[0]], key_array[1:] From 94ec7d89c09b2a276e79fc4c7561828340d5712a Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 21:55:58 +0000 Subject: [PATCH 120/268] Tiny comment changes --- volatility3/framework/plugins/windows/cmdscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 3dc70d649..0cd0addb2 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -67,7 +67,7 @@ class CmdScan(interfaces.plugins.PluginInterface): Args: conhost_proc: the process object for conhost.exe - size_filter: filter (keep) vads less than this size (bytes) + size_filter: size above which vads will not be returned Returns: A list of tuples of: @@ -100,7 +100,7 @@ class CmdScan(interfaces.plugins.PluginInterface): kernel_layer_name: The name of the layer on which to operate kernel_symbol_table_name: The name of the table containing the kernel symbols config_path: The config path where to find symbol files - procs: list of process objects + procs: List of process objects max_history: An initial set of CommandHistorySize values Returns: From a7b4e2fb45bef981eb54c44a5e0cef87b879058f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:02:15 +0100 Subject: [PATCH 121/268] version check_modules --- volatility3/framework/plugins/linux/check_modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/check_modules.py b/volatility3/framework/plugins/linux/check_modules.py index 9b3594c5e..0ed638d9c 100644 --- a/volatility3/framework/plugins/linux/check_modules.py +++ b/volatility3/framework/plugins/linux/check_modules.py @@ -18,6 +18,7 @@ vollog = logging.getLogger(__name__) class Check_modules(plugins.PluginInterface): """Compares module list to sysfs info, if available""" + _version = (1, 0, 0) _required_framework_version = (2, 0, 0) @classmethod From 2d262e7acf5c9aabb32240c01cd57890b7d57647 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:20:56 +0100 Subject: [PATCH 122/268] cut unnecessary intermediate LinuxUtilityInterface --- .../framework/symbols/linux/utilities/__init__.py | 11 ----------- .../framework/symbols/linux/utilities/tainting.py | 5 ++--- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/__init__.py b/volatility3/framework/symbols/linux/utilities/__init__.py index 4225d444b..e69de29bb 100644 --- a/volatility3/framework/symbols/linux/utilities/__init__.py +++ b/volatility3/framework/symbols/linux/utilities/__init__.py @@ -1,11 +0,0 @@ -from volatility3 import framework -from volatility3.framework import interfaces - - -class LinuxUtilityInterface(interfaces.configuration.VersionableInterface): - """Class with multiple useful Linux functions surrounding a specific piece of functionality.""" - - _version = (2, 1, 1) - _required_framework_version = (2, 0, 0) - - framework.require_interface_version(*_required_framework_version) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index fc2f94109..29d7d2b5b 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -1,11 +1,10 @@ from volatility3 import framework from volatility3.framework import interfaces -from volatility3.framework.symbols.linux.utilities import LinuxUtilityInterface from volatility3.framework.constants import linux as linux_constants from typing import List, Optional -class Tainting(LinuxUtilityInterface): +class Tainting(interfaces.configuration.VersionableInterface): """Tainted kernel and modules parsing capabilities. Relevant Linux kernel functions: @@ -14,7 +13,7 @@ class Tainting(LinuxUtilityInterface): """ _version = (1, 0, 0) - _required_framework_version = (2, 16, 0) + _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) From 5c70356c27aedc931c03a8e633952b126ef5254b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:23:16 +0100 Subject: [PATCH 123/268] version check_modules requirement --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 042930740..b44f84c7d 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -38,7 +38,7 @@ class Modxview(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="check_modules", plugin=check_modules.Check_modules, - version=(0, 0, 0), + version=(1, 0, 0), ), requirements.PluginRequirement( name="hidden_modules", From 302f9fdf5ba1c24d07d2fce3d0f7c87c3e6bd1f2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:24:18 +0100 Subject: [PATCH 124/268] cut unnecessary plugin runner functions --- .../framework/plugins/linux/modxview.py | 78 ++++++------------- 1 file changed, 25 insertions(+), 53 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index b44f84c7d..a247bc2cc 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -53,54 +53,6 @@ class Modxview(interfaces.plugins.PluginInterface): ), ] - @classmethod - def run_lsmod( - cls, context: interfaces.context.ContextInterface, kernel_name: str - ) -> List[extensions.module]: - """Wrapper for the lsmod plugin.""" - return list(lsmod.Lsmod.list_modules(context, kernel_name)) - - @classmethod - def run_check_modules( - cls, - context: interfaces.context.ContextInterface, - kernel_name: str, - ) -> List[extensions.module]: - """Wrapper for the check_modules plugin. - Here, we extract the /sys/module/ list.""" - kernel = context.modules[kernel_name] - sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( - context, kernel_name - ) - - # Convert get_kset_modules() offsets back to module objects - return [ - kernel.object(object_type="module", offset=m_offset, absolute=True) - for m_offset in sysfs_modules.values() - ] - - @classmethod - def run_hidden_modules( - cls, - context: interfaces.context.ContextInterface, - kernel_name: str, - known_modules_addresses: Set[int], - ) -> List[extensions.module]: - """Wrapper for the hidden_modules plugin.""" - modules_memory_boundaries = ( - hidden_modules.Hidden_modules.get_modules_memory_boundaries( - context, kernel_name - ) - ) - return list( - hidden_modules.Hidden_modules.get_hidden_modules( - context, - kernel_name, - known_modules_addresses, - modules_memory_boundaries, - ) - ) - @classmethod def flatten_run_modules_results( cls, run_results: Dict[str, List[extensions.module]], deduplicate: bool = True @@ -140,15 +92,35 @@ class Modxview(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_name] run_results = {} - run_results["lsmod"] = cls.run_lsmod(context, kernel_name) - run_results["check_modules"] = cls.run_check_modules(context, kernel_name) + # lsmod + run_results["lsmod"] = list(lsmod.Lsmod.list_modules(context, kernel_name)) + # check_modules + sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( + context, kernel_name + ) + ## Convert get_kset_modules() offsets back to module objects + run_results["check_modules"] = [ + kernel.object(object_type="module", offset=m_offset, absolute=True) + for m_offset in sysfs_modules.values() + ] + # hidden_modules if run_hidden_modules: - known_module_addresses = set( + known_modules_addresses = set( context.layers[kernel.layer_name].canonicalize(module.vol.offset) for module in run_results["lsmod"] + run_results["check_modules"] ) - run_results["hidden_modules"] = cls.run_hidden_modules( - context, kernel_name, known_module_addresses + modules_memory_boundaries = ( + hidden_modules.Hidden_modules.get_modules_memory_boundaries( + context, kernel_name + ) + ) + run_results["hidden_modules"] = list( + hidden_modules.Hidden_modules.get_hidden_modules( + context, + kernel_name, + known_modules_addresses, + modules_memory_boundaries, + ) ) return run_results From 4115c26e7cc119a68aa33fff7f5b8a730b5b2c69 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:24:44 +0100 Subject: [PATCH 125/268] bump framework req to 2.18.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index a247bc2cc..34f5bac8f 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 16, 0) + _required_framework_version = (2, 18, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From bd82f4f33d860cb067e379600da5bbc74f9e2247 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:25:07 +0100 Subject: [PATCH 126/268] 2.16.0 -> 2.18.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 24f96fa89..832b2a5ba 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 16 # Number of changes that only add to the interface +VERSION_MINOR = 18 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From d956742db98610dfa94678ffb98d53c5b6bcd161 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:31:39 +0100 Subject: [PATCH 127/268] remove typing.Set import --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 34f5bac8f..3655200e8 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import List, Dict, Set, Iterator +from typing import List, Dict, Iterator from volatility3.plugins.linux import lsmod, check_modules, hidden_modules from volatility3.framework import interfaces from volatility3.framework.configuration import requirements From b5bc54cfaed91f4d615790305c80ce802658dafe Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:18:34 +0000 Subject: [PATCH 128/268] Use in-place subtraction Also tweak comments. --- volatility3/framework/renderers/conversion.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index e48684b31..f848b2dad 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -18,7 +18,7 @@ def wintime_to_datetime( unix_time = wintime // 10000000 if unix_time == 0: return renderers.NotApplicableValue() - unix_time = unix_time - 11644473600 + unix_time -= 11644473600 try: return datetime.datetime.fromtimestamp(unix_time, datetime.timezone.utc) # Windows sometimes throws OSErrors rather than ValueError/OverflowError when it can't convert a value @@ -71,7 +71,7 @@ def round(addr: int, align: int, up: bool = False) -> int: Args: addr: the address align: the alignment value - up: Whether to round up or not + up: whether to round up or not Returns: The aligned address @@ -122,11 +122,12 @@ def convert_port(port_as_integer): def convert_network_four_tuple(family, four_tuple): - """Converts the connection four_tuple: (source ip, source port, dest ip, - dest port) + """Converts the connection four_tuple: + + (source ip, source port, dest ip, dest port) into their string equivalents. IP addresses are expected as a tuple - of unsigned shorts Ports are converted to proper endianness as well + of unsigned shorts. Ports are converted to proper endianness as well. """ if family == socket.AF_INET: From 43ac6c4d6271c928d9bcdaf6407e01e1c96d7cf9 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 7 Jan 2025 10:24:46 -0600 Subject: [PATCH 129/268] Fix copy-pasted module docstrings This updates the module docstrings for 5 modules that duplicate the docstring from the `proc` module. This was presumably the result of using the `proc` module as a template for the others. --- volatility3/framework/plugins/linux/bash.py | 4 ++-- volatility3/framework/plugins/linux/check_afinfo.py | 4 ++-- volatility3/framework/plugins/linux/check_syscall.py | 3 +-- volatility3/framework/plugins/linux/elfs.py | 4 ++-- volatility3/framework/plugins/linux/lsmod.py | 3 +-- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index 056e3cd51..8acfeb848 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that recovers bash command history +from bash process memory.""" import datetime import struct diff --git a/volatility3/framework/plugins/linux/check_afinfo.py b/volatility3/framework/plugins/linux/check_afinfo.py index 201a443f7..7aa3cbdd2 100644 --- a/volatility3/framework/plugins/linux/check_afinfo.py +++ b/volatility3/framework/plugins/linux/check_afinfo.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that verifies the operation function +pointers of network protocols.""" import logging from typing import List diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 3537a9fa1..13d312f2f 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -1,8 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that checks the system call table for hooks.""" import contextlib import logging from typing import List diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 2fd740941..0d1c9c2dd 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin for enumerating memory-mapped +ELF files across all processes.""" import logging from typing import List, Optional, Type diff --git a/volatility3/framework/plugins/linux/lsmod.py b/volatility3/framework/plugins/linux/lsmod.py index 49e990e93..e9a2a7137 100644 --- a/volatility3/framework/plugins/linux/lsmod.py +++ b/volatility3/framework/plugins/linux/lsmod.py @@ -1,8 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that lists loaded kernel modules.""" import logging from typing import List, Iterable From 32cb6e11f6abe86ce5284e1a618bae9ab1cd4a5f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 7 Jan 2025 19:41:02 +0000 Subject: [PATCH 130/268] Change one letter of a typo --- volatility3/framework/plugins/windows/driverscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/driverscan.py b/volatility3/framework/plugins/windows/driverscan.py index 24d81c3d5..d388ffbb7 100644 --- a/volatility3/framework/plugins/windows/driverscan.py +++ b/volatility3/framework/plugins/windows/driverscan.py @@ -64,7 +64,7 @@ class DriverScan(interfaces.plugins.PluginInterface): names associated with a driver Args: - driver: A Eriver object + driver: A Driver object Returns: A tuple of strings of (driver name, service key, driver alt. name) From 0860441c2fc5a5a97902a7582473d8462c457bc3 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 17:11:02 +0000 Subject: [PATCH 131/268] Core: Improve speed to JSONSchema validation --- pyproject.toml | 3 +-- volatility3/framework/plugins/isfinfo.py | 2 +- volatility3/schemas/__init__.py | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86e3921d2..af22cbe0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,10 +33,9 @@ cloud = [ dev = [ "volatility3[full,cloud]", - "jsonschema>=4.23.0,<5", + "fastjsonschema>=2.21.1,<3", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", - "types-jsonschema>=4.23.0,<5", ] test = [ diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 1c2ac52e9..34b0a5653 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -97,7 +97,7 @@ class IsfInfo(plugins.PluginInterface): if filter_item in isf_file: filtered_list.append(isf_file) - if find_spec("jsonschema") and self.config["validate"]: + if find_spec("fastjsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False" diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 90cfaba48..3964e29a6 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -14,6 +14,8 @@ vollog = logging.getLogger(__name__) cached_validation_filepath = os.path.join(constants.CACHE_PATH, "valid_isf.hashcache") +validators = {} + def load_cached_validations() -> Set[str]: """Loads up the list of successfully cached json objects, so we don't need @@ -92,7 +94,12 @@ def valid( if input_hash in cached_validations and use_cache: return True try: - import jsonschema + import fastjsonschema + + schema_key = json.dumps(schema, sort_keys=True) + if schema_key not in validators: + validator = fastjsonschema.compile(schema) + validators[schema_key] = validator except ImportError: vollog.info("Dependency for validation unavailable: jsonschema") vollog.debug("All validations will report success, even with malformed input") @@ -100,10 +107,13 @@ def valid( try: vollog.debug("Validating JSON against schema...") - jsonschema.validate(input, schema) + validators[schema_key](input) + import pdb + + pdb.set_trace() cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") - except jsonschema.exceptions.SchemaError: + except fastjsonschema.JsonSchemaValueException: vollog.debug("Schema validation error", exc_info=True) return False From e676e6179a3cd9c8d6afb838560d7a9e4d3a5420 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:08:22 +0000 Subject: [PATCH 132/268] Swap fastjsonschema for jsonschema because of date-time validation issues --- volatility3/schemas/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 3964e29a6..b94e0831d 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -94,11 +94,13 @@ def valid( if input_hash in cached_validations and use_cache: return True try: - import fastjsonschema + import jsonschema schema_key = json.dumps(schema, sort_keys=True) if schema_key not in validators: - validator = fastjsonschema.compile(schema) + validator_class = jsonschema.validators.validator_for(schema) + validator_class.check_schema(schema) + validator = validator_class(schema) validators[schema_key] = validator except ImportError: vollog.info("Dependency for validation unavailable: jsonschema") @@ -107,10 +109,7 @@ def valid( try: vollog.debug("Validating JSON against schema...") - validators[schema_key](input) - import pdb - - pdb.set_trace() + validators[schema_key].validate(input) cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") except fastjsonschema.JsonSchemaValueException: From 21decf13708d882369cc1f8fa2884f5a8ae5494d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:12:36 +0000 Subject: [PATCH 133/268] Core: Put the dependencies back for jsonschema --- pyproject.toml | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af22cbe0a..fbdbf0a8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,15 @@ [project] name = "volatility3" description = "Memory forensics framework" -keywords = ["volatility", "memory", "forensics", "framework", "windows", "linux", "volshell"] +keywords = [ + "volatility", + "memory", + "forensics", + "framework", + "windows", + "linux", + "volshell", +] readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, @@ -10,9 +18,7 @@ requires-python = ">=3.8.0" license = { text = "VSL" } dynamic = ["version"] -dependencies = [ - "pefile>=2024.8.26", -] +dependencies = ["pefile>=2024.8.26"] [project.optional-dependencies] full = [ @@ -26,16 +32,14 @@ full = [ "pillow>=10.0.0,<11.0.0", ] -cloud = [ - "gcsfs>=2024.10.0", - "s3fs>=2024.10.0", -] +cloud = ["gcsfs>=2024.10.0", "s3fs>=2024.10.0"] dev = [ "volatility3[full,cloud]", - "fastjsonschema>=2.21.1,<3", + "jsonschema>=4.23.0,<5", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", + "types-jsonschema>=4.23.0,<5", ] test = [ @@ -78,16 +82,16 @@ target-version = "py38" [tool.ruff.lint] select = [ - "F", # pyflakes - "E", # pycodestyle errors - "W", # pycodestyle warnings - "G", # flake8-logging-format - "PIE", # flake8-pie - "UP", # pyupgrade + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "G", # flake8-logging-format + "PIE", # flake8-pie + "UP", # pyupgrade ] ignore = [ - "E501", # ignore due to conflict with formatter + "E501", # ignore due to conflict with formatter ] [build-system] From d07d31047b2ed49a570aa4b3698f6a64145c2d83 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:24:48 +0000 Subject: [PATCH 134/268] Core: Revert the exception catching too --- volatility3/schemas/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index b94e0831d..e894def9f 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -112,7 +112,7 @@ def valid( validators[schema_key].validate(input) cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") - except fastjsonschema.JsonSchemaValueException: + except jsonschema.exceptions.SchemaError: vollog.debug("Schema validation error", exc_info=True) return False From 585901105275a015a3c4326e486f4e2a52d8eb12 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 12:35:04 +0100 Subject: [PATCH 135/268] introduce customizable plugin arparse epilog --- volatility3/cli/__init__.py | 3 +++ volatility3/framework/interfaces/plugins.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 6172a17f3..87caaece6 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -368,6 +368,9 @@ class CommandLine: help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, ) + epilog = getattr(plugin_list[plugin], "_argparse_epilog", None) + if epilog is not None: + plugin_parser.epilog = epilog self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) ### diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index f763815a6..6cd72f02e 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -112,6 +112,8 @@ class PluginInterface( # Be careful with inheritance around this (We default to requiring a version which doesn't exist, so it must be set) _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" + _argparse_epilog: str = None + """Display additional description of the plugin after the description of the arguments. See: https://docs.python.org/3/library/argparse.html#epilog""" def __init__( self, From 530617a700e259f69d53f62f08ccc3382bcdd057 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:52:54 +0000 Subject: [PATCH 136/268] Small readability improvements --- volatility3/framework/automagic/mac.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index f3679d160..a883028d2 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -101,7 +101,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVVV, - f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}", + f"Skipping invalid idlepml4_ptr: {idlepml4_ptr:#x}", ) continue @@ -112,7 +112,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if tmp_dtb % 4096: vollog.log( constants.LOGLEVEL_VVV, - f"Skipping non-page aligned DTB: 0x{tmp_dtb:0x}", + f"Skipping non-page aligned DTB: {tmp_dtb:#x}", ) continue @@ -136,7 +136,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): new_layer.config["kernel_virtual_offset"] = kaslr_shift if new_layer and dtb: - vollog.debug(f"DTB was found at: 0x{dtb:0x}") + vollog.debug(f"DTB was found at: {dtb:#x}") return new_layer vollog.debug("No suitable mac banner could be matched") return None @@ -182,7 +182,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) tmp_aslr_shift = offset - cls.virtual_to_physical_address( version_json_address @@ -208,7 +208,6 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): continue aslr_shift = tmp_aslr_shift & 0xFFFFFFFF - break vollog.log(constants.LOGLEVEL_VVVV, f"Mac find_aslr returned: {aslr_shift:0x}") @@ -219,9 +218,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): """Converts a virtual mac address to a physical one (does not account of ASLR)""" if addr > 0xFFFFFF8000000000: - addr = addr - 0xFFFFFF8000000000 + addr -= 0xFFFFFF8000000000 else: - addr = addr - 0xFF8000000000 + addr -= 0xFF8000000000 return addr From a7661d45e78b10bc736946425055755c9627d111 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 10 Jan 2025 09:21:21 -0600 Subject: [PATCH 137/268] Windows: Certificates - handle uncaught RegistryFormatException Changes variable import to module import, and catches an unhandled `RegistryFormatException` in certificates.py --- .../plugins/windows/registry/certificates.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 8587b3719..a83badb90 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,11 +1,11 @@ import contextlib import logging import struct -from typing import List, Iterator, Optional, Tuple, Type +from typing import Iterator, List, Optional, Tuple, Type from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes +from volatility3.framework.symbols.windows.extensions import registry from volatility3.plugins.windows.registry import hivelist, printkey vollog = logging.getLogger(__name__) @@ -81,7 +81,11 @@ class Certificates(interfaces.plugins.PluginInterface): "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", ]: - with contextlib.suppress(KeyError, exceptions.InvalidAddressException): + with contextlib.suppress( + KeyError, + registry.RegistryFormatException, + exceptions.InvalidAddressException, + ): # Walk it node_path = hive.get_key(top_key, return_list=True) for ( @@ -92,7 +96,11 @@ class Certificates(interfaces.plugins.PluginInterface): _volatility, node, ) in printkey.PrintKey.key_iterator(hive, node_path, recurse=True): - if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": + if ( + not is_key + and registry.RegValueTypes(node.Type) + == registry.RegValueTypes.REG_BINARY + ): name, certificate_data = self.parse_data(node.decode_data()) unique_key_offset = ( key_path.casefold().index(top_key.casefold()) From 96eca6e0162a77699c2befcce6df16f7deac4d23 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 19:38:07 +0100 Subject: [PATCH 138/268] more compact _argparse_epilog --- volatility3/cli/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 87caaece6..37923362a 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -368,9 +368,9 @@ class CommandLine: help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, ) - epilog = getattr(plugin_list[plugin], "_argparse_epilog", None) - if epilog is not None: - plugin_parser.epilog = epilog + plugin_parser.epilog = getattr( + plugin_list[plugin], "_argparse_epilog", None + ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) ### From 615d1d5a2e85dcd2f9d65493690a474c15f691cd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 19:49:25 +0100 Subject: [PATCH 139/268] more compact _argparse_epilog --- volatility3/cli/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 37923362a..fde4fcc6d 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -367,9 +367,7 @@ class CommandLine: plugin, help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, - ) - plugin_parser.epilog = getattr( - plugin_list[plugin], "_argparse_epilog", None + epilog=getattr(plugin_list[plugin], "_argparse_epilog", None), ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) From a26ff8fa6e6ba03a6ea3ebe6c5f3b38b3a4d8851 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 10 Jan 2025 19:03:51 +0000 Subject: [PATCH 140/268] Small readability improvements --- volatility3/framework/automagic/mac.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index a883028d2..3b16eb353 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -184,12 +184,12 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): for offset, banner in offset_generator: banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) - tmp_aslr_shift = offset - cls.virtual_to_physical_address( + aslr_shift = offset - cls.virtual_to_physical_address( version_json_address ) major_string = context.layers[layer_name].read( - version_major_phys_offset + tmp_aslr_shift, 4 + version_major_phys_offset + aslr_shift, 4 ) major = struct.unpack(" Date: Fri, 10 Jan 2025 19:08:56 +0000 Subject: [PATCH 141/268] Small readability improvements --- volatility3/framework/automagic/mac.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 3b16eb353..94c259463 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -184,9 +184,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): for offset, banner in offset_generator: banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) - aslr_shift = offset - cls.virtual_to_physical_address( - version_json_address - ) + aslr_shift = offset - cls.virtual_to_physical_address(version_json_address) major_string = context.layers[layer_name].read( version_major_phys_offset + aslr_shift, 4 From 1cf0232d25fa6dffa21ae3c281e1f568bbb280ab Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 21:19:31 +0100 Subject: [PATCH 142/268] less specific argparse epilog reference --- volatility3/cli/__init__.py | 2 +- volatility3/framework/interfaces/plugins.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index fde4fcc6d..82a2a4205 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -367,7 +367,7 @@ class CommandLine: plugin, help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, - epilog=getattr(plugin_list[plugin], "_argparse_epilog", None), + epilog=plugin_list[plugin].additional_description, ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 6cd72f02e..7ad78d0ba 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -112,7 +112,7 @@ class PluginInterface( # Be careful with inheritance around this (We default to requiring a version which doesn't exist, so it must be set) _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" - _argparse_epilog: str = None + additional_description: str = None """Display additional description of the plugin after the description of the arguments. See: https://docs.python.org/3/library/argparse.html#epilog""" def __init__( From 7913fb2bb0aac4cc390ce6e42ad6621115f0ae7c Mon Sep 17 00:00:00 2001 From: ikelos Date: Fri, 10 Jan 2025 21:07:08 +0000 Subject: [PATCH 143/268] Revert "Small readability improvements" --- volatility3/framework/automagic/mac.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 94c259463..f3679d160 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -101,7 +101,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVVV, - f"Skipping invalid idlepml4_ptr: {idlepml4_ptr:#x}", + f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}", ) continue @@ -112,7 +112,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if tmp_dtb % 4096: vollog.log( constants.LOGLEVEL_VVV, - f"Skipping non-page aligned DTB: {tmp_dtb:#x}", + f"Skipping non-page aligned DTB: 0x{tmp_dtb:0x}", ) continue @@ -136,7 +136,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): new_layer.config["kernel_virtual_offset"] = kaslr_shift if new_layer and dtb: - vollog.debug(f"DTB was found at: {dtb:#x}") + vollog.debug(f"DTB was found at: 0x{dtb:0x}") return new_layer vollog.debug("No suitable mac banner could be matched") return None @@ -182,12 +182,14 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) - aslr_shift = offset - cls.virtual_to_physical_address(version_json_address) + tmp_aslr_shift = offset - cls.virtual_to_physical_address( + version_json_address + ) major_string = context.layers[layer_name].read( - version_major_phys_offset + aslr_shift, 4 + version_major_phys_offset + tmp_aslr_shift, 4 ) major = struct.unpack(" 0xFFFFFF8000000000: - addr -= 0xFFFFFF8000000000 + addr = addr - 0xFFFFFF8000000000 else: - addr -= 0xFF8000000000 + addr = addr - 0xFF8000000000 return addr From 884237534142ec10ba6e7386eedc06ef30d277d0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 23:28:02 +0100 Subject: [PATCH 144/268] 2.15.0 -> 2.16.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 2f0c53093..24f96fa89 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 15 # Number of changes that only add to the interface +VERSION_MINOR = 16 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 0e4e7518447837b9c7f0f30203155b3a3fee0c3a Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 11 Jan 2025 14:05:36 +0100 Subject: [PATCH 145/268] stateless classmethods --- .../symbols/linux/utilities/tainting.py | 67 +++++++++++++------ 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 29d7d2b5b..552f51b98 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -17,26 +17,22 @@ class Tainting(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) - def __init__( - self, + @classmethod + def _get_kernel_taint_flags_list( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self._kernel = context.modules[kernel_module_name] - - @property - def _kernel_taint_flags_list( - self, ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self._kernel.has_symbol("taint_flags"): - return list(self._kernel.object_from_symbol("taint_flags")) + kernel = context.modules[kernel_module_name] + if kernel.has_symbol("taint_flags"): + return list(kernel.object_from_symbol("taint_flags")) return None + @classmethod def _module_flags_taint_pre_4_10_rc1( - self, taints: int, is_module: bool = False + cls, + taints: int, + is_module: bool = False, ) -> str: """Convert the module's taints value to a 1-1 character mapping. Relies on statically defined taints mappings in the framework. @@ -58,8 +54,13 @@ class Tainting(interfaces.configuration.VersionableInterface): return taints_string + @classmethod def _module_flags_taint_post_4_10_rc1( - self, taints: int, is_module: bool = False + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, ) -> str: """Convert the module's taints value to a 1-1 character mapping. Relies on kernel symbol embedded taints definitions. @@ -78,7 +79,9 @@ class Tainting(interfaces.configuration.VersionableInterface): The raw taints string. """ taints_string = "" - for taint_bit, taint_flag in enumerate(self._kernel_taint_flags_list): + for taint_bit, taint_flag in enumerate( + cls._get_kernel_taint_flags_list(context, kernel_module_name) + ): if is_module and not taint_flag.module: continue c_true = chr(taint_flag.c_true) @@ -90,7 +93,14 @@ class Tainting(interfaces.configuration.VersionableInterface): return taints_string - def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + @classmethod + def get_taints_as_plain_string( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, + ) -> str: """Convert the taints value to a 1-1 character mapping. Args: @@ -103,11 +113,22 @@ class Tainting(interfaces.configuration.VersionableInterface): - module_flags_taint kernel function """ - if self._kernel_taint_flags_list: - return self._module_flags_taint_post_4_10_rc1(taints, is_module) - return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + if cls._get_kernel_taint_flags_list(context, kernel_module_name): + return cls._module_flags_taint_post_4_10_rc1( + context, kernel_module_name, taints, is_module + ) + return cls._module_flags_taint_pre_4_10_rc1( + context, kernel_module_name, taints, is_module + ) - def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + @classmethod + def get_taints_parsed( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, + ) -> List[str]: """Convert the taints string to a 1-1 descriptor mapping. Args: @@ -121,7 +142,9 @@ class Tainting(interfaces.configuration.VersionableInterface): - module_flags_taint kernel function """ comprehensive_taints = [] - for character in self.get_taints_as_plain_string(taints, is_module): + for character in cls.get_taints_as_plain_string( + context, kernel_module_name, taints, is_module + ): taint_flag = linux_constants.TAINT_FLAGS.get(character) if not taint_flag: comprehensive_taints.append(f"") From 6817d2c765fb5117a8ec6adb92cf343d37a92595 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 16:00:50 +1100 Subject: [PATCH 146/268] linux: ensure process listing functions yield only valid tasks --- volatility3/framework/plugins/linux/pslist.py | 5 ++- .../symbols/linux/extensions/__init__.py | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 37cf000fc..931acf29a 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -34,7 +34,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 13, 0) - _version = (4, 0, 0) + _version = (4, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -250,6 +250,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Note that the init_task itself is not yielded, since "ps" also never shows it. for task in init_task.tasks: + if not task.is_valid(): + continue + if filter_func(task): continue diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..a50b8ae09 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -307,6 +307,36 @@ class module(generic.GenericIntelProcess): class task_struct(generic.GenericIntelProcess): + def is_valid(self) -> bool: + layer = self._context.layers[self.vol.layer_name] + # Make sure the entire task content is readable + if not layer.is_valid(self.vol.offset, self.vol.size): + return False + + if self.pid < 0: + return False + + if not (self.signal and self.signal.is_readable()): + return False + + if not (self.nsproxy and self.nsproxy.is_readable()): + return False + + if not (self.real_parent and self.real_parent.is_readable()): + return False + + if self.active_mm and not self.active_mm.is_readable(): + return False + + if self.mm: + if not self.mm.is_readable(): + return False + + if self.mm != self.active_mm: + return False + + return True + def add_process_layer( self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: @@ -401,6 +431,8 @@ class task_struct(generic.GenericIntelProcess): tasks_iterable = self._get_tasks_iterable() threads_seen = set([self.vol.offset]) for task in tasks_iterable: + if not task.is_valid(): + continue if task.vol.offset not in threads_seen: threads_seen.add(task.vol.offset) yield task From 093b12b7cdf4a1623a5d534309f0673c0311cc6b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 16:52:45 +1100 Subject: [PATCH 147/268] Linux and Windows: Ensure linked list object extensions consistently yield valid entries --- .../symbols/linux/extensions/__init__.py | 42 ++++++++++------- .../symbols/windows/extensions/__init__.py | 47 +++++++++---------- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..2065b3bb4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1209,35 +1209,43 @@ class list_head(objects.StructType, collections.abc.Iterable): Objects of the type specified via the "symbol_type" argument. """ - layer = layer or self.vol.layer_name + layer_name = layer or self.vol.layer_name + + trans_layer = self._context.layers[layer_name] + if not trans_layer.is_valid(self.vol.offset): + return None relative_offset = self._context.symbol_space.get_type( symbol_type ).relative_child_offset(member) - direction = "prev" - if forward: - direction = "next" - try: - link = getattr(self, direction).dereference() - except exceptions.InvalidAddressException: + direction = "next" if forward else "prev" + + link_ptr = getattr(self, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() + if not sentinel: - yield self._context.object( - symbol_type, layer, offset=self.vol.offset - relative_offset - ) + obj_offset = self.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + + yield self._context.object(symbol_type, layer_name, offset=obj_offset) + seen = {self.vol.offset} while link.vol.offset not in seen: - obj = self._context.object( - symbol_type, layer, offset=link.vol.offset - relative_offset - ) - yield obj + obj_offset = link.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + + yield self._context.object(symbol_type, layer_name, offset=obj_offset) seen.add(link.vol.offset) - try: - link = getattr(link, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(link, direction) + if not (link_ptr and link_ptr.is_readable()): break + link = link_ptr.dereference() def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index f12fd3f5b..214002f49 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -962,56 +962,55 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): ) -> Iterator[interfaces.objects.ObjectInterface]: """Returns an iterator of the entries in the list.""" - layer = layer or self.vol.layer_name + layer_name = layer or self.vol.layer_name + native_layer_name = layer_name or self.vol.native_layer_name + + trans_layer = self._context.layers[layer_name] + if not trans_layer.is_valid(self.vol.offset): + return None relative_offset = self._context.symbol_space.get_type( symbol_type ).relative_child_offset(member) - direction = "Blink" - if forward: - direction = "Flink" + direction = "Flink" if forward else "Blink" - trans_layer = self._context.layers[layer] - - try: - is_valid = trans_layer.is_valid(self.vol.offset) - if not is_valid: - return None - - link = getattr(self, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(self, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() if not sentinel: + obj_offset = self.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + yield self._context.object( symbol_type, - layer, - offset=self.vol.offset - relative_offset, - native_layer_name=layer or self.vol.native_layer_name, + layer_name, + offset=obj_offset, + native_layer_name=native_layer_name, ) seen = {self.vol.offset} while link.vol.offset not in seen: obj_offset = link.vol.offset - relative_offset - if not trans_layer.is_valid(obj_offset): return None - obj = self._context.object( + yield self._context.object( symbol_type, - layer, + layer_name, offset=obj_offset, - native_layer_name=layer or self.vol.native_layer_name, + native_layer_name=native_layer_name, ) - yield obj seen.add(link.vol.offset) - try: - link = getattr(link, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(link, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) From 0d9715136cc9cc96637996b2bb027a76b8b5e87a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:13:44 +1100 Subject: [PATCH 148/268] Linux: Ensure VMA enumration functions yield only valid objects consistently --- .../symbols/linux/extensions/__init__.py | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..61562270a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -811,23 +811,30 @@ class mm_struct(objects.StructType): def _get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or - _get_mmap_iter() automatically as required.""" + _get_mmap_iter() automatically as required. + + Yields: + vm_area_struct objects + """ if not self.has_member("mmap"): raise AttributeError( "_get_mmap_iter called on mm_struct where no mmap member exists." ) - if not self.mmap: + vma_pointer = self.mmap + if not (vma_pointer and vma_pointer.is_readable()): return None - yield self.mmap + vma_object = vma_pointer.dereference() + yield vma_object - seen = {self.mmap.vol.offset} - link = self.mmap.vm_next + seen = {vma_pointer} + vma_pointer = vma_pointer.vm_next - while link != 0 and link.vol.offset not in seen: - yield link - seen.add(link.vol.offset) - link = link.vm_next + while vma_pointer and vma_pointer.is_readable() and vma_pointer not in seen: + vma_object = vma_pointer.dereference() + yield vma_object + seen.add(vma_pointer) + vma_pointer = vma_pointer.vm_next # TODO: As of version 3.0.0 this method should be removed def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: @@ -842,7 +849,11 @@ class mm_struct(objects.StructType): def _get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mm_mt member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or - get_mmap_iter() automatically as required.""" + get_mmap_iter() automatically as required. + + Yields: + vm_area_struct objects + """ if not self.has_member("mm_mt"): raise AttributeError( @@ -850,20 +861,27 @@ class mm_struct(objects.StructType): ) symbol_table_name = self.get_symbol_table_name() for vma_pointer in self.mm_mt.get_slot_iter(): - # convert pointer to vm_area_struct and yield - vma = self._context.object( + # Convert pointer to vm_area_struct and yield + vma_object = self._context.object( symbol_table_name + constants.BANG + "vm_area_struct", layer_name=self.vol.native_layer_name, offset=vma_pointer, ) - yield vma + yield vma_object def get_vma_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns an iterator for the VMAs in an mm_struct. Automatically choosing the mmap or mm_mt as required.""" + """Returns an iterator for the VMAs in an mm_struct. + Automatically choosing the mmap or mm_mt as required. + + Yields: + vm_area_struct objects + """ if self.has_member("mmap"): + # kernels < 6.1 yield from self._get_mmap_iter() elif self.has_member("mm_mt"): + # kernels >= 6.1 d4af56c5c7c6781ca6ca8075e2cf5bc119ed33d1 yield from self._get_maple_tree_iter() else: raise AttributeError("Unable to find mmap or mm_mt in mm_struct") From 35fa9321b3bdac0bc3b8097148eeb6872cade138 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:44:26 +1100 Subject: [PATCH 149/268] Linux: file struct: Remove `f_dentry` and `f_vfsmnt`, as they were preprocessor macro shortcuts, not actual members of the type. --- .../framework/symbols/linux/extensions/__init__.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..22c28f6fa 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1152,19 +1152,15 @@ class struct_file(objects.StructType): """Returns a pointer to the dentry associated with this file""" if self.has_member("f_path"): return self.f_path.dentry - elif self.has_member("f_dentry"): - return self.f_dentry - else: - raise AttributeError("Unable to find file -> dentry") + + raise AttributeError("Unable to find file -> dentry") def get_vfsmnt(self) -> interfaces.objects.ObjectInterface: """Returns the fs (vfsmount) where this file is mounted""" if self.has_member("f_path"): return self.f_path.mnt - elif self.has_member("f_vfsmnt"): - return self.f_vfsmnt - else: - raise AttributeError("Unable to find file -> vfs mount") + + raise AttributeError("Unable to find file -> vfs mount") def get_inode(self) -> interfaces.objects.ObjectInterface: """Returns an inode associated with this file""" From fba8f05c8075e07ccecbaca6299ef106127623e3 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:48:53 +1100 Subject: [PATCH 150/268] linux: Rename variable to clarify pointer type and avoid confusion with the 'dentry' class. --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 22c28f6fa..4d89764f3 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1390,9 +1390,9 @@ class mount(objects.StructType): A dentry pointer """ vfsmnt = self.get_vfsmnt_current() - dentry = vfsmnt.mnt_root + dentry_pointer = vfsmnt.mnt_root - return dentry + return dentry_pointer def get_dentry_parent(self): """Returns the parent root of the mounted tree From 4b10d658509faaf42b21c67ec543b9bd34f57f84 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:51:30 +1100 Subject: [PATCH 151/268] linux: minor docstring improvements --- .../framework/symbols/linux/__init__.py | 7 +-- .../symbols/linux/extensions/__init__.py | 46 ++++++++++--------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 5aa27b964..93ff35a06 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -106,8 +106,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): Args: task (task_struct): A reference task mnt (vfsmount or mount): A mounted filesystem or a mount point. - - kernels < 3.3.8 type is 'vfsmount' - - kernels >= 3.3.8 type is 'mount' + - kernels < 3.3 type is 'vfsmount' + - kernels >= 3.3 type is 'mount' Returns: str: Pathname of the mount point relative to the task's root directory. @@ -129,7 +129,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): rdentry (dentry *): A pointer to the root dentry rmnt (vfsmount *): A pointer to the root vfsmount dentry (dentry *): A pointer to the dentry - vfsmnt (vfsmount *): A pointer to the vfsmount + vfsmnt (vfsmount/vfsmount *): A vfsmount object (kernels >= 3.3) or a + vfsmount pointer (kernels < 3.3) Returns: str: Pathname of the mount point or file diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 4d89764f3..6d341653b 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1500,16 +1500,18 @@ class vfsmount(objects.StructType): ) def _is_kernel_prior_to_struct_mount(self) -> bool: - """Helper to distinguish between kernels prior to version 3.3.8 that - lacked the 'mount' structure and later versions that have it. + """Helper to distinguish between kernels prior to version 3.3 which lacked the + 'mount' struct, versus later versions that include it. + See 7d6fec45a5131918b51dcd76da52f2ec86a85be6. - The 'mnt_parent' member was moved from struct 'vfsmount' to struct - 'mount' when the latter was introduced. + # Following that commit, also in kernel version 3.3 (3376f34fff5be9954fd9a9c4fd68f4a0a36d480e), + # the 'mnt_parent' member was relocated from the 'vfsmount' struct to the newly + # introduced 'mount' struct. Alternatively, vmlinux.has_type('mount') can be used here but it is faster. Returns: - bool: 'True' if the kernel + 'True' if the kernel lacks the 'mount' struct, typically indicating kernel < 3.3. """ return self.has_member("mnt_parent") @@ -1517,22 +1519,21 @@ class vfsmount(objects.StructType): def is_equal(self, vfsmount_ptr) -> bool: """Helper to make sure it is comparing two pointers to 'vfsmount'. - Depending on the kernel version, the calling object (self) could be - a 'vfsmount \\*' (<3.3.8) or a 'vfsmount' (>=3.3.8). This way we trust - in the framework "auto" dereferencing ability to assure that when we - reach this point 'self' will be a 'vfsmount' already and self.vol.offset - a 'vfsmount \\*' and not a 'vfsmount \\*\\*'. The argument must be a 'vfsmount \\*'. + Depending on the kernel version, see 3376f34fff5be9954fd9a9c4fd68f4a0a36d480e, + the calling object (self) could be a 'vfsmount *' (<3.3) or a 'vfsmount' (>=3.3). + This way we trust in the framework "auto" dereferencing ability to assure that + when we reach this point 'self' will be a 'vfsmount' already and self.vol.offset + a 'vfsmount *' and not a 'vfsmount **'. The argument must be a 'vfsmount *'. Typically, it's called from do_get_path(). Args: - vfsmount_ptr (vfsmount *): A pointer to a 'vfsmount' + vfsmount_ptr: A pointer to a 'vfsmount' Raises: - exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \\*' + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount *' Returns: - bool: 'True' if the given argument points to the the same 'vfsmount' - as 'self'. + 'True' if the given argument points to the same 'vfsmount' as 'self'. """ if isinstance(vfsmount_ptr, objects.Pointer): return self.vol.offset == vfsmount_ptr @@ -1541,13 +1542,14 @@ class vfsmount(objects.StructType): "Unexpected argument type. It has to be a 'vfsmount *'" ) - def _get_real_mnt(self): + def _get_real_mnt(self) -> interfaces.objects.ObjectInterface: """Gets the struct 'mount' containing this 'vfsmount'. - It should be only called from kernels >= 3.3.8 when 'struct mount' was introduced. + It should be only called from kernels >= 3.3 when 'struct mount' was introduced. + See 7d6fec45a5131918b51dcd76da52f2ec86a85be6 Returns: - mount: the struct 'mount' containing this 'vfsmount'. + The 'mount' object containing this 'vfsmount'. """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) return linux.LinuxUtilities.container_of( @@ -1566,8 +1568,8 @@ class vfsmount(objects.StructType): """Gets the parent fs (vfsmount) to where it's mounted on Returns: - For kernels < 3.3.8: A vfsmount pointer - For kernels >= 3.3.8: A vfsmount object + For kernels < 3.3: A vfsmount pointer + For kernels >= 3.3: A vfsmount object """ if self._is_kernel_prior_to_struct_mount(): return self.get_mnt_parent() @@ -1600,8 +1602,8 @@ class vfsmount(objects.StructType): """Gets the mnt_parent member. Returns: - For kernels < 3.3.8: A vfsmount pointer - For kernels >= 3.3.8: A mount pointer + For kernels < 3.3: A vfsmount pointer + For kernels >= 3.3: A mount pointer """ if self._is_kernel_prior_to_struct_mount(): return self.mnt_parent @@ -1672,8 +1674,10 @@ class kobject(objects.StructType): class mnt_namespace(objects.StructType): def get_inode(self): if self.has_member("proc_inum"): + # 98f842e675f96ffac96e6c50315790912b2812be 3.8 <= kernels < 3.19 return self.proc_inum elif self.has_member("ns") and self.ns.has_member("inum"): + # kernels >= 3.19 435d5f4bb2ccba3b791d9ef61d2590e30b8e806e return self.ns.inum else: raise AttributeError("Unable to find mnt_namespace inode") From d4803a3884343f475a90a14de4be7fa947e561c5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:58:21 +1100 Subject: [PATCH 152/268] Linux: Ensure mount API consistently returns valid mountpoints and path names --- .../framework/plugins/linux/mountinfo.py | 6 ++++-- .../framework/symbols/linux/__init__.py | 21 ++++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index b4f80e4f5..5a0d39f31 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -36,7 +36,7 @@ class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 2, 0) - _version = (1, 2, 3) + _version = (1, 3, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -152,9 +152,11 @@ class MountInfo(plugins.PluginInterface): if not ( task and task.fs - and task.fs.root + and task.fs.is_readable() and task.nsproxy + and task.nsproxy.is_readable() and task.nsproxy.mnt_ns + and task.nsproxy.mnt_ns.is_readable() ): # This task doesn't have all the information required. # It should be a kernel < 2.6.30 diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 93ff35a06..03f4e501a 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -76,7 +76,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 2, 0) + _version = (2, 3, 0) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -121,7 +121,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt) @classmethod - def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> Union[None, str]: + def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> str: """Returns a pathname of the mount point or file It mimics the Linux kernel prepend_path function. @@ -136,8 +136,19 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): str: Pathname of the mount point or file """ + if not (rdentry and rdentry.is_readable() and rmnt and rmnt.is_readable()): + return "" + + if isinstance(vfsmnt, objects.Pointer) and not (rmnt and rmnt.is_readable()): + # vfsmnt can be the vfsmount object itself (>=3.3) or a vfsmount * (<3.3) + return "" + path_reversed = [] - while dentry != rdentry or not vfsmnt.is_equal(rmnt): + while ( + dentry + and dentry.is_readable() + and (dentry != rdentry or not vfsmnt.is_equal(rmnt)) + ): if dentry == vfsmnt.get_mnt_root() or dentry.is_root(): # Escaped? if dentry != vfsmnt.get_mnt_root(): @@ -450,6 +461,10 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): type_dec = vmlinux.get_type(type_name) member_offset = type_dec.relative_child_offset(member_name) container_addr = addr - member_offset + layer = vmlinux.context.layers[vmlinux.layer_name] + if not layer.is_valid(container_addr): + return None + return vmlinux.object( object_type=type_name, offset=container_addr, absolute=True ) From 1707e0a89ce88696f8585734587cc0f300b160ad Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 18:01:31 +1100 Subject: [PATCH 153/268] Fix array_to_string helper method: If called with other object than array and a count value, it will end up with an AttributeError exception --- volatility3/framework/objects/utility.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/objects/utility.py b/volatility3/framework/objects/utility.py index b241ed56a..0bc285517 100644 --- a/volatility3/framework/objects/utility.py +++ b/volatility3/framework/objects/utility.py @@ -33,11 +33,12 @@ def array_to_string( ) -> interfaces.objects.ObjectInterface: """Takes a volatility Array of characters and returns a string.""" # TODO: Consider checking the Array's target is a native char - if count is None: - count = array.vol.count if not isinstance(array, objects.Array): raise TypeError("Array_to_string takes an Array of char") + if count is None: + count = array.vol.count + return array.cast("string", max_length=count, errors=errors) @@ -45,8 +46,10 @@ def pointer_to_string(pointer: "objects.Pointer", count: int, errors: str = "rep """Takes a volatility Pointer to characters and returns a string.""" if not isinstance(pointer, objects.Pointer): raise TypeError("pointer_to_string takes a Pointer") + if count < 1: raise ValueError("pointer_to_string requires a positive count") + char = pointer.dereference() return char.cast("string", max_length=count, errors=errors) From 77ad6f0d831a33cfbc037012a8f2bec0c57520ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 18:25:04 +1100 Subject: [PATCH 154/268] linux: remove unused import --- volatility3/framework/symbols/linux/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 03f4e501a..931c461b9 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -4,7 +4,7 @@ import math import contextlib from abc import ABC, abstractmethod -from typing import Iterator, List, Tuple, Optional, Union +from typing import Iterator, List, Tuple, Optional from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects From 8bc04529350c3ce5a927099a72bc4e419a049db5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Jan 2025 11:21:23 +1100 Subject: [PATCH 155/268] linux: Improve compatibility with ancient kernels --- .../symbols/linux/extensions/__init__.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index a50b8ae09..4a1a263dd 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -316,16 +316,26 @@ class task_struct(generic.GenericIntelProcess): if self.pid < 0: return False - if not (self.signal and self.signal.is_readable()): + if self.has_member("signal") and not ( + self.signal and self.signal.is_readable() + ): return False - if not (self.nsproxy and self.nsproxy.is_readable()): + if self.has_member("nsproxy") and not ( + self.nsproxy and self.nsproxy.is_readable() + ): return False - if not (self.real_parent and self.real_parent.is_readable()): + if self.has_member("real_parent") and not ( + self.real_parent and self.real_parent.is_readable() + ): return False - if self.active_mm and not self.active_mm.is_readable(): + if ( + self.has_member("active_mm") + and self.active_mm + and not self.active_mm.is_readable() + ): return False if self.mm: From 7fc2af5b4ecf4b1ced5c71357d164981b05ed309 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Jan 2025 11:39:48 +1100 Subject: [PATCH 156/268] linux: Add an additional quick check before validating pointer readability --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 4a1a263dd..e2c9454f6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -313,7 +313,7 @@ class task_struct(generic.GenericIntelProcess): if not layer.is_valid(self.vol.offset, self.vol.size): return False - if self.pid < 0: + if self.pid < 0 or self.tgid < 0: return False if self.has_member("signal") and not ( From d21fdb4211c800404acd863b71b8302022d090e4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 16:52:45 +1100 Subject: [PATCH 157/268] Linux: pagecache: Harden Page Cache API to consistently yield valid entries --- .../framework/plugins/linux/pagecache.py | 14 ++++++-- .../framework/symbols/linux/__init__.py | 9 +++--- .../symbols/linux/extensions/__init__.py | 32 +++++++++++++------ 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 382268515..430190970 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -104,7 +104,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -253,6 +253,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): if not root_inode.is_valid(): continue + if not (root_inode.i_mapping and root_inode.i_mapping.is_readable()): + # Retrieving data from the page cache requires a valid address space + continue + # Inode already processed? if root_inode_ptr in seen_inodes: continue @@ -284,6 +288,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): if not file_inode.is_valid(): continue + if not (file_inode.i_mapping and file_inode.i_mapping.is_readable()): + # Retrieving data from the page cache requires a valid address space + continue + # Inode already processed? if file_inode_ptr in seen_inodes: continue @@ -316,10 +324,12 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): if self.config["find"]: if inode_in.path == self.config["find"]: inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) break # Only the first match else: inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) def generate_timeline(self): @@ -389,7 +399,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 5aa27b964..3265dfd37 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -838,11 +838,12 @@ class PageCache: Yields: Page objects """ - + layer = self.vmlinux.context.layers[self.vmlinux.layer_name] for page_addr in self._idstorage.get_entries(self._page_cache.i_pages): if not page_addr: continue - page = self.vmlinux.object("page", offset=page_addr, absolute=True) - if page: - yield page + if not layer.is_valid(page_addr): + continue + + yield self.vmlinux.object("page", offset=page_addr, absolute=True) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..187b1e280 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2489,7 +2489,12 @@ class inode(objects.StructType): """ if not self.i_size: return - elif not (self.i_mapping and self.i_mapping.nrpages > 0): + + if not ( + self.i_mapping + and self.i_mapping.is_readable() + and self.i_mapping.nrpages > 0 + ): return page_cache = linux.PageCache( @@ -2497,19 +2502,21 @@ class inode(objects.StructType): kernel_module_name="kernel", page_cache=self.i_mapping.dereference(), ) + yield from page_cache.get_cached_pages() - def get_contents(self): + def get_contents(self) -> Iterable[Tuple[int, bytes]]: """Get the inode cached pages from the page cache Yields: page_index (int): The page index in the Tree. File offset is page_index * PAGE_SIZE. - page_content (str): The page content + page_content (bytes): The page content """ for page_obj in self.get_pages(): page_index = int(page_obj.index) page_content = page_obj.get_content() - yield page_index, page_content + if page_content: + yield page_index, page_content class address_space(objects.StructType): @@ -2625,7 +2632,7 @@ class page(objects.StructType): return page_paddr - def get_content(self) -> Union[str, None]: + def get_content(self) -> Union[bytes, None]: """Returns the page content Returns: @@ -2641,8 +2648,13 @@ class page(objects.StructType): if not page_paddr: return None - page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) - return page_data + if not physical_layer.is_valid(page_paddr, length=vmlinux_layer.page_size): + vollog.debug( + "Unable to read page 0x%x content at 0x%x", self.vol.offset, page_paddr + ) + return None + + return physical_layer.read(page_paddr, vmlinux_layer.page_size) def get_flags_list(self) -> List[str]: """Returns a list of page flags @@ -2755,17 +2767,17 @@ class IDR(objects.StructType): class rb_root(objects.StructType): - def _walk_nodes(self, root_node) -> Iterator[int]: + def _walk_nodes(self, root_node: int) -> Iterator[int]: """Traverses the Red-Black tree from the root node and yields a pointer to each node in this tree. Args: - root_node: A Red-Black tree node from which to start descending + root_node: A Red-Black tree node pointer from which to start descending Yields: A pointer to every node descending from the specified root node """ - if not root_node: + if not (root_node and root_node.is_readable()): return yield root_node From 529b67fc96d7e975bdff83edd7096c4c0cd8db80 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Jan 2025 12:31:25 +1100 Subject: [PATCH 158/268] Linux: pagecache: Fix issue reported in #1527 --- volatility3/framework/plugins/linux/pagecache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 430190970..9aff0e4f9 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -469,6 +469,8 @@ class InodePages(plugins.PluginInterface): inode_size, page_idx, ) + continue + f.seek(current_fp) f.write(page_bytes) From 1a84f96c70060bc09aab3c1b3348d980f8b9bc0e Mon Sep 17 00:00:00 2001 From: Kerry Goodwine Date: Thu, 9 Jan 2025 15:49:24 -0500 Subject: [PATCH 159/268] Actions: Add new workflow for generating windows EXEs with pyinstaller --- .github/workflows/build-pyinstaller.yml | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/build-pyinstaller.yml diff --git a/.github/workflows/build-pyinstaller.yml b/.github/workflows/build-pyinstaller.yml new file mode 100644 index 000000000..bcba95403 --- /dev/null +++ b/.github/workflows/build-pyinstaller.yml @@ -0,0 +1,50 @@ +name: build-pyinstaller +on: + push: + branches: + - stable + - develop + - 'release/**' + pull_request: + branches: + - stable + - 'release/**' + +jobs: + + exe: + runs-on: windows-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pyinstaller + + - name: Pyinstall executable + run: | + pyinstaller --clean -y vol.spec + pyinstaller --clean -y volshell.spec + + - name: Move files + run: | + mv dist/vol.exe vol.exe + mv dist/volshell.exe volshell.exe + + - name: Archive + uses: actions/upload-artifact@v4 + with: + name: volatility3-pyinstaller + path: | + vol.exe + volshell.exe + README.md + LICENSE.txt From 9f08af47b161579bf31f9d45c8b248c23861388a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 12:51:52 +1100 Subject: [PATCH 160/268] Linux: Add support for Intel 32bit with PAE --- volatility3/framework/automagic/linux.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f22cae012..542d26a8d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -71,6 +71,12 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): elif "init_level4_pgt" in table.symbols: layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" + elif ( + "pkmap_count" in table.symbols + and table.get_symbol("pkmap_count").type.count == 512 + ): + layer_class = intel.LinuxIntelPAE + dtb_symbol_name = "swapper_pg_dir" else: layer_class = intel.LinuxIntel dtb_symbol_name = "swapper_pg_dir" From 28c74f8c1b853df3680de14f6fdc22958516a9c2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 13:23:39 +1100 Subject: [PATCH 161/268] Linux: Add support for Intel 32bit with PAE in early kernels, including versions 2.3.27 and 2.3.28. --- volatility3/framework/automagic/linux.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 542d26a8d..cb4f3cc64 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -71,10 +71,9 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): elif "init_level4_pgt" in table.symbols: layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" - elif ( - "pkmap_count" in table.symbols - and table.get_symbol("pkmap_count").type.count == 512 - ): + elif "pkmap_count" in table.symbols and table.get_symbol( + "pkmap_count" + ).type.count in (512, 2048): layer_class = intel.LinuxIntelPAE dtb_symbol_name = "swapper_pg_dir" else: From b27f98fed258b597e043a5c80304d8489da27b32 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 14:37:35 +1100 Subject: [PATCH 162/268] linux: pslist: fix task credentials rendering --- volatility3/framework/plugins/linux/pslist.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 37cf000fc..77b57e000 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -179,6 +179,10 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_output = "VMA start matching task start_code not found" return file_output + @staticmethod + def _format_cred(cred): + return renderers.NotAvailableValue() if cred is None else cred + def _generator( self, pid_filter: Callable[[Any], bool], @@ -212,16 +216,21 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): task_fields = self.get_task_fields(task, decorate_comm) + task_uid = self._format_cred(task_fields.uid) + task_gid = self._format_cred(task_fields.gid) + task_euid = self._format_cred(task_fields.euid) + task_egid = self._format_cred(task_fields.egid) + yield 0, ( format_hints.Hex(task_fields.offset), task_fields.user_pid, task_fields.user_tid, task_fields.user_ppid, task_fields.name, - task_fields.uid or renderers.NotAvailableValue(), - task_fields.gid or renderers.NotAvailableValue(), - task_fields.euid or renderers.NotAvailableValue(), - task_fields.egid or renderers.NotAvailableValue(), + task_uid, + task_gid, + task_euid, + task_egid, task_fields.creation_time or renderers.NotAvailableValue(), file_output, ) From 66084878627f636d0bebabbecc79842ac954d209 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 15:08:57 +1100 Subject: [PATCH 163/268] Linux: pagecache: Fix issue with incosistent inode page caches --- volatility3/framework/plugins/linux/pagecache.py | 6 +++++- volatility3/framework/symbols/linux/__init__.py | 14 ++++++++++++-- .../framework/symbols/linux/extensions/__init__.py | 9 +++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 9aff0e4f9..b2766be8d 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -520,7 +520,11 @@ class InodePages(plugins.PluginInterface): page_mapping_addr = page_obj.mapping page_index = int(page_obj.index) page_file_offset = page_index * vmlinux_layer.page_size - dump_safe = page_file_offset < inode_size + dump_safe = ( + page_file_offset < inode_size + and page_mapping_addr + and page_mapping_addr.is_readable() + ) page_flags_list = page_obj.get_flags_list() page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) fields = ( diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3265dfd37..08f69c326 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -3,6 +3,7 @@ # import math import contextlib +import logging from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union @@ -12,6 +13,8 @@ from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions +vollog = logging.getLogger(__name__) + class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): provides = {"type": "interface"} @@ -612,7 +615,7 @@ class IDStorage(ABC): raise NotImplementedError def nodep_to_node(self, nodep) -> interfaces.objects.ObjectInterface: - """Instanciates a tree node from its pointer + """Instantiates a tree node from its pointer Args: nodep: Pointer to the XArray/RadixTree node @@ -846,4 +849,11 @@ class PageCache: if not layer.is_valid(page_addr): continue - yield self.vmlinux.object("page", offset=page_addr, absolute=True) + page = self.vmlinux.object("page", offset=page_addr, absolute=True) + if not page.is_valid(): + vollog.error( + f"Invalid cached page at {page.vol.offset:#x}, aborting", + ) + break + + yield page diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 187b1e280..6ae022923 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2534,6 +2534,15 @@ class address_space(objects.StructType): class page(objects.StructType): + def is_valid(self) -> bool: + if self.mapping and not self.mapping.is_readable(): + return False + + if self.to_paddr() < 0: + return False + + return True + @functools.cached_property def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values From ce40659d8728f57f9b6e63709af4b7a17588a309 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:19:47 +1100 Subject: [PATCH 164/268] linux: radix_tree: Fix various issues, enhance early inconsistency detection, and improve compatibility with older kernel versions --- volatility3/framework/exceptions.py | 4 ++ .../framework/symbols/linux/__init__.py | 62 ++++++++++++------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index c44fb4f2e..41c67b88d 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -130,3 +130,7 @@ class OfflineException(VolatilityException): class RenderException(VolatilityException): """Thrown if there is an error during rendering""" + + +class LinuxPageCacheException(VolatilityException): + """Thrown if there is an error during Linux Page Cache processing""" diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 08f69c326..4c9934439 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -3,6 +3,7 @@ # import math import contextlib +import functools import logging from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union @@ -662,7 +663,7 @@ class IDStorage(ABC): height = self.get_tree_height(root.vol.offset) nodep = self.get_head_node(root) - if not nodep: + if not (nodep and nodep.is_readable()): return # Keep the internal flag before untagging it @@ -697,7 +698,7 @@ class XArray(IDStorage): def get_node_height(self, nodep) -> int: node = self.nodep_to_node(nodep) - return (node.shift / self.CHUNK_SHIFT) + 1 + return (node.shift // self.CHUNK_SHIFT) + 1 def get_head_node(self, tree) -> int: return tree.xa_head @@ -720,6 +721,7 @@ class RadixTree(IDStorage): RADIX_TREE_INTERNAL_NODE = 1 RADIX_TREE_EXCEPTIONAL_ENTRY = 2 RADIX_TREE_ENTRY_MASK = 3 + RADIX_TREE_MAP_SHIFT = 6 # CONFIG_BASE_FULL # Dynamic values. These will be initialized later RADIX_TREE_INDEX_BITS = None @@ -756,43 +758,57 @@ class RadixTree(IDStorage): def get_tree_height(self, treep) -> int: with contextlib.suppress(exceptions.SymbolError): if self.vmlinux.get_type("radix_tree_root").has_member("height"): - # kernels < 4.7.10 + # kernels < 4.7 d0891265bbc988dc91ed8580b38eb3dac128581b radix_tree_root = self.vmlinux.object( "radix_tree_root", offset=treep, absolute=True ) return radix_tree_root.height - # kernels >= 4.7.10 + # kernels >= 4.7 return 0 + @functools.cached_property + def _max_height_array(self): + if self.vmlinux.has_symbol("height_to_maxindex"): + # 2.6.24 26fb1589cb0aaec3a0b4418c54f30c1a2b1781f6 <= Kernels < 4.7 d0891265bbc988dc91ed8580b38eb3dac128581b + return self.vmlinux.object_from_symbol("height_to_maxindex") + elif self.vmlinux.has_symbol("height_to_maxnodes"): + # 4.8 c78c66d1ddfdbd2353f3fcfeba0268524537b096 <= kernels < 4.20 8cf2f98411e3a0865026a1061af637161b16d32b + return self.vmlinux.object_from_symbol("height_to_maxnodes") + + return None + def _radix_tree_maxindex(self, node, height) -> int: """Return the maximum key which can be store into a radix tree with this height.""" - if not self.vmlinux.has_symbol("height_to_maxindex"): - # Kernels >= 4.7 - return (self.CHUNK_SIZE << node.shift) - 1 + if self._max_height_array: + # 2.6.24 <= kernels <= 4.20 See _max_height_array() + return self._max_height_array[height] else: - # Kernels < 4.7 - height_to_maxindex_array = self.vmlinux.object_from_symbol( - "height_to_maxindex" - ) - maxindex = height_to_maxindex_array[height] - return maxindex + # Kernels >= 4.20 + return (self.CHUNK_SIZE << node.shift) - 1 def get_node_height(self, nodep) -> int: node = self.nodep_to_node(nodep) if hasattr(node, "shift"): # 4.7 <= Kernels < 4.20 - return (node.shift / self.CHUNK_SHIFT) + 1 + height = (node.shift // self.CHUNK_SHIFT) + 1 elif hasattr(node, "path"): # 3.15 <= Kernels < 4.7 - return node.path & self.RADIX_TREE_HEIGHT_MASK + height = node.path & self.RADIX_TREE_HEIGHT_MASK elif hasattr(node, "height"): # Kernels < 3.15 - return node.height + height = node.height else: raise exceptions.VolatilityException("Cannot find radix-tree node height") + if self._max_height_array and not (0 <= height < self._max_height_array.count): + error_msg = f"Radix Tree node {node.vol.offset:#x} height {height} exceeds max height of {self._max_height_array.count}" + vollog.error(error_msg) + raise exceptions.LinuxPageCacheException(error_msg) + + return height + def get_head_node(self, tree) -> int: return tree.rnode @@ -805,14 +821,16 @@ class RadixTree(IDStorage): def untag_node(self, nodep) -> int: return nodep & (~self.RADIX_TREE_ENTRY_MASK) - def is_valid_node(self, nodep) -> bool: + def _is_exceptional_node(self, nodep) -> bool: # In kernels 4.20, exceptional nodes were removed and internal entries took their bitmask - if self.vmlinux.has_type("radix_tree_root"): - return ( - nodep & self.RADIX_TREE_ENTRY_MASK - ) != self.RADIX_TREE_EXCEPTIONAL_ENTRY + return ( + self.vmlinux.has_type("radix_tree_root") + and (nodep & self.RADIX_TREE_ENTRY_MASK) + == self.RADIX_TREE_EXCEPTIONAL_ENTRY + ) - return True + def is_valid_node(self, nodep) -> bool: + return not self._is_exceptional_node(nodep) class PageCache: From 3c92d7f7b7d3d7d9548275e960571d4ac37a6a21 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:22:46 +1100 Subject: [PATCH 165/268] linux: page_cache: enhance early inconsistency detection --- volatility3/framework/symbols/linux/__init__.py | 14 ++++++-------- .../framework/symbols/linux/extensions/__init__.py | 7 ++++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 4c9934439..a7e6ef405 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -861,17 +861,15 @@ class PageCache: """ layer = self.vmlinux.context.layers[self.vmlinux.layer_name] for page_addr in self._idstorage.get_entries(self._page_cache.i_pages): - if not page_addr: - continue - if not layer.is_valid(page_addr): - continue + error_msg = f"Invalid cached page address at {page_addr:#x}, aborting" + vollog.error(error_msg) + raise exceptions.LinuxPageCacheException(error_msg) page = self.vmlinux.object("page", offset=page_addr, absolute=True) if not page.is_valid(): - vollog.error( - f"Invalid cached page at {page.vol.offset:#x}, aborting", - ) - break + error_msg = f"Invalid cached page at {page_addr:#x}, aborting" + vollog.error(error_msg) + raise exceptions.LinuxPageCacheException(error_msg) yield page diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 6ae022923..997370bbb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2513,6 +2513,11 @@ class inode(objects.StructType): page_content (bytes): The page content """ for page_obj in self.get_pages(): + if page_obj.mapping != self.i_mapping: + vollog.warning( + f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" + ) + continue page_index = int(page_obj.index) page_content = page_obj.get_content() if page_content: @@ -2524,7 +2529,7 @@ class address_space(objects.StructType): def i_pages(self): """Returns the appropriate member containing the page cache tree""" if self.has_member("i_pages"): - # Kernel >= 4.17 + # Kernel >= 4.17 b93b016313b3ba8003c3b8bb71f569af91f19fc7 return self.member("i_pages") elif self.has_member("page_tree"): # Kernel < 4.17 From c1497410b4721703ebbb9f30f2ab3db497a4438b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:28:25 +1100 Subject: [PATCH 166/268] linux: page_cache plugin: lazy file initialization and avoid redundant inode page cache walk during dumps not showing output with dumping to file. --- .../framework/plugins/linux/pagecache.py | 89 ++++++++++++------- 1 file changed, 57 insertions(+), 32 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index b2766be8d..7ad0f5a8f 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -8,7 +8,7 @@ import datetime from dataclasses import dataclass, astuple from typing import List, Set, Type, Iterable -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins from volatility3.framework.configuration import requirements @@ -453,16 +453,18 @@ class InodePages(plugins.PluginInterface): # created, saving both disk space and I/O time. # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. + inode_size = inode.i_size try: - with open_method(filename) as f: - inode_size = inode.i_size - f.truncate(inode_size) - + file_initialized = False + with open_method(filename) as file_obj: for page_idx, page_content in inode.get_contents(): current_fp = page_idx * vmlinux_layer.page_size max_length = inode_size - current_fp - page_bytes = page_content[:max_length] - if current_fp + len(page_bytes) > inode_size: + page_bytes_len = min(max_length, len(page_content)) + if ( + current_fp >= inode_size + or current_fp + page_bytes_len > inode_size + ): vollog.error( "Page out of file bounds: inode 0x%x, inode size %d, page index %d", inode.vol.offset, @@ -470,10 +472,20 @@ class InodePages(plugins.PluginInterface): page_idx, ) continue + page_bytes = page_content[:page_bytes_len] - f.seek(current_fp) - f.write(page_bytes) + if not file_initialized: + # Lazy initialization to avoid truncating the file until we are + # certain there is something to write + file_obj.truncate(inode_size) + file_initialized = True + file_obj.seek(current_fp) + file_obj.write(page_bytes) + except exceptions.LinuxPageCacheException: + vollog.error( + f"Error dumping cached pages for inode at {inode.vol.offset:#x}" + ) except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) @@ -514,31 +526,44 @@ class InodePages(plugins.PluginInterface): return None inode_size = inode.i_size - for page_obj in inode.get_pages(): - page_vaddr = page_obj.vol.offset - page_paddr = page_obj.to_paddr() - page_mapping_addr = page_obj.mapping - page_index = int(page_obj.index) - page_file_offset = page_index * vmlinux_layer.page_size - dump_safe = ( - page_file_offset < inode_size - and page_mapping_addr - and page_mapping_addr.is_readable() - ) - page_flags_list = page_obj.get_flags_list() - page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) - fields = ( - page_vaddr, - page_paddr, - page_mapping_addr, - page_index, - dump_safe, - page_flags, - ) + if not self.config["dump"]: + try: + for page_obj in inode.get_pages(): + if page_obj.mapping != inode.i_mapping: + vollog.warning( + f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" + ) + continue + page_vaddr = page_obj.vol.offset + page_paddr = page_obj.to_paddr() + page_mapping_addr = page_obj.mapping + page_index = int(page_obj.index) + page_file_offset = page_index * vmlinux_layer.page_size + dump_safe = ( + page_file_offset < inode_size + and page_mapping_addr + and page_mapping_addr.is_readable() + ) + page_flags_list = page_obj.get_flags_list() + page_flags = ",".join( + [x.replace("PG_", "") for x in page_flags_list] + ) + fields = ( + page_vaddr, + page_paddr, + page_mapping_addr, + page_index, + dump_safe, + page_flags, + ) - yield 0, fields + yield 0, fields + except exceptions.LinuxPageCacheException: + vollog.warning( + f"Page cache for inode at {inode.vol.offset:#x} is corrupt" + ) - if self.config["dump"]: + else: open_method = self.open inode_address = inode.vol.offset filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") From 5502a54198fc7617bb22e2e96519114c59704ab0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:38:41 +1100 Subject: [PATCH 167/268] linux: page_cache plugin: Refactor to make _generator more readable --- .../framework/plugins/linux/pagecache.py | 83 ++++++++++--------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 7ad0f5a8f..39ed60486 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type, Iterable +from typing import List, Set, Type, Iterable, Tuple from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.renderers import format_hints @@ -489,6 +489,44 @@ class InodePages(plugins.PluginInterface): except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) + def _generate_inode_fields( + self, + inode: interfaces.objects.ObjectInterface, + vmlinux_layer: interfaces.layers.TranslationLayerInterface, + ) -> Iterable[Tuple[int, int, int, int, bool, str]]: + inode_size = inode.i_size + try: + for page_obj in inode.get_pages(): + if page_obj.mapping != inode.i_mapping: + vollog.warning( + f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" + ) + continue + page_vaddr = page_obj.vol.offset + page_paddr = page_obj.to_paddr() + page_mapping_addr = page_obj.mapping + page_index = int(page_obj.index) + page_file_offset = page_index * vmlinux_layer.page_size + dump_safe = ( + page_file_offset < inode_size + and page_mapping_addr + and page_mapping_addr.is_readable() + ) + page_flags_list = page_obj.get_flags_list() + page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) + fields = ( + page_vaddr, + page_paddr, + page_mapping_addr, + page_index, + dump_safe, + page_flags, + ) + + yield 0, fields + except exceptions.LinuxPageCacheException: + vollog.warning(f"Page cache for inode at {inode.vol.offset:#x} is corrupt") + def _generator(self): vmlinux_module_name = self.config["kernel"] vmlinux = self.context.modules[vmlinux_module_name] @@ -510,7 +548,6 @@ class InodePages(plugins.PluginInterface): else: vollog.error("Unable to find inode with path %s", self.config["find"]) return None - elif self.config["inode"]: inode = vmlinux.object("inode", self.config["inode"], absolute=True) else: @@ -525,45 +562,7 @@ class InodePages(plugins.PluginInterface): vollog.error("The inode is not a regular file") return None - inode_size = inode.i_size - if not self.config["dump"]: - try: - for page_obj in inode.get_pages(): - if page_obj.mapping != inode.i_mapping: - vollog.warning( - f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" - ) - continue - page_vaddr = page_obj.vol.offset - page_paddr = page_obj.to_paddr() - page_mapping_addr = page_obj.mapping - page_index = int(page_obj.index) - page_file_offset = page_index * vmlinux_layer.page_size - dump_safe = ( - page_file_offset < inode_size - and page_mapping_addr - and page_mapping_addr.is_readable() - ) - page_flags_list = page_obj.get_flags_list() - page_flags = ",".join( - [x.replace("PG_", "") for x in page_flags_list] - ) - fields = ( - page_vaddr, - page_paddr, - page_mapping_addr, - page_index, - dump_safe, - page_flags, - ) - - yield 0, fields - except exceptions.LinuxPageCacheException: - vollog.warning( - f"Page cache for inode at {inode.vol.offset:#x} is corrupt" - ) - - else: + if self.config["dump"]: open_method = self.open inode_address = inode.vol.offset filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") @@ -571,6 +570,8 @@ class InodePages(plugins.PluginInterface): self.write_inode_content_to_file( inode, filename, open_method, vmlinux_layer ) + else: + yield from self._generate_inode_fields(inode, vmlinux_layer) def run(self): headers = [ From 9944fcc61f179a0d949266a90d3e0e290d5017c6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 21:15:00 +1100 Subject: [PATCH 168/268] linux: page_cache test case: Since the --dump no longer generate output, we test both modes, listing and dumping. --- test/test_volatility.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index bb7c9a851..e25e8278b 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -714,7 +714,7 @@ def test_linux_page_cache_inodepages(image, volatility, python): image, volatility, python, - pluginargs=["--inode", inode_address, "--dump"], + pluginargs=["--inode", inode_address], ) assert rc == 0 @@ -725,6 +725,14 @@ def test_linux_page_cache_inodepages(image, volatility, python): rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", out, ) + + rc, out, _err = runvol_plugin( + "linux.pagecache.InodePages", + image, + volatility, + python, + pluginargs=["--inode", inode_address, "--dump"], + ) assert os.path.exists(inode_dump_filename) with open(inode_dump_filename, "rb") as fp: inode_contents = fp.read() From d8254b63735388b9ef6be27009474ccea5726650 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 21:39:37 +1100 Subject: [PATCH 169/268] linux: page_cache test case: Improve test --- test/test_volatility.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index e25e8278b..8676d1f3e 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -708,24 +708,25 @@ def test_linux_page_cache_inodepages(image, volatility, python): inode_address = hex(0x88001AB5C270) inode_dump_filename = f"inode_{inode_address}.dmp" + + rc, out, _err = runvol_plugin( + "linux.pagecache.InodePages", + image, + volatility, + python, + pluginargs=["--inode", inode_address], + ) + + assert rc == 0 + assert out.count(b"\n") > 4 + + # PageVAddr PagePAddr MappingAddr .. DumpSafe + assert re.search( + rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", + out, + ) + try: - rc, out, _err = runvol_plugin( - "linux.pagecache.InodePages", - image, - volatility, - python, - pluginargs=["--inode", inode_address], - ) - - assert rc == 0 - assert out.count(b"\n") > 4 - - # PageVAddr PagePAddr MappingAddr .. DumpSafe - assert re.search( - rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", - out, - ) - rc, out, _err = runvol_plugin( "linux.pagecache.InodePages", image, @@ -733,6 +734,10 @@ def test_linux_page_cache_inodepages(image, volatility, python): python, pluginargs=["--inode", inode_address, "--dump"], ) + + assert rc == 0 + assert out.count(b"\n") >= 4 + assert os.path.exists(inode_dump_filename) with open(inode_dump_filename, "rb") as fp: inode_contents = fp.read() From 749a0f9c656d8da0f5087faf25f225a6097ac8b4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 16 Jan 2025 12:06:22 +0000 Subject: [PATCH 170/268] Core: Fix up ISFinfo looking for fastjsonschema --- volatility3/framework/plugins/isfinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 34b0a5653..1c2ac52e9 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -97,7 +97,7 @@ class IsfInfo(plugins.PluginInterface): if filter_item in isf_file: filtered_list.append(isf_file) - if find_spec("fastjsonschema") and self.config["validate"]: + if find_spec("jsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False" From b447bfa81c36e91c3cf30bdc432e6eba48afbc53 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:24:38 +0100 Subject: [PATCH 171/268] remove module tainting proxies --- .../framework/plugins/linux/modxview.py | 16 ++++++++++-- .../symbols/linux/extensions/__init__.py | 25 ------------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 3655200e8..69c6ac8bb 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -140,9 +140,21 @@ class Modxview(interfaces.plugins.PluginInterface): seen_addresses.add(module.vol.offset) if self.config.get("plain_taints"): - taints = module.get_taints_as_plain_string() + taints = tainting.Tainting.get_taints_as_plain_string( + self.context, + kernel_name, + module.taints, + True, + ) else: - taints = ",".join(module.get_taints_parsed()) + taints = ",".join( + tainting.Tainting.get_taints_parsed( + self.context, + kernel_name, + module.taints, + True, + ) + ) yield ( 0, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index ac2f87df0..289d6c0a4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -19,7 +19,6 @@ from volatility3.framework.layers import linear, intel from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf -from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -279,30 +278,6 @@ class module(generic.GenericIntelProcess): return None - def get_taints_as_plain_string(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Convenient wrapper around framework's Tainting capabilities. - - Returns: - The raw taints string. - """ - return tainting.Tainting( - self._context, - linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, - ).get_taints_as_plain_string(self.taints, True) - - def get_taints_parsed(self) -> List[str]: - """Convert the module's taints string to a 1-1 descriptor mapping. - Convenient wrapper around framework's Tainting capabilities. - - Returns: - A comprehensive (user-friendly) taint descriptor list. - """ - return tainting.Tainting( - self._context, - linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, - ).get_taints_parsed(self.taints, True) - @property def section_symtab(self): if self.has_member("kallsyms"): From 94704c6674d7f5fb9d57698faa0d9ed943c6158c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:26:57 +0100 Subject: [PATCH 172/268] 2.16.0 -> 2.17.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 832b2a5ba..3d68ab810 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 18 # Number of changes that only add to the interface +VERSION_MINOR = 17 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From cd8690a8059836f7e216c211c4397924ae311c84 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:27:22 +0100 Subject: [PATCH 173/268] require framework version 2.17.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 69c6ac8bb..3c2c5f05e 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 18, 0) + _required_framework_version = (2, 17, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 72056a8d0006471c2f2ed58ce36714bd0ac5fb96 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 07:28:13 +1100 Subject: [PATCH 174/268] linux: mount api: fix vfsmount pointer check --- volatility3/framework/symbols/linux/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 931c461b9..f9a3ddde5 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -139,7 +139,9 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): if not (rdentry and rdentry.is_readable() and rmnt and rmnt.is_readable()): return "" - if isinstance(vfsmnt, objects.Pointer) and not (rmnt and rmnt.is_readable()): + if isinstance(vfsmnt, objects.Pointer) and not ( + vfsmnt and vfsmnt.is_readable() + ): # vfsmnt can be the vfsmount object itself (>=3.3) or a vfsmount * (<3.3) return "" From 4d05e8a76c4b91ed9c998bce316d230614af031e Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 08:49:20 +1100 Subject: [PATCH 175/268] linux: mount API: escape '*' in docstrings to ensure correct documentation rendering --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 6d341653b..9ba3cabab 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1520,17 +1520,17 @@ class vfsmount(objects.StructType): """Helper to make sure it is comparing two pointers to 'vfsmount'. Depending on the kernel version, see 3376f34fff5be9954fd9a9c4fd68f4a0a36d480e, - the calling object (self) could be a 'vfsmount *' (<3.3) or a 'vfsmount' (>=3.3). + the calling object (self) could be a 'vfsmount \\*' (<3.3) or a 'vfsmount' (>=3.3). This way we trust in the framework "auto" dereferencing ability to assure that when we reach this point 'self' will be a 'vfsmount' already and self.vol.offset - a 'vfsmount *' and not a 'vfsmount **'. The argument must be a 'vfsmount *'. + a 'vfsmount \\*' and not a 'vfsmount \\*\\*'. The argument must be a 'vfsmount \\*'. Typically, it's called from do_get_path(). Args: vfsmount_ptr: A pointer to a 'vfsmount' Raises: - exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount *' + exceptions.VolatilityException: If vfsmount_ptr is not a 'vfsmount \\*' Returns: 'True' if the given argument points to the same 'vfsmount' as 'self'. From ad0c48e8c9871538e8e31e7047553c68ddcfc69f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 08:50:57 +1100 Subject: [PATCH 176/268] linux: mount info plugin: revert minor version increment in favor of a patch-level update --- volatility3/framework/plugins/linux/mountinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 5a0d39f31..47d8705c8 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -36,7 +36,7 @@ class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 2, 0) - _version = (1, 3, 0) + _version = (1, 2, 4) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From 50c690e280dedfeb8f64a46f908c9bddc9e06f70 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:27:07 +1100 Subject: [PATCH 177/268] linux: proc.Maps plugin: improve pointers validation --- volatility3/framework/plugins/linux/proc.py | 59 ++++++++++++--------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 441c6bc93..23d6605b7 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -21,7 +21,7 @@ class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @@ -83,18 +83,24 @@ class Maps(plugins.PluginInterface): Returns: Yields vmas based on the task and filtered based on the filter function """ - if task.mm: - for vma in task.mm.get_vma_iter(): - if filter_func(vma): - yield vma - else: - vollog.debug( - f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.pid} due to filter_func" - ) - else: + mm_pointer = task.mm + if not mm_pointer: vollog.debug( - f"Excluded pid {task.pid} as there is no mm member. It is likely a kernel thread." + f"Excluded pid {task.pid} as there is no mm member. It is likely a kernel thread" ) + return + + if not mm_pointer.is_readable(): + vollog.error(f"Task {task.pid} has an invalid mm member") + return + + for vma in mm_pointer.get_vma_iter(): + if filter_func(vma): + yield vma + else: + vollog.debug( + f"Excluded vma at offset {vma.vol.offset:#x} for pid {task.pid} due to filter_func" + ) @classmethod def vma_dump( @@ -174,31 +180,32 @@ class Maps(plugins.PluginInterface): ] # if any of the user supplied addresses would fall within this vma return true - if addrs_in_vma: - return True - else: - return False + return bool(addrs_in_vma) vma_filter_func = vma_filter_function + for task in tasks: - if not task.mm: + if not (task.mm and task.mm.is_readable()): continue name = utility.array_to_string(task.comm) for vma in self.list_vmas(task, filter_func=vma_filter_func): flags = vma.get_protection() page_offset = vma.get_page_offset() - major = 0 - minor = 0 - inode = 0 - if vma.vm_file != 0: + inode_num = None + try: dentry = vma.vm_file.get_dentry() - if dentry != 0: - inode_object = dentry.d_inode - major = inode_object.i_sb.major - minor = inode_object.i_sb.minor - inode = inode_object.i_ino + inode_ptr = dentry.d_inode + inode_num = inode_ptr.i_ino + major = inode_ptr.i_sb.major + minor = inode_ptr.i_sb.minor + except exceptions.InvalidAddressException: + if not inode_num: + inode_num = 0 + major = 0 + minor = 0 + path = vma.get_name(self.context, task) file_output = "Disabled" @@ -238,7 +245,7 @@ class Maps(plugins.PluginInterface): format_hints.Hex(page_offset), major, minor, - inode, + inode_num, path, file_output, ), From 5dc4dfcee8aca6b3e1b0a314e53e8476aa499339 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:30:59 +1100 Subject: [PATCH 178/268] linux: bpf_prog extension object: improve pointers validation --- volatility3/framework/symbols/linux/extensions/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..a3cd66238 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2023,8 +2023,11 @@ class bpf_prog(objects.StructType): prog_tag_addr = self.tag.vol.offset prog_tag_size = self.tag.count - prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) + if not vmlinux_layer.is_valid(prog_tag_addr, prog_tag_size): + vollog.debug("Unable to read bpf tag string from 0x%x", prog_tag_addr) + return None + prog_tag_bytes = vmlinux_layer.read(prog_tag_addr, prog_tag_size) prog_tag = binascii.hexlify(prog_tag_bytes).decode() return prog_tag From c28944cdd3c65c72e033d1fc344fdfa858a2545a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:33:01 +1100 Subject: [PATCH 179/268] linux:sockstat plugin and list_sockets API: improve pointers validation --- volatility3/framework/plugins/linux/sockstat.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 7376bcbee..764c04563 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -438,7 +438,7 @@ class Sockstat(plugins.PluginInterface): """Lists all network connections for all processes.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 2) + _version = (3, 0, 3) @classmethod def get_requirements(cls): @@ -514,25 +514,28 @@ class Sockstat(plugins.PluginInterface): fd_num, filp, _full_path = fd_internal.fd_fields task = fd_internal.task + if not (filp.f_op and filp.f_op.is_readable()): + continue + if filp.f_op not in (sfop_addr, dfop_addr): continue dentry = filp.get_dentry() - if not dentry: + if not (dentry and dentry.is_readable()): continue d_inode = dentry.d_inode - if not d_inode: + if not (d_inode and d_inode.is_readable()): continue socket_alloc = linux.LinuxUtilities.container_of( d_inode, "socket_alloc", "vfs_inode", vmlinux ) - socket = socket_alloc.socket - - if not (socket and socket.sk): + if not socket_alloc: + continue + socket = socket_alloc.socket + if not (socket.sk and socket.sk.is_readable()): continue - sock = socket.sk.dereference() sock_type = sock.get_type() From 77b8058913910badde4f6a48c0af568d45ceae6f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:41:25 +1100 Subject: [PATCH 180/268] linux:kmsg plugin: improve pointers validation --- volatility3/framework/plugins/linux/kmsg.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 638c2ccf2..248e37dd8 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -136,7 +136,12 @@ class ABCKmsg(ABC): """ def get_string(self, addr: int, length: int) -> str: - txt = self._context.layers[self.layer_name].read(addr, length) # type: ignore + layer = self._context.layers[self.layer_name] + if not layer.is_valid(addr, length): + return "" + + txt = layer.read(addr, length) + return txt.decode(encoding="utf8", errors="replace") def nsec_to_sec_str(self, nsec: int) -> str: @@ -281,9 +286,13 @@ class Kmsg_3_5_to_3_11(ABCKmsg): log_struct_name = self._get_log_struct_name() log_struct_size = self.vmlinux.get_type(log_struct_name).size dict_offset = msg.vol.offset + log_struct_size + msg.text_len - dict_data = self._context.layers[self.layer_name].read( - dict_offset, msg.dict_len - ) + layer = self._context.layers[self.layer_name] + try: + dict_data = layer.read(dict_offset, msg.dict_len) + except exceptions.InvalidAddressException: + vollog.debug("Unable to read kmsg dict from 0x%x", dict_offset) + return None + for chunk in dict_data.split(b"\x00"): yield " " + chunk.decode() From 583a06630f9cef5659fb5667c74911701e0e095c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:49:53 +1100 Subject: [PATCH 181/268] linux:check_syscall plugin: improve pointers validation --- volatility3/framework/plugins/linux/check_syscall.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 13d312f2f..9ffd4c497 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -82,7 +82,7 @@ class Check_syscall(plugins.PluginInterface): return table_size - def _get_table_info_disassembly(self, ptr_sz, vmlinux): + def _get_table_info_disassembly(self, ptr_sz, vmlinux) -> int: """Find the size of the system call table by disassembling functions that immediately reference it in their first instruction This is in the form 'cmp reg,NR_syscalls'.""" @@ -107,9 +107,13 @@ class Check_syscall(plugins.PluginInterface): return 0 vmlinux = self.context.modules[self.config["kernel"]] - data = self.context.layers.read(vmlinux.layer_name, func_addr, 6) + vmlinux_layer = self.context.layers[vmlinux.layer_name] + try: + data = vmlinux_layer.read(func_addr, 6) + except exceptions.InvalidAddressException: + return 0 - for address, size, mnemonic, op_str in md.disasm_lite(data, func_addr): + for _address, _size, mnemonic, op_str in md.disasm_lite(data, func_addr): if mnemonic == "CMP": table_size = int(op_str.split(",")[1].strip()) & 0xFFFF break From 6dac7195dc88518fa8b40a70d57e855d40b30b01 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 17 Jan 2025 09:50:53 +1100 Subject: [PATCH 182/268] linux: elf_linkmap object extension: Use a more generic invalid address exception --- volatility3/framework/symbols/linux/extensions/elf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index eadcbbae0..fb5f89f60 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -437,7 +437,7 @@ class elf_linkmap(objects.StructType): def get_name(self): try: buf = self._context.layers.read(self.vol.layer_name, self.l_name, 256) - except exceptions.PagedInvalidAddressException: + except exceptions.InvalidAddressException: # Protection against memory smear vollog.log( constants.LOGLEVEL_VVVV, From adf81bc74a388d6ff5bffabe588bc5ca72147506 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 16 Jan 2025 19:59:31 +0000 Subject: [PATCH 183/268] Update copyright dates --- README.md | 2 +- doc/source/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cc33d3cc4..b74bdab0b 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ The latest generated copy of the documentation can be found at: Date: Fri, 17 Jan 2025 16:03:47 +0000 Subject: [PATCH 184/268] Core: Correct version dependencies to avoid conflicts Fixes #1546 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86e3921d2..542a1480a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ cloud = [ dev = [ "volatility3[full,cloud]", "jsonschema>=4.23.0,<5", - "pyinstaller>=6.11.0,<7", + "pyinstaller>=6.5.0,<7", "pyinstaller-hooks-contrib>=2024.9", "types-jsonschema>=4.23.0,<5", ] @@ -48,7 +48,7 @@ test = [ docs = [ "volatility3[dev]", - "sphinx>=8.0.0,<7", + "sphinx>=8.0.0,<9", "sphinx-autodoc-typehints>=2.5.0,<3", "sphinx-rtd-theme>=3.0.1,<4", ] From c4430cda8d6b13b0d69a787fe32e8158ae471c3a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 17 Jan 2025 16:09:35 +0000 Subject: [PATCH 185/268] Core: Try to maintain python-3.8 support for the documentation --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 542a1480a..8944bb058 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ test = [ docs = [ "volatility3[dev]", "sphinx>=8.0.0,<9", - "sphinx-autodoc-typehints>=2.5.0,<3", + "sphinx-autodoc-typehints>=2.0.0,<3", "sphinx-rtd-theme>=3.0.1,<4", ] From 13a8c53f7b64bc7180b665e363b2a3f0348e8b04 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 17 Jan 2025 16:13:35 +0000 Subject: [PATCH 186/268] Core: There was no clear reason to stop supporting older versions of sphinx --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8944bb058..3f16eeece 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ test = [ docs = [ "volatility3[dev]", - "sphinx>=8.0.0,<9", + "sphinx>=4.0.0,<9", "sphinx-autodoc-typehints>=2.0.0,<3", "sphinx-rtd-theme>=3.0.1,<4", ] From cc9486cf03f6f8b8035069f02702ce30a589cb7e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 02:19:11 +0100 Subject: [PATCH 187/268] pre-process module triaging to improve readability --- .../framework/plugins/linux/modxview.py | 71 ++++++++++--------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 3c2c5f05e..125c1cc33 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -82,7 +82,8 @@ class Modxview(interfaces.plugins.PluginInterface): kernel_name: str, run_hidden_modules: bool = True, ) -> Dict[str, List[extensions.module]]: - """Run module scanning plugins and aggregate the results. + """Run module scanning plugins and aggregate the results. It is designed + to not operate any inter-plugin results triage. Args: run_hidden_modules: specify if the hidden_modules plugin should be run @@ -128,46 +129,50 @@ class Modxview(interfaces.plugins.PluginInterface): def _generator(self): kernel_name = self.config["kernel"] run_results = self.run_modules_scanners(self.context, kernel_name) - modules_offsets = {} - for key in ["lsmod", "check_modules", "hidden_modules"]: - modules_offsets[key] = set(module.vol.offset for module in run_results[key]) + aggregated_modules = {} + # We want to be explicit on the plugins results we are interested in + for plugin_name in ["lsmod", "check_modules", "hidden_modules"]: + # Iterate over each recovered module + for module in run_results[plugin_name]: + # Use offsets as unique keys, whether a module + # appears in many plugin runs or not + if aggregated_modules.get(module.vol.offset): + # Append the plugin to the list of originating plugins + aggregated_modules[module.vol.offset][1].append(plugin_name) + else: + aggregated_modules[module.vol.offset] = (module, [plugin_name]) - seen_addresses = set() - for modules_list in run_results.values(): - for module in modules_list: - if module.vol.offset in seen_addresses: - continue - seen_addresses.add(module.vol.offset) - - if self.config.get("plain_taints"): - taints = tainting.Tainting.get_taints_as_plain_string( + for module_offset, (module, originating_plugins) in aggregated_modules.items(): + # Tainting parsing capabilities applied to the module + if self.config.get("plain_taints"): + taints = tainting.Tainting.get_taints_as_plain_string( + self.context, + kernel_name, + module.taints, + True, + ) + else: + taints = ",".join( + tainting.Tainting.get_taints_parsed( self.context, kernel_name, module.taints, True, ) - else: - taints = ",".join( - tainting.Tainting.get_taints_parsed( - self.context, - kernel_name, - module.taints, - True, - ) - ) - - yield ( - 0, - ( - module.get_name() or NotAvailableValue(), - format_hints.Hex(module.vol.offset), - module.vol.offset in modules_offsets["lsmod"], - module.vol.offset in modules_offsets["check_modules"], - module.vol.offset in modules_offsets["hidden_modules"], - taints or NotAvailableValue(), - ), ) + yield ( + 0, + ( + module.get_name() or NotAvailableValue(), + format_hints.Hex(module_offset), + "lsmod" in originating_plugins, + "check_modules" in originating_plugins, + "hidden_modules" in originating_plugins, + taints or NotAvailableValue(), + ), + ) + def run(self): columns = [ ("Name", str), From 3b679cbafbb50a2c986a63efd223cf9088bbc330 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:26:43 +0100 Subject: [PATCH 188/268] explicit None check --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 125c1cc33..c74bf28e8 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -136,7 +136,7 @@ class Modxview(interfaces.plugins.PluginInterface): for module in run_results[plugin_name]: # Use offsets as unique keys, whether a module # appears in many plugin runs or not - if aggregated_modules.get(module.vol.offset): + if aggregated_modules.get(module.vol.offset, None) is not None: # Append the plugin to the list of originating plugins aggregated_modules[module.vol.offset][1].append(plugin_name) else: From bb6556dbc0145682866d56bb2608b5b841e381e8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:37:52 +0100 Subject: [PATCH 189/268] correct arguments for pre_4_10_rc1 --- volatility3/framework/symbols/linux/utilities/tainting.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 552f51b98..14b69d3d6 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -117,9 +117,7 @@ class Tainting(interfaces.configuration.VersionableInterface): return cls._module_flags_taint_post_4_10_rc1( context, kernel_module_name, taints, is_module ) - return cls._module_flags_taint_pre_4_10_rc1( - context, kernel_module_name, taints, is_module - ) + return cls._module_flags_taint_pre_4_10_rc1(taints, is_module) @classmethod def get_taints_parsed( From 0b82f731375583076abdfabd332ce067612d69f5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:45:30 +0100 Subject: [PATCH 190/268] functools caching and doc. --- .../framework/symbols/linux/utilities/tainting.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 14b69d3d6..c1136436e 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -1,3 +1,5 @@ +import functools + from volatility3 import framework from volatility3.framework import interfaces from volatility3.framework.constants import linux as linux_constants @@ -18,11 +20,18 @@ class Tainting(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) @classmethod + @functools.lru_cache def _get_kernel_taint_flags_list( cls, context: interfaces.context.ContextInterface, kernel_module_name: str, ) -> Optional[List[interfaces.objects.ObjectInterface]]: + """Determine whether the kernel embeds taint flags definition + in-memory or not. + + Returns: + A list of "taint_flag" kernel objects if taint_flags symbok exists + """ kernel = context.modules[kernel_module_name] if kernel.has_symbol("taint_flags"): return list(kernel.object_from_symbol("taint_flags")) From 8095924e8a926990f6002f16d2c7259c5c750980 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:46:13 +0100 Subject: [PATCH 191/268] typo --- volatility3/framework/symbols/linux/utilities/tainting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index c1136436e..2360401d5 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -30,7 +30,7 @@ class Tainting(interfaces.configuration.VersionableInterface): in-memory or not. Returns: - A list of "taint_flag" kernel objects if taint_flags symbok exists + A list of "taint_flag" kernel objects if taint_flags symbol exists """ kernel = context.modules[kernel_module_name] if kernel.has_symbol("taint_flags"): From 2fe8ee5983bd5faf8a89db5712512aa411329dbc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 18 Jan 2025 13:18:53 +0000 Subject: [PATCH 192/268] Layers: Update LeechCore RawIO with better error handling for readlines Fixes #1419 --- volatility3/framework/layers/leechcore.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index eeede1673..06c359203 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -129,6 +129,8 @@ if HAS_LEECHCORE: def readline(self, __size: Optional[int] = ...) -> bytes: data = b"" + if not __size: + __size = 0 while __size > self._chunk_size or __size < 0: data += self.read(self._chunk_size) index = data.find(b"\n") From 0849c163a1c517fa8595f9cc7610a737d1904fc2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:48:37 +0100 Subject: [PATCH 193/268] appropriate symbols type hinting --- volatility3/framework/contexts/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index f527544c0..17a91e827 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -337,7 +337,7 @@ class Module(interfaces.context.ModuleInterface): ) @property - def symbols(self): + def symbols(self) -> Iterable[str]: return self.context.symbol_space[self.symbol_table_name].symbols get_symbol = get_module_wrapper("get_symbol") From d46cb3328d07ae2216045ba3fc33679c2ab13fbc Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:48:46 +0100 Subject: [PATCH 194/268] appropriate symbols type hinting --- volatility3/framework/interfaces/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index a87e0f1e8..2b95a18ad 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -303,8 +303,8 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): """Determines whether an enumeration is present in the module's symbol table.""" @abstractmethod - def symbols(self) -> List: - """Lists the symbols contained in the symbol table for this module""" + def symbols(self) -> Iterable[str]: + """Returns an iterable of the symbols contained in the symbol table for this module""" @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: From fb93d2333b8d3854d348548decc76ac67f358699 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:50:14 +0100 Subject: [PATCH 195/268] improve comments --- volatility3/framework/interfaces/symbols.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index b8712e38d..c0bebe1e2 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -122,7 +122,7 @@ class BaseSymbolTableInterface: @property def symbols(self) -> Iterable[str]: - """Returns an iterator of the Symbol names.""" + """Returns an iterable of the available symbol names.""" raise NotImplementedError( "Abstract property symbols not implemented by subclass." ) @@ -131,7 +131,7 @@ class BaseSymbolTableInterface: @property def types(self) -> Iterable[str]: - """Returns an iterator of the Symbol type names.""" + """Returns an iterable of the available symbol type names.""" raise NotImplementedError( "Abstract property types not implemented by subclass." ) @@ -149,7 +149,7 @@ class BaseSymbolTableInterface: @property def enumerations(self) -> Iterable[Any]: - """Returns an iterator of the Enumeration names.""" + """Returns an iterable of the available enumerations names.""" raise NotImplementedError( "Abstract property enumerations not implemented by subclass." ) @@ -366,6 +366,7 @@ class NativeTableInterface(BaseSymbolTableInterface): @property def symbols(self) -> Iterable[str]: + """Returns an iterable of the available symbol names.""" return [] def get_enumeration(self, name: str) -> objects.Template: From 1e9551b0530be824ab8d9a40db57cbd813d48136 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:50:28 +0100 Subject: [PATCH 196/268] types base class and comments improvements --- volatility3/framework/interfaces/symbols.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index c0bebe1e2..752d288f7 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -375,7 +375,13 @@ class NativeTableInterface(BaseSymbolTableInterface): ) @property - def enumerations(self) -> Iterable[str]: + def enumerations(self) -> Iterable[Any]: + """Returns an iterable of the available enumerations.""" + return [] + + @property + def types(self) -> Iterable[str]: + """Returns an iterable of the available symbol type names.""" return [] From aa99410dd2c50ee556293db959c469739c882684 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:51:31 +0100 Subject: [PATCH 197/268] prefer KeysView iterable to lists --- volatility3/framework/symbols/intermed.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 6802af7d6..0a30148aa 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -411,18 +411,23 @@ class Version1Format(ISFormatTable): @property def symbols(self) -> Iterable[str]: - """Returns an iterator of the symbol names.""" - return list(self._json_object.get("symbols", {})) + """Returns an iterable (KeysView) of the available symbol names.""" + return self._json_object.get("symbols", {}).keys() @property - def enumerations(self) -> Iterable[str]: - """Returns an iterator of the available enumerations.""" - return list(self._json_object.get("enums", {})) + def enumerations(self) -> Iterable[Any]: + """Returns an iterable (KeysView) of the available enumerations.""" + return self._json_object.get("enums", {}).keys() @property - def types(self) -> Iterable[str]: - """Returns an iterator of the symbol type names.""" - return list(self._json_object.get("user_types", {})) + list(self.natives.types) + def types(self): + """Returns an iterable (KeysView) of the available symbol type names.""" + # self.natives.types (set) is generally very small compared to user_types, + # so the dict conversion overhead can be neglected + return { + **self._json_object.get("user_types", {}), + **dict.fromkeys(self.natives.types), + }.keys() def get_type_class(self, name: str) -> Type[interfaces.objects.ObjectInterface]: return self._overrides.get(name, objects.AggregateType) From 3a2933155b6f92a8585666f611cd2069424ce5a9 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:51:52 +0100 Subject: [PATCH 198/268] improve comments --- volatility3/framework/symbols/native.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/native.py b/volatility3/framework/symbols/native.py index 7c3e1b312..61417532e 100644 --- a/volatility3/framework/symbols/native.py +++ b/volatility3/framework/symbols/native.py @@ -30,7 +30,7 @@ class NativeTable(interfaces.symbols.NativeTableInterface): @property def types(self) -> Iterable[str]: - """Returns an iterator of the symbol type names.""" + """Returns an iterable (set) of the available symbol type names.""" return self._types def get_type(self, type_name: str) -> interfaces.objects.Template: From ba09db6952d37590f62a647265c2bb5bb903ec3c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:54:04 +0100 Subject: [PATCH 199/268] improve comments --- volatility3/framework/interfaces/symbols.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index 752d288f7..2d142de9a 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -149,7 +149,7 @@ class BaseSymbolTableInterface: @property def enumerations(self) -> Iterable[Any]: - """Returns an iterable of the available enumerations names.""" + """Returns an iterable of the available enumerations.""" raise NotImplementedError( "Abstract property enumerations not implemented by subclass." ) From 4b3d93b0f0637c7d41acc545f397e3522a913978 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:00:03 +0100 Subject: [PATCH 200/268] 2.17.0 -> 2.18.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 3d68ab810..832b2a5ba 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 17 # Number of changes that only add to the interface +VERSION_MINOR = 18 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From c5628c5d79496ae051942598bab08c19d3632a18 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:48:44 +0100 Subject: [PATCH 201/268] revert the mistakenly removed types type hinting --- volatility3/framework/symbols/intermed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 0a30148aa..9ece69d8b 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -420,7 +420,7 @@ class Version1Format(ISFormatTable): return self._json_object.get("enums", {}).keys() @property - def types(self): + def types(self) -> Iterable[str]: """Returns an iterable (KeysView) of the available symbol type names.""" # self.natives.types (set) is generally very small compared to user_types, # so the dict conversion overhead can be neglected From 0ee016e65554539318705a5b7c292864fcc2f436 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:50:28 +0100 Subject: [PATCH 202/268] 2.17.0 -> 2.17.1 bump --- volatility3/framework/constants/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 832b2a5ba..041439909 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 18 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_MINOR = 17 # Number of changes that only add to the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From c2ef3c2fe575f2c3ea49541b7e1207e7d93884f1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:59:51 +0100 Subject: [PATCH 203/268] add fixme about merge operator --- volatility3/framework/symbols/intermed.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 9ece69d8b..cb0b67969 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -422,8 +422,12 @@ class Version1Format(ISFormatTable): @property def types(self) -> Iterable[str]: """Returns an iterable (KeysView) of the available symbol type names.""" - # self.natives.types (set) is generally very small compared to user_types, - # so the dict conversion overhead can be neglected + # We use ** instead of + # `set(self._json_object.get("user_types", {}).keys()).union(self.natives.types)` + # because converting user_types dict to a set is costly. + # It is more efficient to convert the (very small) self.natives.types set to a dict. + # FIXME: On Python3.8 support drop, merge the two dicts using the merge operator: + # (self._json_object.get("user_types", {}) | dict.fromkeys(self.natives.types)).keys() return { **self._json_object.get("user_types", {}), **dict.fromkeys(self.natives.types), From 726fbe6ccec8480c1baa74fadcc7fc4e87377bcd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 13:01:14 +0100 Subject: [PATCH 204/268] 2.17.1 -> 2.18.0 bump --- volatility3/framework/constants/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 041439909..832b2a5ba 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 17 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 18 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From 08830fee05c30adfc5ea4e997ce317f6f6927033 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:06:42 +0100 Subject: [PATCH 205/268] use architectures.LINUX_ARCHS --- volatility3/framework/plugins/linux/pagecache.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 382268515..408a9b98a 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -8,6 +8,7 @@ import datetime from dataclasses import dataclass, astuple from typing import List, Set, Type, Iterable +from volatility3.framework.constants import architectures from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins @@ -112,7 +113,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=["Intel32", "Intel64"], + architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 2, 0) @@ -397,7 +398,7 @@ class InodePages(plugins.PluginInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=["Intel32", "Intel64"], + architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( name="files", plugin=Files, version=(1, 0, 0) From d325e1ca176b55f68de03f462ca855631736ef6c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:08:27 +0100 Subject: [PATCH 206/268] add inode_size and format_symlink to Inode* dataclasses --- volatility3/framework/plugins/linux/pagecache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 408a9b98a..c86664f3d 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -38,6 +38,11 @@ class InodeUser: modification_time: str change_time: str path: str + inode_size: int + + @staticmethod + def format_symlink(symlink_source: str, symlink_dest: str): + return f"{symlink_source} -> {symlink_dest}" @dataclass @@ -81,6 +86,7 @@ class InodeInternal: access_time_dt = self.inode.get_access_time() modification_time_dt = self.inode.get_modification_time() change_time_dt = self.inode.get_change_time() + inode_size = int(self.inode.i_size) inode_user = InodeUser( superblock_addr=superblock_addr, @@ -96,6 +102,7 @@ class InodeInternal: modification_time=modification_time_dt, change_time=change_time_dt, path=self.path, + inode_size=inode_size, ) return inode_user From 1d0159325fbf04bb5029a9ed4ae2ea1acc770cee Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:09:20 +0100 Subject: [PATCH 207/268] switch to InodeUser.format_symlink --- volatility3/framework/plugins/linux/pagecache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index c86664f3d..a0dd8efe4 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -156,10 +156,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): """ # i_link (fast symlinks) were introduced in 4.2 if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: - i_link_str = inode.i_link.dereference().cast( + symlink_dest = inode.i_link.dereference().cast( "string", max_length=255, encoding="utf-8", errors="replace" ) - symlink_path = f"{symlink_path} -> {i_link_str}" + symlink_path = InodeUser.format_symlink(symlink_path, symlink_dest) return symlink_path From 48a8f3929bbe992ff687a494e3ea0e6a7446f42c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:10:32 +0100 Subject: [PATCH 208/268] add and leverage follow_symlinks parameter --- volatility3/framework/plugins/linux/pagecache.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index a0dd8efe4..4871d0d0f 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -220,12 +220,14 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, + follow_symlinks: bool = True, ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: context: The context that the plugin will operate within vmlinux_module_name: The name of the kernel module on which to operate + follow_symlinks: Whether to follow symlinks or not Yields: An InodeInternal object @@ -297,7 +299,9 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): continue seen_inodes.add(file_inode_ptr) - file_path = cls._follow_symlink(file_inode_ptr, file_path) + if follow_symlinks: + file_path = cls._follow_symlink(file_inode_ptr, file_path) + inode_in = InodeInternal( superblock=superblock, mountpoint=mountpoint, From 6f2ff4f7c6f702c11bf7d75b5ab78cb8f5881a20 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:11:03 +0100 Subject: [PATCH 209/268] add InodeSize column to Files --- volatility3/framework/plugins/linux/pagecache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 4871d0d0f..770c6391a 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -389,6 +389,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): ("ModificationTime", datetime.datetime), ("ChangeTime", datetime.datetime), ("FilePath", str), + ("InodeSize", int), ] return renderers.TreeGrid( From 85941060051b530350ad8e770de4c1f2d3edefee Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:12:10 +0100 Subject: [PATCH 210/268] 1.0.1 -> 1.2.0 Files bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 770c6391a..d57bd77f8 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -112,7 +112,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 2, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From e8b44efc17dbfd6e412436d324702ed3e4cc7c7f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:13:26 +0100 Subject: [PATCH 211/268] add and leverage write_inode_content_to_stream --- .../framework/plugins/linux/pagecache.py | 60 ++++++++++++------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index d57bd77f8..42fa7538f 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type, Iterable +from typing import List, Set, Type, Iterable, IO from volatility3.framework.constants import architectures from volatility3.framework import renderers, interfaces @@ -452,31 +452,47 @@ class InodePages(plugins.PluginInterface): vollog.error("The inode is not a regular file") return None - # By using truncate/seek, provided the filesystem supports it, a sparse file will be + try: + with open_method(filename) as f: + InodePages.write_inode_content_to_stream(inode, f, vmlinux_layer) + except OSError as e: + vollog.error("Unable to write to file (%s): %s", filename, e) + + @staticmethod + def write_inode_content_to_stream( + inode: interfaces.objects.ObjectInterface, + stream: IO, + vmlinux_layer: interfaces.layers.TranslationLayerInterface, + ) -> None: + """Extracts the inode's contents from the page cache and saves them to a stream + + Args: + inode: The inode to dump + stream: A IO steam to write to, typically FileHandlerInterface or BytesIO + vmlinux_layer: The kernel layer to obtain the page size + """ + + # By using truncate/seek, provided the filesystem supports it, and the + # stream is a File interface, a sparse file will be # created, saving both disk space and I/O time. # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. - try: - with open_method(filename) as f: - inode_size = inode.i_size - f.truncate(inode_size) + inode_size = inode.i_size + stream.truncate(inode_size) - for page_idx, page_content in inode.get_contents(): - current_fp = page_idx * vmlinux_layer.page_size - max_length = inode_size - current_fp - page_bytes = page_content[:max_length] - if current_fp + len(page_bytes) > inode_size: - vollog.error( - "Page out of file bounds: inode 0x%x, inode size %d, page index %d", - inode.vol.offset, - inode_size, - page_idx, - ) - f.seek(current_fp) - f.write(page_bytes) - - except OSError as e: - vollog.error("Unable to write to file (%s): %s", filename, e) + for page_idx, page_content in inode.get_contents(): + current_fp = page_idx * vmlinux_layer.page_size + max_length = inode_size - current_fp + page_bytes = page_content[:max_length] + if current_fp + len(page_bytes) > inode_size: + vollog.error( + "Page out of file bounds: inode 0x%x, inode size %d, page index %d", + inode.vol.offset, + inode_size, + page_idx, + ) + stream.seek(current_fp) + stream.write(page_bytes) def _generator(self): vmlinux_module_name = self.config["kernel"] From 818ddb746bb5bcbf45fe22fb54e8022d75d5fedd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:15:55 +0100 Subject: [PATCH 212/268] 2.0.0 -> 2.1.0 InodePages bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 42fa7538f..feac31bb7 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -402,7 +402,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From da068676d13eeefae033e46f8a916d91881bc2a6 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:16:09 +0100 Subject: [PATCH 213/268] typo --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index feac31bb7..e87bc2c9d 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -468,7 +468,7 @@ class InodePages(plugins.PluginInterface): Args: inode: The inode to dump - stream: A IO steam to write to, typically FileHandlerInterface or BytesIO + stream: An IO steam to write to, typically FileHandlerInterface or BytesIO vmlinux_layer: The kernel layer to obtain the page size """ From 452d6e705b973213238becd22d736f6ee1cb45e0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 16:26:25 +0100 Subject: [PATCH 214/268] 1.0.1 -> 1.1.0 Files bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index e87bc2c9d..89d30a9eb 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -112,7 +112,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 2, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From ecae545f5897e7c9a59d231d0b3f04b195cbf169 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:05:52 +0100 Subject: [PATCH 215/268] typo --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7d211150a..850045244 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2948,7 +2948,7 @@ class scatterlist(objects.StructType): Returns: An iterator of bytes """ - # Either "physical" is layer-1 because this is a module layer, either "physical" is the current layer + # Either "physical" is layer-1 because this is a module layer, or "physical" is the current layer physical_layer_name = self._context.layers[self.vol.layer_name].config.get( "memory_layer", self.vol.layer_name ) From eddba98ef7c8c72162233dd19332e770d2a916d7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:09:59 +0100 Subject: [PATCH 216/268] type hint format_symlink --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 89d30a9eb..d98103368 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -41,7 +41,7 @@ class InodeUser: inode_size: int @staticmethod - def format_symlink(symlink_source: str, symlink_dest: str): + def format_symlink(symlink_source: str, symlink_dest: str) -> str: return f"{symlink_source} -> {symlink_dest}" From 59703045c78941a12087a295b18cc8bc98414a06 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:26:42 +0100 Subject: [PATCH 217/268] switch calling convention to context and layer name --- .../framework/plugins/linux/pagecache.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index d98103368..d5297ab10 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -435,18 +435,20 @@ class InodePages(plugins.PluginInterface): @staticmethod def write_inode_content_to_file( + context: interfaces.context.ContextInterface, + layer_name: str, inode: interfaces.objects.ObjectInterface, filename: str, open_method: Type[interfaces.plugins.FileHandlerInterface], - vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a file Args: + context: The context on which to operate + layer_name: The name of the layer on which to operate inode: The inode to dump filename: Filename for writing the inode content open_method: class for constructing output files - vmlinux_layer: The kernel layer to obtain the page size """ if not inode.is_reg: vollog.error("The inode is not a regular file") @@ -454,24 +456,26 @@ class InodePages(plugins.PluginInterface): try: with open_method(filename) as f: - InodePages.write_inode_content_to_stream(inode, f, vmlinux_layer) + InodePages.write_inode_content_to_stream(context, layer_name, inode, f) except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) @staticmethod def write_inode_content_to_stream( + context: interfaces.context.ContextInterface, + layer_name: str, inode: interfaces.objects.ObjectInterface, stream: IO, - vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a stream Args: + context: The context on which to operate + layer_name: The name of the layer on which to operate inode: The inode to dump stream: An IO steam to write to, typically FileHandlerInterface or BytesIO - vmlinux_layer: The kernel layer to obtain the page size """ - + layer = context.layers[layer_name] # By using truncate/seek, provided the filesystem supports it, and the # stream is a File interface, a sparse file will be # created, saving both disk space and I/O time. @@ -481,7 +485,7 @@ class InodePages(plugins.PluginInterface): stream.truncate(inode_size) for page_idx, page_content in inode.get_contents(): - current_fp = page_idx * vmlinux_layer.page_size + current_fp = page_idx * layer.page_size max_length = inode_size - current_fp page_bytes = page_content[:max_length] if current_fp + len(page_bytes) > inode_size: @@ -557,7 +561,7 @@ class InodePages(plugins.PluginInterface): filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") vollog.info("[*] Writing inode at 0x%x to '%s'", inode_address, filename) self.write_inode_content_to_file( - inode, filename, open_method, vmlinux_layer + self.context, vmlinux_layer.name, inode, filename, open_method ) def run(self): From a02243bb3c4a14076cda7a516c7499e1734f19d0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:27:27 +0100 Subject: [PATCH 218/268] 2.1.0 -> 3.0.0 InodePages bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index d5297ab10..a86c1b936 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -402,7 +402,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 1, 0) + _version = (3, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From d42ffc01e22672602a346b21f90b3733cd02db3e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:33:47 +0100 Subject: [PATCH 219/268] typo --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index a86c1b936..77aa42338 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -473,7 +473,7 @@ class InodePages(plugins.PluginInterface): context: The context on which to operate layer_name: The name of the layer on which to operate inode: The inode to dump - stream: An IO steam to write to, typically FileHandlerInterface or BytesIO + stream: An IO stream to write to, typically FileHandlerInterface or BytesIO """ layer = context.layers[layer_name] # By using truncate/seek, provided the filesystem supports it, and the From 74b98e62c7aa4ca721f48651243122fa222548a5 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 19 Jan 2025 23:42:41 +0000 Subject: [PATCH 220/268] Revert "Add missing exception handling in env var recovery. Prevent backtraces" --- volatility3/framework/plugins/linux/envars.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 04b75c8a8..8cdbfe493 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -5,7 +5,7 @@ import logging from typing import Iterable, Tuple -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -58,16 +58,10 @@ class Envars(plugins.PluginInterface): Tuples of (key, value) representing each environment variable. """ - # This ensures the `task` is valid as well as its - # memory mapping structures - try: - task_name = utility.array_to_string(task.comm) - env_start = task.mm.env_start - env_end = task.mm.env_end - except exceptions.InvalidAddressException: - return None - + task_name = utility.array_to_string(task.comm) task_pid = task.pid + env_start = task.mm.env_start + env_end = task.mm.env_end env_area_size = env_end - env_start if not (0 < env_area_size <= env_area_max_size): vollog.debug( From e18bffdde95e4adb0ee89aeb28b3845c53b37687 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 19 Jan 2025 23:51:28 +0000 Subject: [PATCH 221/268] Revert "Pre linux.pagecache.recoverfs support" --- .../framework/plugins/linux/pagecache.py | 97 ++++++------------- 1 file changed, 32 insertions(+), 65 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 77aa42338..382268515 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,9 +6,8 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type, Iterable, IO +from typing import List, Set, Type, Iterable -from volatility3.framework.constants import architectures from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins @@ -38,11 +37,6 @@ class InodeUser: modification_time: str change_time: str path: str - inode_size: int - - @staticmethod - def format_symlink(symlink_source: str, symlink_dest: str) -> str: - return f"{symlink_source} -> {symlink_dest}" @dataclass @@ -86,7 +80,6 @@ class InodeInternal: access_time_dt = self.inode.get_access_time() modification_time_dt = self.inode.get_modification_time() change_time_dt = self.inode.get_change_time() - inode_size = int(self.inode.i_size) inode_user = InodeUser( superblock_addr=superblock_addr, @@ -102,7 +95,6 @@ class InodeInternal: modification_time=modification_time_dt, change_time=change_time_dt, path=self.path, - inode_size=inode_size, ) return inode_user @@ -112,7 +104,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -120,7 +112,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=architectures.LINUX_ARCHS, + architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 2, 0) @@ -156,10 +148,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): """ # i_link (fast symlinks) were introduced in 4.2 if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: - symlink_dest = inode.i_link.dereference().cast( + i_link_str = inode.i_link.dereference().cast( "string", max_length=255, encoding="utf-8", errors="replace" ) - symlink_path = InodeUser.format_symlink(symlink_path, symlink_dest) + symlink_path = f"{symlink_path} -> {i_link_str}" return symlink_path @@ -220,14 +212,12 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, - follow_symlinks: bool = True, ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: context: The context that the plugin will operate within vmlinux_module_name: The name of the kernel module on which to operate - follow_symlinks: Whether to follow symlinks or not Yields: An InodeInternal object @@ -299,9 +289,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): continue seen_inodes.add(file_inode_ptr) - if follow_symlinks: - file_path = cls._follow_symlink(file_inode_ptr, file_path) - + file_path = cls._follow_symlink(file_inode_ptr, file_path) inode_in = InodeInternal( superblock=superblock, mountpoint=mountpoint, @@ -389,7 +377,6 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): ("ModificationTime", datetime.datetime), ("ChangeTime", datetime.datetime), ("FilePath", str), - ("InodeSize", int), ] return renderers.TreeGrid( @@ -402,7 +389,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -410,7 +397,7 @@ class InodePages(plugins.PluginInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=architectures.LINUX_ARCHS, + architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( name="files", plugin=Files, version=(1, 0, 0) @@ -435,68 +422,48 @@ class InodePages(plugins.PluginInterface): @staticmethod def write_inode_content_to_file( - context: interfaces.context.ContextInterface, - layer_name: str, inode: interfaces.objects.ObjectInterface, filename: str, open_method: Type[interfaces.plugins.FileHandlerInterface], + vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a file Args: - context: The context on which to operate - layer_name: The name of the layer on which to operate inode: The inode to dump filename: Filename for writing the inode content open_method: class for constructing output files + vmlinux_layer: The kernel layer to obtain the page size """ if not inode.is_reg: vollog.error("The inode is not a regular file") return None - try: - with open_method(filename) as f: - InodePages.write_inode_content_to_stream(context, layer_name, inode, f) - except OSError as e: - vollog.error("Unable to write to file (%s): %s", filename, e) - - @staticmethod - def write_inode_content_to_stream( - context: interfaces.context.ContextInterface, - layer_name: str, - inode: interfaces.objects.ObjectInterface, - stream: IO, - ) -> None: - """Extracts the inode's contents from the page cache and saves them to a stream - - Args: - context: The context on which to operate - layer_name: The name of the layer on which to operate - inode: The inode to dump - stream: An IO stream to write to, typically FileHandlerInterface or BytesIO - """ - layer = context.layers[layer_name] - # By using truncate/seek, provided the filesystem supports it, and the - # stream is a File interface, a sparse file will be + # By using truncate/seek, provided the filesystem supports it, a sparse file will be # created, saving both disk space and I/O time. # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. - inode_size = inode.i_size - stream.truncate(inode_size) + try: + with open_method(filename) as f: + inode_size = inode.i_size + f.truncate(inode_size) - for page_idx, page_content in inode.get_contents(): - current_fp = page_idx * layer.page_size - max_length = inode_size - current_fp - page_bytes = page_content[:max_length] - if current_fp + len(page_bytes) > inode_size: - vollog.error( - "Page out of file bounds: inode 0x%x, inode size %d, page index %d", - inode.vol.offset, - inode_size, - page_idx, - ) - stream.seek(current_fp) - stream.write(page_bytes) + for page_idx, page_content in inode.get_contents(): + current_fp = page_idx * vmlinux_layer.page_size + max_length = inode_size - current_fp + page_bytes = page_content[:max_length] + if current_fp + len(page_bytes) > inode_size: + vollog.error( + "Page out of file bounds: inode 0x%x, inode size %d, page index %d", + inode.vol.offset, + inode_size, + page_idx, + ) + f.seek(current_fp) + f.write(page_bytes) + + except OSError as e: + vollog.error("Unable to write to file (%s): %s", filename, e) def _generator(self): vmlinux_module_name = self.config["kernel"] @@ -561,7 +528,7 @@ class InodePages(plugins.PluginInterface): filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") vollog.info("[*] Writing inode at 0x%x to '%s'", inode_address, filename) self.write_inode_content_to_file( - self.context, vmlinux_layer.name, inode, filename, open_method + inode, filename, open_method, vmlinux_layer ) def run(self): From bf76aad1e2367e2db4d561d9cc1cd76a42162420 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 20 Jan 2025 10:28:46 +1100 Subject: [PATCH 222/268] linux: page_cache.Files plugin: Ensure the inode's i_link pointer is readable --- volatility3/framework/plugins/linux/pagecache.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 39ed60486..f265241b6 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -147,7 +147,13 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): Otherwise, it returns the same symlink_path """ # i_link (fast symlinks) were introduced in 4.2 - if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: + if ( + inode + and inode.is_link + and inode.has_member("i_link") + and inode.i_link + and inode.i_link.is_readable() + ): i_link_str = inode.i_link.dereference().cast( "string", max_length=255, encoding="utf-8", errors="replace" ) From ec7a101eb92499741cafeb4dea1ab1ef1683a27c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 20 Jan 2025 11:05:09 +1100 Subject: [PATCH 223/268] linux: page_cache.InodePages plugin: Remove unnecesary casting --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index f265241b6..32b176b72 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -511,7 +511,7 @@ class InodePages(plugins.PluginInterface): page_vaddr = page_obj.vol.offset page_paddr = page_obj.to_paddr() page_mapping_addr = page_obj.mapping - page_index = int(page_obj.index) + page_index = page_obj.index page_file_offset = page_index * vmlinux_layer.page_size dump_safe = ( page_file_offset < inode_size From dfe3d255c064b9c78edf4f5f58eff6c15cc56486 Mon Sep 17 00:00:00 2001 From: Odysseas Stavrou Date: Mon, 20 Jan 2025 22:25:01 +0200 Subject: [PATCH 224/268] Volshell: Update Process retrieval methods with virtual/physical offsets --- volatility3/cli/volshell/linux.py | 58 +++++++++++++++++++++++++++++ volatility3/cli/volshell/windows.py | 47 +++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index cc58fa1c2..9ea3ea1f5 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -3,6 +3,7 @@ # from typing import Any, List, Optional, Tuple, Union +from enum import Enum from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -10,6 +11,16 @@ from volatility3.framework.configuration import requirements from volatility3.plugins.linux import pslist +# Could import the enum from psscan.py to avoid code duplication +class DescExitStateEnum(Enum): + """Enum for linux task exit_state as defined in include/linux/sched.h""" + + TASK_RUNNING = 0x00000000 + EXIT_DEAD = 0x00000010 + EXIT_ZOMBIE = 0x00000020 + EXIT_TRACE = EXIT_ZOMBIE | EXIT_DEAD + + class Volshell(generic.Volshell): """Shell environment to directly interact with a linux memory image.""" @@ -40,6 +51,52 @@ class Volshell(generic.Volshell): return None print(f"No task with task ID {pid} found") + def get_process(self, pid=None, offset=None): + """Get Task based on a process ID. Does not retrieve the layer, to change layer use the .pid attribute. The offset argument can be used both for physical or virtual offsets""" + + if pid is not None and offset is not None: + print("Only one parameter is accepted") + return None + + if offset is not None: + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] + + kernel_layer_name = vmlinux.layer_name + kernel_layer = self.context.layers[kernel_layer_name] + + memory_layer_name = kernel_layer.dependencies[0] + + ptask = self.context.object( + vmlinux.symbol_table_name + constants.BANG + "task_struct", + layer_name=memory_layer_name, + offset=offset, + native_layer_name=kernel_layer_name, + ) + + try: + DescExitStateEnum(ptask.exit_state) + except ValueError: + print( + f"task_struct @ {hex(ptask.vol.offset)} as exit_state {ptask.exit_state} is likely not valid" + ) + + if not (0 < ptask.pid < 65535): + print( + f"task_struct @ {hex(ptask.vol.offset)} as pid {ptask.pid} is likely not valid" + ) + + return ptask + + if pid is not None: + tasks = self.list_tasks() + for task in tasks: + if task.pid == pid: + return task + print(f"No task with task ID {pid} found") + + return None + def list_tasks(self): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols @@ -50,6 +107,7 @@ class Volshell(generic.Volshell): result += [ (["ct", "change_task", "cp"], self.change_task), (["lt", "list_tasks", "ps"], self.list_tasks), + (["gp", "get_process"], self.get_process), (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get("pid", None) is not None: diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 303d4d5c3..a77392561 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -44,11 +44,58 @@ class Volshell(generic.Volshell): ) ) + def get_process(self, pid=None, v_offset=None, p_offset=None): + """Returns the EPROCESS object that matches the pid. If v_offset/p_offset is provided, construct the EPROCESS object at the provided address. Only one parameter is allowed.""" + + if sum(1 if x is not None else 0 for x in [pid, v_offset, p_offset]) != 1: + print("Only one parameter is accepted") + return None + + kernel_name = self.config["kernel"] + kernel = self.context.modules[kernel_name] + + kernel_layer_name = kernel.layer_name + + kernel_layer = self.context.layers[kernel_layer_name] + memory_layer_name = kernel_layer.dependencies[0] + + eprocess_symbol = kernel.symbol_table_name + constants.BANG + "_EPROCESS" + + if v_offset is not None: + eproc = self.context.object( + eprocess_symbol, + layer_name=kernel_layer_name, + offset=v_offset, + ) + + return eproc + + if p_offset is not None: + eproc = self.context.object( + eprocess_symbol, + layer_name=memory_layer_name, + offset=p_offset, + native_layer_name=kernel_layer_name, + ) + + return eproc + + if pid is not None: + processes = self.list_processes() + for process in processes: + if process.UniqueProcessId == pid: + return process + print(f"No process with process ID {pid} found") + return None + + return None + def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (["cp", "change_process"], self.change_process), (["lp", "list_processes", "ps"], self.list_processes), + (["gp", "get_process"], self.get_process), (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get("pid", None) is not None: From b59f051353cd58b7d6e4bfda2f07746820f4f32a Mon Sep 17 00:00:00 2001 From: Odysseas Stavrou Date: Wed, 22 Jan 2025 02:39:35 +0200 Subject: [PATCH 225/268] Volshell: Updates to the get_process() methods --- volatility3/cli/volshell/linux.py | 55 +++++++++++++++++++---------- volatility3/cli/volshell/windows.py | 23 ++++++++---- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 9ea3ea1f5..b3689c3ae 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -51,42 +51,61 @@ class Volshell(generic.Volshell): return None print(f"No task with task ID {pid} found") - def get_process(self, pid=None, offset=None): - """Get Task based on a process ID. Does not retrieve the layer, to change layer use the .pid attribute. The offset argument can be used both for physical or virtual offsets""" + def get_process(self, pid=None, virtaddr=None, physaddr=None): + """Return the task_struct object that matches the pid. If a physical or a virtual address is provided, construct the task_struct object at said address. Only one parameter is allowed. - if pid is not None and offset is not None: + Args: + pid (int, optional): PID to search for + virtaddr (int, optional): Virtual address to construct object at + physaddr (int, optional): Physical address to construct object at + + Returns: + ObjectInterface: task_struct Object + """ + + if sum(1 if x is not None else 0 for x in [pid, virtaddr, physaddr]) != 1: print("Only one parameter is accepted") return None - if offset is not None: - vmlinux_module_name = self.config["kernel"] - vmlinux = self.context.modules[vmlinux_module_name] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] - kernel_layer_name = vmlinux.layer_name - kernel_layer = self.context.layers[kernel_layer_name] + kernel_layer_name = vmlinux.layer_name + kernel_layer = self.context.layers[kernel_layer_name] - memory_layer_name = kernel_layer.dependencies[0] + memory_layer_name = kernel_layer.dependencies[0] - ptask = self.context.object( - vmlinux.symbol_table_name + constants.BANG + "task_struct", + task_struct_symbol = vmlinux.symbol_table_name + constants.BANG + "task_struct" + + if virtaddr is not None: + task = self.context.object( + task_struct_symbol, + layer_name=kernel_layer_name, + offset=virtaddr, + ) + + if physaddr is not None: + task = self.context.object( + task_struct_symbol, layer_name=memory_layer_name, - offset=offset, + offset=physaddr, native_layer_name=kernel_layer_name, ) + if physaddr is not None or virtaddr is not None: try: - DescExitStateEnum(ptask.exit_state) + DescExitStateEnum(task.exit_state) except ValueError: print( - f"task_struct @ {hex(ptask.vol.offset)} as exit_state {ptask.exit_state} is likely not valid" + f"task_struct @ {hex(task.vol.offset)} as exit_state {task.exit_state} is likely not valid" ) - if not (0 < ptask.pid < 65535): + if not (0 < task.pid < 65535): print( - f"task_struct @ {hex(ptask.vol.offset)} as pid {ptask.pid} is likely not valid" + f"task_struct @ {hex(task.vol.offset)} as pid {task.pid} is likely not valid" ) - return ptask + return task if pid is not None: tasks = self.list_tasks() @@ -107,7 +126,7 @@ class Volshell(generic.Volshell): result += [ (["ct", "change_task", "cp"], self.change_task), (["lt", "list_tasks", "ps"], self.list_tasks), - (["gp", "get_process"], self.get_process), + (["gp", "get_process", "get_task"], self.get_process), (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get("pid", None) is not None: diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index a77392561..9b89a8b81 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -44,10 +44,19 @@ class Volshell(generic.Volshell): ) ) - def get_process(self, pid=None, v_offset=None, p_offset=None): - """Returns the EPROCESS object that matches the pid. If v_offset/p_offset is provided, construct the EPROCESS object at the provided address. Only one parameter is allowed.""" + def get_process(self, pid=None, virtaddr=None, physaddr=None): + """Returns the _EPROCESS object that matches the pid. If a physical or a virtual address is provided, construct the _EPROCESS object at said address. Only one parameter is allowed. - if sum(1 if x is not None else 0 for x in [pid, v_offset, p_offset]) != 1: + Args: + pid (int, optional): PID / UniqueProcessId to search for. + virtaddr (int, optional): Virtual address to construct object at + physaddr (int, optional): Physical address to construct object at + + Returns: + ObjectInterface: _EPROCESS Object + """ + + if sum(1 if x is not None else 0 for x in [pid, virtaddr, physaddr]) != 1: print("Only one parameter is accepted") return None @@ -61,20 +70,20 @@ class Volshell(generic.Volshell): eprocess_symbol = kernel.symbol_table_name + constants.BANG + "_EPROCESS" - if v_offset is not None: + if virtaddr is not None: eproc = self.context.object( eprocess_symbol, layer_name=kernel_layer_name, - offset=v_offset, + offset=virtaddr, ) return eproc - if p_offset is not None: + if physaddr is not None: eproc = self.context.object( eprocess_symbol, layer_name=memory_layer_name, - offset=p_offset, + offset=physaddr, native_layer_name=kernel_layer_name, ) From c10572905f3f3760594db07575534a5805a10fe3 Mon Sep 17 00:00:00 2001 From: Daniel Davidov <35842733+Danking555@users.noreply.github.com> Date: Wed, 22 Jan 2025 10:45:52 +0200 Subject: [PATCH 226/268] Add low stub offset kernel detection reference: Memprocfs and https://www.youtube.com/watch?v=_ShCSth6dWM --- volatility3/framework/automagic/pdbscan.py | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 729c48063..1ccecf97a 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -11,6 +11,7 @@ import contextlib import logging import math import os +import struct from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, layers @@ -376,8 +377,43 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): valid_kernel = (virtual_layer_name, address, res[0]) return valid_kernel + def method_low_stub_offset(self, + context: interfaces.context.ContextInterface, + vlayer: layers.intel.Intel, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[ValidKernelType]: + kernel_hint = 0 + kernel_base = 0 + physical_layer = context.layers.get('memory_layer') + + # try locating kernel base via x64 Low Stub in lower 1MB starting from second page (4KB) + # if "Discard Low Memory" setting is disabled in BIOS, the Low Stub may be at the third/fourth or further pages + for offset in range(0x1000,0x100000, 0x1000): + if 0xffffffffffff00ff & int.from_bytes(physical_layer.read(offset, 0x8), "little") != 0x00000001000600E9: + continue # not _PROCESSOR_START_BLOCK->Jmp + potential_kernel_hint = int.from_bytes(physical_layer.read(offset + 0x70, 0x8), "little") + if (0xfffff80000000003 & potential_kernel_hint) != 0xfffff80000000000: + continue # not _PROCESSOR_START_BLOCK->LmTarget + kernel_hint = potential_kernel_hint & 0xffffffffffff + kernel_base = kernel_hint & (~0x1fffff) & 0xffffffffffff + break + + if kernel_base: + # Scanning 32mb in 2mb chunks for the 'ntoskrnl' base address + while (kernel_base + 0x2000000) > kernel_hint: + for i in range(0, 0x200000, 0x1000): + valid_kernel = self.check_kernel_offset( + context, vlayer, kernel_base, progress_callback + ) + if valid_kernel: + return valid_kernel + kernel_base -= 0x200000 + + return None + # List of methods to be run, in order, to determine the valid kernels methods = [ + method_low_stub_offset, method_kdbg_offset, method_module_offset, method_fixed_mapping, From 34a7dfc72fb3dced313e9a52b73b96dff31b778a Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 23 Jan 2025 10:42:52 +0100 Subject: [PATCH 227/268] split linux modules utilities --- .../symbols/linux/utilities/modules.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 volatility3/framework/symbols/linux/utilities/modules.py diff --git a/volatility3/framework/symbols/linux/utilities/modules.py b/volatility3/framework/symbols/linux/utilities/modules.py new file mode 100644 index 000000000..ac9b2afaf --- /dev/null +++ b/volatility3/framework/symbols/linux/utilities/modules.py @@ -0,0 +1,68 @@ +from typing import Iterator, List, Tuple + +from volatility3 import framework +from volatility3.framework import constants, interfaces +from volatility3.framework.objects import utility + + +class Modules(interfaces.configuration.VersionableInterface): + """Kernel modules related utilities.""" + + _version = (1, 0, 0) + _required_framework_version = (2, 0, 0) + + framework.require_interface_version(*_required_framework_version) + + @staticmethod + def mask_mods_list( + context: interfaces.context.ContextInterface, + layer_name: str, + mods: Iterator[interfaces.objects.ObjectInterface], + ) -> List[Tuple[str, int, int]]: + """ + A helper function to mask the starting and end address of kernel modules + """ + mask = context.layers[layer_name].address_mask + + return [ + ( + utility.array_to_string(mod.name), + mod.get_module_base() & mask, + (mod.get_module_base() & mask) + mod.get_core_size(), + ) + for mod in mods + ] + + @staticmethod + def lookup_module_address( + context: interfaces.context.ContextInterface, + kernel_module_name: str, + handlers: List[Tuple[str, int, int]], + target_address: int, + ) -> Tuple[str, str]: + """ + Searches between the start and end address of the kernel module using target_address. + Returns the module and symbol name of the address provided. + """ + kernel_module = context.modules[kernel_module_name] + mod_name = "UNKNOWN" + symbol_name = "N/A" + + for name, start, end in handlers: + if start <= target_address <= end: + mod_name = name + if name == constants.linux.KERNEL_NAME: + symbols = list( + kernel_module.get_symbols_by_absolute_location(target_address) + ) + + if len(symbols): + symbol_name = ( + symbols[0].split(constants.BANG)[1] + if constants.BANG in symbols[0] + else symbols[0] + ) + + break + + return mod_name, symbol_name From 43ab0b0c314832742c00f7821cd6f3327529894e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 23 Jan 2025 10:43:14 +0100 Subject: [PATCH 228/268] add deprecation decorator --- .../framework/configuration/requirements.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 3e3608000..3af5601dc 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -11,6 +11,7 @@ expect to be in the context (such as particular layers or symboltables). import abc import logging import os +import functools from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, Type from urllib import parse, request @@ -723,3 +724,25 @@ class ModuleRequirement( """Builds the appropriate configuration for the specified requirement.""" return context.modules[value].build_configuration() + + +def deprecated_method(replacement: str, additional_information: str = ""): + """A decorator for marking functions as deprecated. + + Args: + replacement: The replacement function overriding the deprecated API (full path preferred, starting from "volatility3."). String was preferred, for convenience and to prevent import conflicts on caller side. + additional_information: Information appended at the end of the deprecation message + """ + + def decorator(deprecated_func): + @functools.wraps(deprecated_func) + def wrapper(*args, **kwargs): + nonlocal replacement, additional_information + deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__name__}\" is deprecated, use \"{replacement}\" instead. {additional_information}" + vollog.warning(deprecation_msg) + # Return the wrapped function with its original arguments + return deprecated_func(*args, **kwargs) + + return wrapper + + return decorator From 7575aa5d6354419b51d1ac563c1de4db62da0ca1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 23 Jan 2025 10:44:20 +0100 Subject: [PATCH 229/268] deprecate lookup_module_address and mask_mods_list --- .../framework/symbols/linux/__init__.py | 101 ++++++++---------- 1 file changed, 44 insertions(+), 57 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 423284b03..3dc744f78 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -8,11 +8,13 @@ import logging from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3 import framework from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions +from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -81,7 +83,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (2, 2, 0) + _version = (2, 2, 1) _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) @@ -338,27 +340,6 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): yield fd_num, filp, full_path - @classmethod - def mask_mods_list( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - mods: Iterator[interfaces.objects.ObjectInterface], - ) -> List[Tuple[str, int, int]]: - """ - A helper function to mask the starting and end address of kernel modules - """ - mask = context.layers[layer_name].address_mask - - return [ - ( - utility.array_to_string(mod.name), - mod.get_module_base() & mask, - (mod.get_module_base() & mask) + mod.get_core_size(), - ) - for mod in mods - ] - @classmethod def generate_kernel_handler_info( cls, @@ -382,41 +363,9 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return [ (constants.linux.KERNEL_NAME, start_addr, end_addr) - ] + LinuxUtilities.mask_mods_list(context, kernel.layer_name, mods_list) - - @classmethod - def lookup_module_address( - cls, - kernel_module: interfaces.context.ModuleInterface, - handlers: List[Tuple[str, int, int]], - target_address: int, - ): - """ - Searches between the start and end address of the kernel module using target_address. - Returns the module and symbol name of the address provided. - """ - - mod_name = "UNKNOWN" - symbol_name = "N/A" - - for name, start, end in handlers: - if start <= target_address <= end: - mod_name = name - if name == constants.linux.KERNEL_NAME: - symbols = list( - kernel_module.get_symbols_by_absolute_location(target_address) - ) - - if len(symbols): - symbol_name = ( - symbols[0].split(constants.BANG)[1] - if constants.BANG in symbols[0] - else symbols[0] - ) - - break - - return mod_name, symbol_name + ] + linux_utilities_modules.Modules.mask_mods_list( + context, kernel.layer_name, mods_list + ) @classmethod def walk_internal_list(cls, vmlinux, struct_name, list_member, list_start): @@ -504,6 +453,44 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): [chr((code >> (i * 8)) & 0xFF) for i in range(code_bytes_length)] ) + ## Deprecated APIs ## + @classmethod + @requirements.deprecated_method( + replacement="volatility3.framework.symbols.linux.utilities.modules.Modules.mask_mods_list" + ) + def mask_mods_list( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + mods: Iterator[interfaces.objects.ObjectInterface], + ) -> List[Tuple[str, int, int]]: + """ + DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.mask_mods_list" instead. + + A helper function to mask the starting and end address of kernel modules + """ + return linux_utilities_modules.Modules.mask_mods_list(context, layer_name, mods) + + @classmethod + @requirements.deprecated_method( + replacement="volatility3.framework.symbols.linux.utilities.modules.Modules.lookup_module_address" + ) + def lookup_module_address( + cls, + kernel_module: interfaces.context.ModuleInterface, + handlers: List[Tuple[str, int, int]], + target_address: int, + ) -> Tuple[str, str]: + """ + DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.lookup_module_address" instead. + + Searches between the start and end address of the kernel module using target_address. + Returns the module and symbol name of the address provided. + """ + return linux_utilities_modules.Modules.lookup_module_address( + kernel_module.context, kernel_module.name, handlers, target_address + ) + class IDStorage(ABC): """Abstraction to support both XArray and RadixTree""" From fd77c041537b40f38db7428ce05a876ad5b1e08b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 23 Jan 2025 10:46:42 +0100 Subject: [PATCH 230/268] move to linux_utilities_modules APIs --- volatility3/framework/plugins/linux/check_idt.py | 7 +++++-- .../framework/plugins/linux/keyboard_notifiers.py | 7 +++++-- volatility3/framework/plugins/linux/kthreads.py | 9 ++++++--- volatility3/framework/plugins/linux/netfilter.py | 7 +++++-- volatility3/framework/plugins/linux/tty_check.py | 7 +++++-- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/plugins/linux/check_idt.py b/volatility3/framework/plugins/linux/check_idt.py index 07582e2c1..5859e73d6 100644 --- a/volatility3/framework/plugins/linux/check_idt.py +++ b/volatility3/framework/plugins/linux/check_idt.py @@ -5,6 +5,7 @@ import logging from typing import List +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3.framework import interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -99,8 +100,10 @@ class Check_idt(interfaces.plugins.PluginInterface): idt_addr = idt_addr & address_mask - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( - vmlinux, handlers, idt_addr + module_name, symbol_name = ( + linux_utilities_modules.Modules.lookup_module_address( + self.context, vmlinux.name, handlers, idt_addr + ) ) yield ( diff --git a/volatility3/framework/plugins/linux/keyboard_notifiers.py b/volatility3/framework/plugins/linux/keyboard_notifiers.py index 72273a77b..c1b7572c6 100644 --- a/volatility3/framework/plugins/linux/keyboard_notifiers.py +++ b/volatility3/framework/plugins/linux/keyboard_notifiers.py @@ -4,6 +4,7 @@ import logging +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints @@ -66,8 +67,10 @@ class Keyboard_notifiers(interfaces.plugins.PluginInterface): ): call_addr = call_back.notifier_call - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( - vmlinux, handlers, call_addr + module_name, symbol_name = ( + linux_utilities_modules.Modules.lookup_module_address( + self.context, vmlinux.name, handlers, call_addr + ) ) yield (0, [format_hints.Hex(call_addr), module_name, symbol_name]) diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index 40e992069..2e1bbed47 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -4,6 +4,7 @@ import logging from typing import List +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins @@ -20,7 +21,7 @@ class Kthreads(plugins.PluginInterface): """Enumerates kthread functions""" _required_framework_version = (2, 11, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -88,8 +89,10 @@ class Kthreads(plugins.PluginInterface): if kthread.has_member("full_name") else task_name ) - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( - vmlinux, handlers, threadfn + module_name, symbol_name = ( + linux_utilities_modules.Modules.lookup_module_address( + self.context, vmlinux.name, handlers, threadfn + ) ) fields = [ diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 73496dfd9..ccb831b61 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field from abc import ABC, abstractmethod import logging +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from typing import Iterator, List, Tuple from volatility3 import framework from volatility3.framework import ( @@ -263,8 +264,10 @@ class AbstractNetfilter(ABC): """Helper to obtain the module and symbol name in the format needed for the output of this plugin. """ - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( - self.vmlinux, self.handlers, addr + module_name, symbol_name = ( + linux_utilities_modules.Modules.lookup_module_address( + self._context, self.vmlinux.name, self.handlers, addr + ) ) if module_name == "UNKNOWN": diff --git a/volatility3/framework/plugins/linux/tty_check.py b/volatility3/framework/plugins/linux/tty_check.py index 45238ef8c..f375968a4 100644 --- a/volatility3/framework/plugins/linux/tty_check.py +++ b/volatility3/framework/plugins/linux/tty_check.py @@ -5,6 +5,7 @@ import logging from typing import List +import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins @@ -79,8 +80,10 @@ class tty_check(plugins.PluginInterface): recv_buf = tty_dev.ldisc.ops.receive_buf - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address( - vmlinux, handlers, recv_buf + module_name, symbol_name = ( + linux_utilities_modules.Modules.lookup_module_address( + self.context, vmlinux.name, handlers, recv_buf + ) ) yield (0, (name, format_hints.Hex(recv_buf), module_name, symbol_name)) From b8d9c7b88311016cea97b8b60afeec4d47558af0 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 23 Jan 2025 11:57:43 -0600 Subject: [PATCH 231/268] #1473 - add missing exception handling for get_key --- volatility3/framework/plugins/windows/envars.py | 12 ++++++------ .../framework/plugins/windows/getservicesids.py | 5 +++-- .../framework/plugins/windows/getsids.py | 2 +- .../plugins/windows/registry/userassist.py | 17 ++++++++++++----- .../framework/plugins/windows/svcscan.py | 6 +++--- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index cac4ecf40..48e1ef671 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -76,14 +76,14 @@ class Envars(interfaces.plugins.PluginInterface): "CurrentControlSet\\Control\\Session Manager\\Environment" ) sys = True - except KeyError: - with contextlib.suppress(KeyError): + except (KeyError, registry.RegistryFormatException): + with contextlib.suppress(KeyError, registry.RegistryFormatException): key = hive.get_key( "ControlSet001\\Control\\Session Manager\\Environment" ) sys = True if sys: - with contextlib.suppress(KeyError): + with contextlib.suppress(KeyError, registry.RegistryFormatException): for node in key.get_values(): try: value_node_name = node.get_name() @@ -100,11 +100,11 @@ class Envars(interfaces.plugins.PluginInterface): continue ## The user-specific variables - with contextlib.suppress(KeyError): + with contextlib.suppress(KeyError, registry.RegistryFormatException): key = hive.get_key("Environment") ntuser = True if ntuser: - with contextlib.suppress(KeyError): + with contextlib.suppress(KeyError, registry.RegistryFormatException): for node in key.get_values(): try: value_node_name = node.get_name() @@ -123,7 +123,7 @@ class Envars(interfaces.plugins.PluginInterface): ## The volatile user variables try: key = hive.get_key("Volatile Environment") - except KeyError: + except (KeyError, registry.RegistryFormatException): continue try: for node in key.get_values(): diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index eece7fb6c..b97d2bb46 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -10,6 +10,7 @@ from typing import List from volatility3.framework import renderers, interfaces, constants, exceptions from volatility3.framework.configuration import requirements +from volatility3.framework.layers import registry from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) @@ -86,10 +87,10 @@ class GetServiceSIDs(interfaces.plugins.PluginInterface): # Get ControlSet\Services. try: services = hive.get_key(r"CurrentControlSet\Services") - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): try: services = hive.get_key(r"ControlSet001\Services") - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): continue if services: diff --git a/volatility3/framework/plugins/windows/getsids.py b/volatility3/framework/plugins/windows/getsids.py index df0c7a835..00c78e1cf 100644 --- a/volatility3/framework/plugins/windows/getsids.py +++ b/volatility3/framework/plugins/windows/getsids.py @@ -158,7 +158,7 @@ class GetSIDs(interfaces.plugins.PluginInterface): layers.registry.RegistryFormatException, ): continue - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, layers.registry.RegistryFormatException): continue return sids diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 932ee9d6f..646fb1d7f 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -13,7 +13,7 @@ from typing import Any, Generator, List, Tuple from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers.physical import BufferDataLayer -from volatility3.framework.layers.registry import RegistryHive +from volatility3.framework.layers.registry import RegistryHive, RegistryFormatException from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.plugins.windows.registry import hivelist @@ -167,10 +167,17 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac self._determine_userassist_type() - userassist_node_path = hive.get_key( - "software\\microsoft\\windows\\currentversion\\explorer\\userassist", - return_list=True, - ) + try: + userassist_node_path = hive.get_key( + "software\\microsoft\\windows\\currentversion\\explorer\\userassist", + return_list=True, + ) + except RegistryFormatException as e: + vollog.warning(f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}") + return None + except KeyError: + vollog.warning(f"UserAssist key not found in {hive_name} at {hive.hive_offset:#x}") + return None if not userassist_node_path: vollog.warning("list_userassist did not find a valid node_path (or None)") diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index bd477ba27..93087f352 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -15,7 +15,7 @@ from volatility3.framework import ( symbols, ) from volatility3.framework.configuration import requirements -from volatility3.framework.layers import scanners +from volatility3.framework.layers import scanners, registry from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions @@ -159,12 +159,12 @@ class SvcScan(interfaces.plugins.PluginInterface): return cast( objects.StructType, hive.get_key(r"CurrentControlSet\Services") ) - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): try: return cast( objects.StructType, hive.get_key(r"ControlSet001\Services") ) - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): vollog.log( constants.LOGLEVEL_VVVV, "Could not retrieve any control set from SYSTEM hive", From 6c4cafa64f68e6b001cd1ed32e8e5fb3d9993f30 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 23 Jan 2025 11:59:37 -0600 Subject: [PATCH 232/268] #1473 - black fixes --- .../framework/plugins/windows/getservicesids.py | 12 ++++++++++-- volatility3/framework/plugins/windows/getsids.py | 6 +++++- .../framework/plugins/windows/registry/userassist.py | 8 ++++++-- volatility3/framework/plugins/windows/svcscan.py | 12 ++++++++++-- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index b97d2bb46..207d0e2ad 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -87,10 +87,18 @@ class GetServiceSIDs(interfaces.plugins.PluginInterface): # Get ControlSet\Services. try: services = hive.get_key(r"CurrentControlSet\Services") - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): try: services = hive.get_key(r"ControlSet001\Services") - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): continue if services: diff --git a/volatility3/framework/plugins/windows/getsids.py b/volatility3/framework/plugins/windows/getsids.py index 00c78e1cf..a75bbe7ea 100644 --- a/volatility3/framework/plugins/windows/getsids.py +++ b/volatility3/framework/plugins/windows/getsids.py @@ -158,7 +158,11 @@ class GetSIDs(interfaces.plugins.PluginInterface): layers.registry.RegistryFormatException, ): continue - except (KeyError, exceptions.InvalidAddressException, layers.registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + layers.registry.RegistryFormatException, + ): continue return sids diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 646fb1d7f..d50b5216e 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -173,10 +173,14 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac return_list=True, ) except RegistryFormatException as e: - vollog.warning(f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}") + vollog.warning( + f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}" + ) return None except KeyError: - vollog.warning(f"UserAssist key not found in {hive_name} at {hive.hive_offset:#x}") + vollog.warning( + f"UserAssist key not found in {hive_name} at {hive.hive_offset:#x}" + ) return None if not userassist_node_path: diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 93087f352..17baac5b0 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -159,12 +159,20 @@ class SvcScan(interfaces.plugins.PluginInterface): return cast( objects.StructType, hive.get_key(r"CurrentControlSet\Services") ) - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): try: return cast( objects.StructType, hive.get_key(r"ControlSet001\Services") ) - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): vollog.log( constants.LOGLEVEL_VVVV, "Could not retrieve any control set from SYSTEM hive", From 81ba89eef663727608886e66595dfd7b3dcd9831 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 23 Jan 2025 12:20:14 -0600 Subject: [PATCH 233/268] #1473 - update exception message --- volatility3/framework/plugins/windows/registry/userassist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index d50b5216e..87016553a 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -174,7 +174,7 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac ) except RegistryFormatException as e: vollog.warning( - f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}" + f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}: {e}" ) return None except KeyError: From a68be50798254cbadc490393721e74180b4117cc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 23 Jan 2025 20:30:21 +0000 Subject: [PATCH 234/268] Revert "Typing fix" This reverts commit c82d432b10258136ff0777dfec1fbf5844316132. --- volatility3/framework/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index a1925faef..754939460 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,6 +5,7 @@ # Check the python version to ensure it's suitable import glob import sys +from volatility3.framework import check_python_version as check_python_version import zipfile import importlib import inspect @@ -57,7 +58,7 @@ class NonInheritable: self.default_value = value self.cls = cls - def __get__(self, obj: Any, get_type: Optional[Type] = None) -> Any: + def __get__(self, obj: Any, get_type: Type = Optional[None]) -> Any: if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) From 61f60ac464ba01885216704faf37ed6b6fc4beb6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 23 Jan 2025 20:52:08 +0000 Subject: [PATCH 235/268] Core: Move the python check somewhere it can't accidentally be removed --- volatility3/framework/__init__.py | 12 +++++++++++- volatility3/framework/check_python_version.py | 14 -------------- volatility3/framework/constants/__init__.py | 2 ++ 3 files changed, 13 insertions(+), 15 deletions(-) delete mode 100644 volatility3/framework/check_python_version.py diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 754939460..466e697bb 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,7 +5,6 @@ # Check the python version to ensure it's suitable import glob import sys -from volatility3.framework import check_python_version as check_python_version import zipfile import importlib import inspect @@ -16,6 +15,17 @@ from typing import Any, Dict, Generator, List, Optional, Tuple, Type, TypeVar from volatility3.framework import constants, interfaces +if ( + sys.version_info.major != constants.REQUIRED_PYTHON_VERSION[0] + or sys.version_info.minor < constants.REQUIRED_PYTHON_VERSION[1] + or ( + sys.version_info.minor == constants.REQUIRED_PYTHON_VERSION[1] + and sys.version_info.micro < constants.REQUIRED_PYTHON_VERSION[2] + ) +): + raise RuntimeError( + f"Volatility framework requires python version {".".join(str(x) for x in constants.REQUIRED_PYTHON_VERSION)} or greater" + ) # ## # diff --git a/volatility3/framework/check_python_version.py b/volatility3/framework/check_python_version.py deleted file mode 100644 index f2d284f2a..000000000 --- a/volatility3/framework/check_python_version.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys - -required_python_version = (3, 8, 0) -if ( - sys.version_info.major != required_python_version[0] - or sys.version_info.minor < required_python_version[1] - or ( - sys.version_info.minor == required_python_version[1] - and sys.version_info.micro < required_python_version[2] - ) -): - raise RuntimeError( - f"Volatility framework requires python version {required_python_version[0]}.{required_python_version[1]}.{required_python_version[2]} or greater" - ) diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 23cc2dde5..2e6ae0261 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -23,6 +23,8 @@ from volatility3.framework.constants._version import ( VERSION_SUFFIX as VERSION_SUFFIX, ) +REQUIRED_PYTHON_VERSION = (3, 8, 0) + PLUGINS_PATH = [ os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "plugins")), os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "plugins")), From 128e1be154cc5a9853da3413565685689bf702e5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 23 Jan 2025 20:57:44 +0000 Subject: [PATCH 236/268] Core: Fix f-string quotes --- volatility3/framework/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 466e697bb..0bbdefa43 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -24,7 +24,7 @@ if ( ) ): raise RuntimeError( - f"Volatility framework requires python version {".".join(str(x) for x in constants.REQUIRED_PYTHON_VERSION)} or greater" + f"Volatility framework requires python version {'.'.join(str(x) for x in constants.REQUIRED_PYTHON_VERSION)} or greater" ) # ## From 17d52c27bfc31ad9ecad7fb0b8604586039ce2b0 Mon Sep 17 00:00:00 2001 From: Daniel Davidov <35842733+Danking555@users.noreply.github.com> Date: Fri, 24 Jan 2025 21:09:28 +0200 Subject: [PATCH 237/268] Update method_low_stub_offset & run ruff & black * Eliminate unnecessary scanning for 32 bit processors where the structure PROCESSOR_START_BLOCK doesn't exist * Put offsets as values of constants in a class - LowStubLayout. * Add documentation in the class and in the function method_low_stub_offset * Run "ruff check --fix" and "black ." * Checked the method works on 3 physical machines --- volatility3/framework/automagic/pdbscan.py | 74 ++++++++++++++++++---- 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 1ccecf97a..7d289bcb6 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -11,7 +11,6 @@ import contextlib import logging import math import os -import struct from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, exceptions, interfaces, layers @@ -377,25 +376,73 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): valid_kernel = (virtual_layer_name, address, res[0]) return valid_kernel - def method_low_stub_offset(self, + class LowStubLayout: + """ + Represents the layout of the Low Stub which exists only on x64 machines with no virtualization/emulation, + responsible for transitioning from Real Mode(16 bit) to Protected Mode(32 bit) and Long Mode(64 bit) on boot/return from sleep. + Contains offsets to fields and structures within the undocumented structure _PROCESSOR_START_BLOCK. + Here's a reference: https://github.com/mic101/windows/blob/master/WRK-v1.2/base/ntos/inc/amd64.h#L3334 + """ + + # Expected signature for validation, constructed from: + # PROCESSOR_START_BLOCK->Jmp->OpCode | PROCESSOR_START_BLOCK->Jmp->Offset | PROCESSOR_START_BLOCK->CompletionFlag + JMP_AND_COMPLETION_SIGNATURE = 0x00000001000600E9 + + # Address of LmTarget (Long Mode target) + PROCESSOR_START_BLOCK_LM_TARGET_OFFSET = ( + 0x70 # PROCESSOR_START_BLOCK->LmTarget, PVOID 8 bytes + ) + + # CR3 register within structures describing initial processor state to be started + PROCESSOR_START_BLOCK_CR3_OFFSET = 0xA0 # PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3, ULONG64 8 bytes + + def method_low_stub_offset( + self, context: interfaces.context.ContextInterface, vlayer: layers.intel.Intel, progress_callback: constants.ProgressCallback = None, ) -> Optional[ValidKernelType]: + # This method is only valid for x64 systems + if not isinstance(vlayer, intel.Intel32e): + return None kernel_hint = 0 kernel_base = 0 - physical_layer = context.layers.get('memory_layer') + physical_layer = context.layers.get("memory_layer") - # try locating kernel base via x64 Low Stub in lower 1MB starting from second page (4KB) - # if "Discard Low Memory" setting is disabled in BIOS, the Low Stub may be at the third/fourth or further pages - for offset in range(0x1000,0x100000, 0x1000): - if 0xffffffffffff00ff & int.from_bytes(physical_layer.read(offset, 0x8), "little") != 0x00000001000600E9: - continue # not _PROCESSOR_START_BLOCK->Jmp - potential_kernel_hint = int.from_bytes(physical_layer.read(offset + 0x70, 0x8), "little") - if (0xfffff80000000003 & potential_kernel_hint) != 0xfffff80000000000: - continue # not _PROCESSOR_START_BLOCK->LmTarget - kernel_hint = potential_kernel_hint & 0xffffffffffff - kernel_base = kernel_hint & (~0x1fffff) & 0xffffffffffff + # Try locating kernel base via x64 Low Stub in lower 1MB starting from second page (4KB) + # If "Discard Low Memory" setting is disabled in BIOS, the Low Stub may be at the third/fourth or further pages + for offset in range(0x1000, 0x100000, 0x1000): + jmp_and_completion_values = int.from_bytes( + physical_layer.read(offset, 0x8), "little" + ) + if ( + 0xFFFFFFFFFFFF00FF & jmp_and_completion_values + != self.LowStubLayout.JMP_AND_COMPLETION_SIGNATURE + ): + continue + cr3_value = int.from_bytes( + physical_layer.read( + offset + self.LowStubLayout.PROCESSOR_START_BLOCK_CR3_OFFSET, 0x8 + ), + "little", + ) + + # Compare previously observed valid page table address that's stored in vlayer._initial_entry + # with PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3 + # which was observed to be an invalid page address, so add 1 (to make it valid too) + if (cr3_value + 1) != vlayer._initial_entry: + continue + potential_kernel_hint = int.from_bytes( + physical_layer.read( + offset + self.LowStubLayout.PROCESSOR_START_BLOCK_LM_TARGET_OFFSET, + 0x8, + ), + "little", + ) + if 0x3 & potential_kernel_hint: + continue + kernel_hint = potential_kernel_hint & 0xFFFFFFFFFFFF + kernel_base = kernel_hint & (~0x1FFFFF) & 0xFFFFFFFFFFFF break if kernel_base: @@ -408,7 +455,6 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): if valid_kernel: return valid_kernel kernel_base -= 0x200000 - return None # List of methods to be run, in order, to determine the valid kernels From e9088be0d86fa2f68774d47371ebd2736287f1c5 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 28 Dec 2024 22:46:39 +0000 Subject: [PATCH 238/268] Prevent infinite looping and out of memory errors #1482 --- .../framework/symbols/windows/extensions/registry.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index c9544a8ba..b282b13cf 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -133,8 +133,17 @@ class CM_KEY_BODY(objects.StructType): def get_full_key_name(self) -> str: output = [] + seen = set() + kcb = self.KeyControlBlock while kcb.ParentKcb: + if kcb.ParentKcb.vol.offset in seen: + return "" + seen.add(kcb.ParentKcb.vol.offset) + + if len(output) > 128: + return "" + if kcb.NameBlock.Name is None: break From 506a61d8846e6a3399ab1f964d41b09a197592a6 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 24 Jan 2025 22:09:24 +0000 Subject: [PATCH 239/268] Address feedback --- volatility3/framework/plugins/windows/handles.py | 4 ++-- volatility3/framework/symbols/windows/extensions/registry.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 38ccfbfbc..6a391fe35 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -341,7 +341,7 @@ class Handles(interfaces.plugins.PluginInterface): try: obj_name = entry.NameInfo.Name.String except (ValueError, exceptions.InvalidAddressException): - obj_name = "" + obj_name = None except exceptions.InvalidAddressException: vollog.log( @@ -359,7 +359,7 @@ class Handles(interfaces.plugins.PluginInterface): format_hints.Hex(entry.HandleValue), obj_type, format_hints.Hex(entry.GrantedAccess), - obj_name, + obj_name or renderers.NotAvailableValue(), ), ) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index b282b13cf..a8cc7703c 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -138,11 +138,11 @@ class CM_KEY_BODY(objects.StructType): kcb = self.KeyControlBlock while kcb.ParentKcb: if kcb.ParentKcb.vol.offset in seen: - return "" + return None seen.add(kcb.ParentKcb.vol.offset) if len(output) > 128: - return "" + return None if kcb.NameBlock.Name is None: break From 8b0165b6f6ea7a6ebb70ddb01b2de29b627ddcfd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 14:45:21 +0100 Subject: [PATCH 240/268] add linux_utilities_modules requirement --- volatility3/framework/plugins/linux/check_idt.py | 5 +++++ volatility3/framework/plugins/linux/keyboard_notifiers.py | 5 +++++ volatility3/framework/plugins/linux/kthreads.py | 5 +++++ volatility3/framework/plugins/linux/netfilter.py | 5 +++++ volatility3/framework/plugins/linux/tty_check.py | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/volatility3/framework/plugins/linux/check_idt.py b/volatility3/framework/plugins/linux/check_idt.py index 5859e73d6..ffb707af5 100644 --- a/volatility3/framework/plugins/linux/check_idt.py +++ b/volatility3/framework/plugins/linux/check_idt.py @@ -28,6 +28,11 @@ class Check_idt(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.VersionRequirement( + name="linux_utilities_modules", + component=linux_utilities_modules.Modules, + version=(1, 0, 0), + ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) ), diff --git a/volatility3/framework/plugins/linux/keyboard_notifiers.py b/volatility3/framework/plugins/linux/keyboard_notifiers.py index c1b7572c6..8577de848 100644 --- a/volatility3/framework/plugins/linux/keyboard_notifiers.py +++ b/volatility3/framework/plugins/linux/keyboard_notifiers.py @@ -27,6 +27,11 @@ class Keyboard_notifiers(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.VersionRequirement( + name="linux_utilities_modules", + component=linux_utilities_modules.Modules, + version=(1, 0, 0), + ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index 2e1bbed47..bd0e895a4 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -31,6 +31,11 @@ class Kthreads(plugins.PluginInterface): description="Linux kernel", architectures=architectures.LINUX_ARCHS, ), + requirements.VersionRequirement( + name="linux_utilities_modules", + component=linux_utilities_modules.Modules, + version=(1, 0, 0), + ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index ccb831b61..33a8ca7cc 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -691,6 +691,11 @@ class Netfilter(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.VersionRequirement( + name="linux_utilities_modules", + component=linux_utilities_modules.Modules, + version=(1, 0, 0), + ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=cls._required_lsmod_version ), diff --git a/volatility3/framework/plugins/linux/tty_check.py b/volatility3/framework/plugins/linux/tty_check.py index f375968a4..9bbca246c 100644 --- a/volatility3/framework/plugins/linux/tty_check.py +++ b/volatility3/framework/plugins/linux/tty_check.py @@ -30,6 +30,11 @@ class tty_check(plugins.PluginInterface): description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.VersionRequirement( + name="linux_utilities_modules", + component=linux_utilities_modules.Modules, + version=(1, 0, 0), + ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), From ce671bb2fa3a7d03042a993814b4c11f19650cf4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 15:02:25 +0100 Subject: [PATCH 241/268] transfer deprecated_method into framework module --- volatility3/framework/__init__.py | 29 ++++++++++++++++++- .../framework/configuration/requirements.py | 23 --------------- .../framework/symbols/linux/__init__.py | 17 +++++++---- 3 files changed, 39 insertions(+), 30 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index a1925faef..12254ca77 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -11,7 +11,8 @@ import inspect import logging import os import traceback -from typing import Any, Dict, Generator, List, Optional, Tuple, Type, TypeVar +import functools +from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Type, TypeVar from volatility3.framework import constants, interfaces @@ -52,6 +53,32 @@ def require_interface_version(*args) -> None: ) +class Deprecation: + """Deprecation related methods.""" + + @staticmethod + def deprecated_method(replacement: Callable, additional_information: str = ""): + """A decorator for marking functions as deprecated. + + Args: + replacement: The replacement function overriding the deprecated API, in the form of a Callable (typically a method) + additional_information: Information appended at the end of the deprecation message + """ + + def decorator(deprecated_func): + @functools.wraps(deprecated_func) + def wrapper(*args, **kwargs): + nonlocal replacement, additional_information + deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__name__}\" is deprecated, use \"{replacement.__module__ + '.' + replacement.__name__}\" instead. {additional_information}" + vollog.warning(deprecation_msg) + # Return the wrapped function with its original arguments + return deprecated_func(*args, **kwargs) + + return wrapper + + return decorator + + class NonInheritable: def __init__(self, value: Any, cls: Type) -> None: self.default_value = value diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 3af5601dc..3e3608000 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -11,7 +11,6 @@ expect to be in the context (such as particular layers or symboltables). import abc import logging import os -import functools from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, Type from urllib import parse, request @@ -724,25 +723,3 @@ class ModuleRequirement( """Builds the appropriate configuration for the specified requirement.""" return context.modules[value].build_configuration() - - -def deprecated_method(replacement: str, additional_information: str = ""): - """A decorator for marking functions as deprecated. - - Args: - replacement: The replacement function overriding the deprecated API (full path preferred, starting from "volatility3."). String was preferred, for convenience and to prevent import conflicts on caller side. - additional_information: Information appended at the end of the deprecation message - """ - - def decorator(deprecated_func): - @functools.wraps(deprecated_func) - def wrapper(*args, **kwargs): - nonlocal replacement, additional_information - deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__name__}\" is deprecated, use \"{replacement}\" instead. {additional_information}" - vollog.warning(deprecation_msg) - # Return the wrapped function with its original arguments - return deprecated_func(*args, **kwargs) - - return wrapper - - return decorator diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3dc744f78..0b7ef751c 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -10,11 +10,16 @@ from typing import Iterator, List, Tuple, Optional, Union import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3 import framework -from volatility3.framework import constants, exceptions, interfaces, objects +from volatility3.framework import ( + constants, + exceptions, + interfaces, + objects, + Deprecation, +) from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions -from volatility3.framework.configuration import requirements vollog = logging.getLogger(__name__) @@ -455,8 +460,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): ## Deprecated APIs ## @classmethod - @requirements.deprecated_method( - replacement="volatility3.framework.symbols.linux.utilities.modules.Modules.mask_mods_list" + @Deprecation.deprecated_method( + replacement=linux_utilities_modules.Modules.mask_mods_list ) def mask_mods_list( cls, @@ -472,8 +477,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return linux_utilities_modules.Modules.mask_mods_list(context, layer_name, mods) @classmethod - @requirements.deprecated_method( - replacement="volatility3.framework.symbols.linux.utilities.modules.Modules.lookup_module_address" + @Deprecation.deprecated_method( + replacement=linux_utilities_modules.Modules.lookup_module_address ) def lookup_module_address( cls, From 2e1b77f4b3186bae3dfbc7e51e2ba51e9fd850e3 Mon Sep 17 00:00:00 2001 From: Daniel Davidov <35842733+Danking555@users.noreply.github.com> Date: Sat, 25 Jan 2025 16:22:49 +0200 Subject: [PATCH 242/268] Move LowStubLayout constants to windows.constants * Moved constants out of the class and moved to constants.windows * Applied ruff and black --- volatility3/framework/automagic/pdbscan.py | 26 +++---------------- .../framework/constants/windows/__init__.py | 18 +++++++++++++ 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 7d289bcb6..f9c0d853d 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -376,26 +376,6 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): valid_kernel = (virtual_layer_name, address, res[0]) return valid_kernel - class LowStubLayout: - """ - Represents the layout of the Low Stub which exists only on x64 machines with no virtualization/emulation, - responsible for transitioning from Real Mode(16 bit) to Protected Mode(32 bit) and Long Mode(64 bit) on boot/return from sleep. - Contains offsets to fields and structures within the undocumented structure _PROCESSOR_START_BLOCK. - Here's a reference: https://github.com/mic101/windows/blob/master/WRK-v1.2/base/ntos/inc/amd64.h#L3334 - """ - - # Expected signature for validation, constructed from: - # PROCESSOR_START_BLOCK->Jmp->OpCode | PROCESSOR_START_BLOCK->Jmp->Offset | PROCESSOR_START_BLOCK->CompletionFlag - JMP_AND_COMPLETION_SIGNATURE = 0x00000001000600E9 - - # Address of LmTarget (Long Mode target) - PROCESSOR_START_BLOCK_LM_TARGET_OFFSET = ( - 0x70 # PROCESSOR_START_BLOCK->LmTarget, PVOID 8 bytes - ) - - # CR3 register within structures describing initial processor state to be started - PROCESSOR_START_BLOCK_CR3_OFFSET = 0xA0 # PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3, ULONG64 8 bytes - def method_low_stub_offset( self, context: interfaces.context.ContextInterface, @@ -417,12 +397,12 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): ) if ( 0xFFFFFFFFFFFF00FF & jmp_and_completion_values - != self.LowStubLayout.JMP_AND_COMPLETION_SIGNATURE + != constants.windows.JMP_AND_COMPLETION_SIGNATURE ): continue cr3_value = int.from_bytes( physical_layer.read( - offset + self.LowStubLayout.PROCESSOR_START_BLOCK_CR3_OFFSET, 0x8 + offset + constants.windows.PROCESSOR_START_BLOCK_CR3_OFFSET, 0x8 ), "little", ) @@ -434,7 +414,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): continue potential_kernel_hint = int.from_bytes( physical_layer.read( - offset + self.LowStubLayout.PROCESSOR_START_BLOCK_LM_TARGET_OFFSET, + offset + constants.windows.PROCESSOR_START_BLOCK_LM_TARGET_OFFSET, 0x8, ), "little", diff --git a/volatility3/framework/constants/windows/__init__.py b/volatility3/framework/constants/windows/__init__.py index 7face984a..6f37acd2d 100644 --- a/volatility3/framework/constants/windows/__init__.py +++ b/volatility3/framework/constants/windows/__init__.py @@ -10,3 +10,21 @@ KERNEL_MODULE_NAMES = ["ntkrnlmp", "ntkrnlpa", "ntkrpamp", "ntoskrnl"] """The list of names that kernel modules can have within the windows OS""" PE_MAX_EXTRACTION_SIZE = 1024 * 1024 * 256 + +""" +The following constants represent the layout of the Low Stub which exists only on x64 machines with no virtualization/emulation, +responsible for transitioning from Real Mode(16 bit) to Protected Mode(32 bit) and Long Mode(64 bit) on boot/return from sleep. +Contains offsets to fields and structures within the undocumented structure _PROCESSOR_START_BLOCK. +Here's a reference: https://github.com/mic101/windows/blob/master/WRK-v1.2/base/ntos/inc/amd64.h#L3334 +""" +# Expected signature for validation, constructed from: +# PROCESSOR_START_BLOCK->Jmp->OpCode | PROCESSOR_START_BLOCK->Jmp->Offset | PROCESSOR_START_BLOCK->CompletionFlag +JMP_AND_COMPLETION_SIGNATURE = 0x00000001000600E9 + +# Address of LmTarget (Long Mode target) +PROCESSOR_START_BLOCK_LM_TARGET_OFFSET = ( + 0x70 # PROCESSOR_START_BLOCK->LmTarget, PVOID 8 bytes +) + +# CR3 register within structures describing initial processor state to be started +PROCESSOR_START_BLOCK_CR3_OFFSET = 0xA0 # PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3, ULONG64 8 bytes From d8658f0729f7abbd8410f76571aad6369deabee6 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 17:08:27 +0100 Subject: [PATCH 243/268] put deprecated functions order back --- .../framework/symbols/linux/__init__.py | 75 +++++++++---------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0b7ef751c..bc2492f7a 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -345,6 +345,23 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): yield fd_num, filp, full_path + @classmethod + @Deprecation.deprecated_method( + replacement=linux_utilities_modules.Modules.mask_mods_list + ) + def mask_mods_list( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + mods: Iterator[interfaces.objects.ObjectInterface], + ) -> List[Tuple[str, int, int]]: + """ + DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.mask_mods_list" instead. + + A helper function to mask the starting and end address of kernel modules + """ + return linux_utilities_modules.Modules.mask_mods_list(context, layer_name, mods) + @classmethod def generate_kernel_handler_info( cls, @@ -372,6 +389,26 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): context, kernel.layer_name, mods_list ) + @classmethod + @Deprecation.deprecated_method( + replacement=linux_utilities_modules.Modules.lookup_module_address + ) + def lookup_module_address( + cls, + kernel_module: interfaces.context.ModuleInterface, + handlers: List[Tuple[str, int, int]], + target_address: int, + ) -> Tuple[str, str]: + """ + DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.lookup_module_address" instead. + + Searches between the start and end address of the kernel module using target_address. + Returns the module and symbol name of the address provided. + """ + return linux_utilities_modules.Modules.lookup_module_address( + kernel_module.context, kernel_module.name, handlers, target_address + ) + @classmethod def walk_internal_list(cls, vmlinux, struct_name, list_member, list_start): while list_start: @@ -458,44 +495,6 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): [chr((code >> (i * 8)) & 0xFF) for i in range(code_bytes_length)] ) - ## Deprecated APIs ## - @classmethod - @Deprecation.deprecated_method( - replacement=linux_utilities_modules.Modules.mask_mods_list - ) - def mask_mods_list( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - mods: Iterator[interfaces.objects.ObjectInterface], - ) -> List[Tuple[str, int, int]]: - """ - DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.mask_mods_list" instead. - - A helper function to mask the starting and end address of kernel modules - """ - return linux_utilities_modules.Modules.mask_mods_list(context, layer_name, mods) - - @classmethod - @Deprecation.deprecated_method( - replacement=linux_utilities_modules.Modules.lookup_module_address - ) - def lookup_module_address( - cls, - kernel_module: interfaces.context.ModuleInterface, - handlers: List[Tuple[str, int, int]], - target_address: int, - ) -> Tuple[str, str]: - """ - DEPRECATED: use "volatility3.framework.symbols.linux.utilities.modules.Modules.lookup_module_address" instead. - - Searches between the start and end address of the kernel module using target_address. - Returns the module and symbol name of the address provided. - """ - return linux_utilities_modules.Modules.lookup_module_address( - kernel_module.context, kernel_module.name, handlers, target_address - ) - class IDStorage(ABC): """Abstraction to support both XArray and RadixTree""" From 825720ed5d8f290b244735c758149e5b9d208c12 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 19:48:31 +0100 Subject: [PATCH 244/268] catch UnsatisfiedException at plugin runtime --- volatility3/cli/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 82a2a4205..d3ce74847 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -500,6 +500,16 @@ class CommandLine: renderer.filter = text_filter.CLIFilter(grid, args.filters) renderer.column_hide_list = args.hide_columns renderer.render(grid) + except exceptions.UnsatisfiedException as excp: + output = sys.stderr + output.write( + "An unsatisfied framework exception was encountered post plugin construction:\n" + ) + self.process_unsatisfied_exceptions(excp) + output.write( + f"Unable to validate the requirements: {[x for x in excp.unsatisfied]}\n", + ) + sys.exit(1) except exceptions.VolatilityException as excp: self.process_exceptions(excp) From 2ed00cc91a6764d05162c01055fc17c18124c6aa Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 19:52:47 +0100 Subject: [PATCH 245/268] add optional version requirement to deprecated_method --- volatility3/framework/__init__.py | 56 ++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 12254ca77..bf4ec4447 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -12,9 +12,11 @@ import logging import os import traceback import functools +import warnings from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Type, TypeVar -from volatility3.framework import constants, interfaces +from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework.configuration import requirements # ## @@ -57,20 +59,64 @@ class Deprecation: """Deprecation related methods.""" @staticmethod - def deprecated_method(replacement: Callable, additional_information: str = ""): + def deprecated_method( + replacement: Callable, + replacement_base_class_required_version: Tuple[int, int, int] = None, + additional_information: str = "", + ): """A decorator for marking functions as deprecated. Args: replacement: The replacement function overriding the deprecated API, in the form of a Callable (typically a method) + replacement_base_class_required_version: The "replacement" base class version that the deprecated method expects before proxying to it. This implies that "replacement" is a method from a class that inherits from VersionableInterface. additional_information: Information appended at the end of the deprecation message """ def decorator(deprecated_func): @functools.wraps(deprecated_func) def wrapper(*args, **kwargs): - nonlocal replacement, additional_information - deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__name__}\" is deprecated, use \"{replacement.__module__ + '.' + replacement.__name__}\" instead. {additional_information}" - vollog.warning(deprecation_msg) + nonlocal replacement, replacement_base_class_required_version, additional_information + # Prevent version mismatches between deprecated (proxy) methods and the ones they proxy + if replacement_base_class_required_version is not None and callable( + replacement + ): + # example: replacement = volatility3.MyClass.my_dummy_function + # "MyClass.my_dummy_function" -> "MyClass" + replacement_base_class_name = replacement.__qualname__.split(".")[0] + # replacement.__globals__ example: {'MyClass': } + replacement_base_class = replacement.__globals__.get( + replacement_base_class_name + ) + + # Verify that the base class inherits from VersionableInterface + if inspect.isclass(replacement_base_class) and issubclass( + replacement_base_class, + interfaces.configuration.VersionableInterface, + ): + # Construct a requirement + req = requirements.VersionRequirement( + name=replacement_base_class.__name__, + component=replacement_base_class, + version=replacement_base_class_required_version, + ) + # Verify the requirement + if not req.matches_required( + req._version, req._component.version + ): + full_unsat_req_path = ( + deprecated_func.__module__ + + "." + + deprecated_func.__qualname__ + + "." + + req.name + ) + # Catched by the cli and redirected to process_unsatisfied_exceptions + raise exceptions.UnsatisfiedException( + {full_unsat_req_path: req} + ) + + deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__qualname__}\" is deprecated, use \"{replacement.__module__ + '.' + replacement.__qualname__}\" instead. {additional_information}" + warnings.warn(deprecation_msg, FutureWarning) # Return the wrapped function with its original arguments return deprecated_func(*args, **kwargs) From 3657c6fe5e8db92b1a6364a962b778a89e802624 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 19:54:20 +0100 Subject: [PATCH 246/268] require Modules >= 1.0.0 on deprecated methods --- volatility3/framework/symbols/linux/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index bc2492f7a..6a01efc3a 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -347,7 +347,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): @classmethod @Deprecation.deprecated_method( - replacement=linux_utilities_modules.Modules.mask_mods_list + replacement=linux_utilities_modules.Modules.mask_mods_list, + replacement_base_class_required_version=(1, 0, 0), ) def mask_mods_list( cls, @@ -391,7 +392,8 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): @classmethod @Deprecation.deprecated_method( - replacement=linux_utilities_modules.Modules.lookup_module_address + replacement=linux_utilities_modules.Modules.lookup_module_address, + replacement_base_class_required_version=(1, 0, 0), ) def lookup_module_address( cls, From 4262eff8898b3fbc017abe3c1d1e4f13fa6fb189 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 25 Jan 2025 20:04:51 +0100 Subject: [PATCH 247/268] adhere to AbstractNetfilter requirement checking --- .../framework/plugins/linux/netfilter.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/netfilter.py b/volatility3/framework/plugins/linux/netfilter.py index 33a8ca7cc..ccb7509aa 100644 --- a/volatility3/framework/plugins/linux/netfilter.py +++ b/volatility3/framework/plugins/linux/netfilter.py @@ -99,6 +99,20 @@ class AbstractNetfilter(ABC): f"linux.LinuxUtilities version not suitable: required {linuxutils_required_version} found {linuxutils_current_version}" ) + linux_utilities_modules_required_version = ( + Netfilter._required_linux_utilities_modules_version + ) + linux_utilities_modules_current_version = ( + linux_utilities_modules.Modules._version + ) + if not requirements.VersionRequirement.matches_required( + linux_utilities_modules_required_version, + linux_utilities_modules_current_version, + ): + raise exceptions.PluginRequirementException( + f"linux_utilities_modules.Modules version not suitable: required {linux_utilities_modules_required_version} found {linux_utilities_modules_current_version}" + ) + modules = lsmod.Lsmod.list_modules(context, kernel_module_name) self.handlers = linux.LinuxUtilities.generate_kernel_handler_info( context, kernel_module_name, modules @@ -680,6 +694,7 @@ class Netfilter(interfaces.plugins.PluginInterface): _version = (1, 1, 0) + _required_linux_utilities_modules_version = (1, 0, 0) _required_linuxutils_version = (2, 1, 0) _required_lsmod_version = (2, 0, 0) @@ -694,7 +709,7 @@ class Netfilter(interfaces.plugins.PluginInterface): requirements.VersionRequirement( name="linux_utilities_modules", component=linux_utilities_modules.Modules, - version=(1, 0, 0), + version=cls._required_linux_utilities_modules_version, ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=cls._required_lsmod_version From 6d43dcd3a842d308705f3ecd3c023c94be473846 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 10:31:46 +0100 Subject: [PATCH 248/268] add VersionMismatchException --- volatility3/framework/exceptions.py | 31 ++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index 41c67b88d..0409ae5f0 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -8,9 +8,10 @@ space or symbol tables, and by layers when an address is invalid. The :class:`PagedInvalidAddressException` contains information about the size of the invalid page. """ -from typing import Dict, Optional +from typing import Callable, Dict, Optional, Tuple from volatility3.framework import interfaces +from volatility3.framework.interfaces.configuration import VersionableInterface class VolatilityException(Exception): @@ -134,3 +135,31 @@ class RenderException(VolatilityException): class LinuxPageCacheException(VolatilityException): """Thrown if there is an error during Linux Page Cache processing""" + + +class VersionMismatchException(VolatilityException): + """Thrown if a version mismatch has been encountered between two components.""" + + def __init__( + self, + source_component: Callable, + target_component: VersionableInterface, + target_version: Tuple[int, int, int], + failure_reason: str = None, + *args, + ): + """ + Args: + source_component: The component that required the target component + target_component: The component that is required. Must inherit from VersionableInterface + target_version: The version of the target component that was required, and ultimately was not satisfied + failure_reason: A detailed failure reason to enhande debugging and bug tracking + """ + super().__init__(*args) + self.source_component = source_component + self.target_component = target_component + self.target_version = target_version + self.failure_reason = failure_reason + + def __str__(self): + return f"{self.source_component.__module__+ '.' + self.source_component.__qualname__}: Version {self.target_version} dependency on {self.target_component.__module__+ '.' + self.target_component.__qualname__} {self.target_component.version} unmet." From 4bd385cc9dadbfe8800fddc2bce9cef1ae4dadd5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 10:33:02 +0100 Subject: [PATCH 249/268] handle VersionMismatchException --- volatility3/cli/__init__.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index d3ce74847..b57d9a3f3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -500,16 +500,6 @@ class CommandLine: renderer.filter = text_filter.CLIFilter(grid, args.filters) renderer.column_hide_list = args.hide_columns renderer.render(grid) - except exceptions.UnsatisfiedException as excp: - output = sys.stderr - output.write( - "An unsatisfied framework exception was encountered post plugin construction:\n" - ) - self.process_unsatisfied_exceptions(excp) - output.write( - f"Unable to validate the requirements: {[x for x in excp.unsatisfied]}\n", - ) - sys.exit(1) except exceptions.VolatilityException as excp: self.process_exceptions(excp) @@ -583,6 +573,8 @@ class CommandLine: fulltrace = traceback.TracebackException.from_exception(excp).format(chain=True) vollog.debug("".join(fulltrace)) + file_a_bug_msg = f"Please re-run with -vvv and file a bug with the output at {constants.BUG_URL}" + if isinstance(excp, exceptions.InvalidAddressException): general = "Volatility was unable to read a requested page:" if isinstance(excp, exceptions.SwappedInvalidAddressException): @@ -627,9 +619,7 @@ class CommandLine: elif isinstance(excp, exceptions.LayerException): general = f"Volatility experienced a layer-related issue: {excp.layer_name}" detail = f"{excp}" - caused_by = [ - "A faulty layer implementation (re-run with -vvv and file a bug)" - ] + caused_by = [f"A faulty layer implementation. {file_a_bug_msg}"] elif isinstance(excp, exceptions.MissingModuleException): general = f"Volatility could not import a necessary module: {excp.module}" detail = f"{excp}" @@ -640,13 +630,17 @@ class CommandLine: general = "Volatility experienced an issue when rendering the output:" detail = f"{excp}" caused_by = ["An invalid renderer option, such as no visible columns"] + elif isinstance(excp, exceptions.VersionMismatchException): + general = "A version mismatch was detected between two components:" + detail = f"{excp}" + caused_by = [ + excp.failure_reason or "An outdated API caller, such as a method.", + file_a_bug_msg, + ] else: general = "Volatility encountered an unexpected situation." detail = "" - caused_by = [ - "Please re-run using with -vvv and file a bug with the output", - f"at {constants.BUG_URL}", - ] + caused_by = [file_a_bug_msg] # Code that actually renders the exception output = sys.stderr From ba6b709aba9496f75a61a806f06719189ac97491 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 10:34:18 +0100 Subject: [PATCH 250/268] use VersionMismatchException --- volatility3/framework/__init__.py | 37 ++++++++++--------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index bf4ec4447..5d4e0f3c4 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -61,25 +61,23 @@ class Deprecation: @staticmethod def deprecated_method( replacement: Callable, - replacement_base_class_required_version: Tuple[int, int, int] = None, + replacement_version: Tuple[int, int, int] = None, additional_information: str = "", ): """A decorator for marking functions as deprecated. Args: replacement: The replacement function overriding the deprecated API, in the form of a Callable (typically a method) - replacement_base_class_required_version: The "replacement" base class version that the deprecated method expects before proxying to it. This implies that "replacement" is a method from a class that inherits from VersionableInterface. + replacement_version: The "replacement" base class version that the deprecated method expects before proxying to it. This implies that "replacement" is a method from a class that inherits from VersionableInterface. additional_information: Information appended at the end of the deprecation message """ def decorator(deprecated_func): @functools.wraps(deprecated_func) def wrapper(*args, **kwargs): - nonlocal replacement, replacement_base_class_required_version, additional_information + nonlocal replacement, replacement_version, additional_information # Prevent version mismatches between deprecated (proxy) methods and the ones they proxy - if replacement_base_class_required_version is not None and callable( - replacement - ): + if replacement_version is not None and callable(replacement): # example: replacement = volatility3.MyClass.my_dummy_function # "MyClass.my_dummy_function" -> "MyClass" replacement_base_class_name = replacement.__qualname__.split(".")[0] @@ -93,26 +91,15 @@ class Deprecation: replacement_base_class, interfaces.configuration.VersionableInterface, ): - # Construct a requirement - req = requirements.VersionRequirement( - name=replacement_base_class.__name__, - component=replacement_base_class, - version=replacement_base_class_required_version, - ) - # Verify the requirement - if not req.matches_required( - req._version, req._component.version + # SemVer check + if not requirements.VersionRequirement.matches_required( + replacement_version, replacement_base_class.version ): - full_unsat_req_path = ( - deprecated_func.__module__ - + "." - + deprecated_func.__qualname__ - + "." - + req.name - ) - # Catched by the cli and redirected to process_unsatisfied_exceptions - raise exceptions.UnsatisfiedException( - {full_unsat_req_path: req} + raise exceptions.VersionMismatchException( + deprecated_func, + replacement_base_class, + replacement_version, + "A deprecated method was unable to proxy the call to its replacement", ) deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__qualname__}\" is deprecated, use \"{replacement.__module__ + '.' + replacement.__qualname__}\" instead. {additional_information}" From 28081ed989599bc395b190cf503e8d83ef5b8fed Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 10:35:25 +0100 Subject: [PATCH 251/268] tidy up replacement_base_class_required_version --- volatility3/framework/symbols/linux/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 6a01efc3a..397c36c01 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -348,7 +348,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): @classmethod @Deprecation.deprecated_method( replacement=linux_utilities_modules.Modules.mask_mods_list, - replacement_base_class_required_version=(1, 0, 0), + replacement_version=(1, 0, 0), ) def mask_mods_list( cls, @@ -393,7 +393,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): @classmethod @Deprecation.deprecated_method( replacement=linux_utilities_modules.Modules.lookup_module_address, - replacement_base_class_required_version=(1, 0, 0), + replacement_version=(1, 0, 0), ) def lookup_module_address( cls, From 459483651b847241cb4189c4d1449fd18c0bdca7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 10:53:33 +0100 Subject: [PATCH 252/268] typo --- volatility3/framework/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index 0409ae5f0..99c5f155e 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -153,7 +153,7 @@ class VersionMismatchException(VolatilityException): source_component: The component that required the target component target_component: The component that is required. Must inherit from VersionableInterface target_version: The version of the target component that was required, and ultimately was not satisfied - failure_reason: A detailed failure reason to enhande debugging and bug tracking + failure_reason: A detailed failure reason to enhance debugging and bug tracking """ super().__init__(*args) self.source_component = source_component From 80eebd2f49bee665f194e4adba92cf12a192a997 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 13:18:30 +0100 Subject: [PATCH 253/268] use classmethod instead of staticmethod --- volatility3/framework/symbols/linux/utilities/modules.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/modules.py b/volatility3/framework/symbols/linux/utilities/modules.py index ac9b2afaf..82c63fc18 100644 --- a/volatility3/framework/symbols/linux/utilities/modules.py +++ b/volatility3/framework/symbols/linux/utilities/modules.py @@ -13,8 +13,9 @@ class Modules(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) - @staticmethod + @classmethod def mask_mods_list( + cls, context: interfaces.context.ContextInterface, layer_name: str, mods: Iterator[interfaces.objects.ObjectInterface], @@ -33,8 +34,9 @@ class Modules(interfaces.configuration.VersionableInterface): for mod in mods ] - @staticmethod + @classmethod def lookup_module_address( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str, handlers: List[Tuple[str, int, int]], From 5d58ba63dbd6a60fd36faca6f23a3a1b4a11e724 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 13:19:01 +0100 Subject: [PATCH 254/268] use __name__ instead of overkill __qualname__ --- volatility3/framework/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index 99c5f155e..a3d660444 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -162,4 +162,4 @@ class VersionMismatchException(VolatilityException): self.failure_reason = failure_reason def __str__(self): - return f"{self.source_component.__module__+ '.' + self.source_component.__qualname__}: Version {self.target_version} dependency on {self.target_component.__module__+ '.' + self.target_component.__qualname__} {self.target_component.version} unmet." + return f"{self.source_component.__module__+ '.' + self.source_component.__qualname__}: Version {self.target_version} dependency on {self.target_component.__module__+ '.' + self.target_component.__name__} {self.target_component.version} unmet." From 4cb386858ebe7c62f05b16c9f869fdf762084121 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 13:20:48 +0100 Subject: [PATCH 255/268] use __self__ and enhance exception msg --- volatility3/framework/__init__.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 5d4e0f3c4..aa93340cd 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -77,14 +77,12 @@ class Deprecation: def wrapper(*args, **kwargs): nonlocal replacement, replacement_version, additional_information # Prevent version mismatches between deprecated (proxy) methods and the ones they proxy - if replacement_version is not None and callable(replacement): - # example: replacement = volatility3.MyClass.my_dummy_function - # "MyClass.my_dummy_function" -> "MyClass" - replacement_base_class_name = replacement.__qualname__.split(".")[0] - # replacement.__globals__ example: {'MyClass': } - replacement_base_class = replacement.__globals__.get( - replacement_base_class_name - ) + if ( + replacement_version is not None + and callable(replacement) + and hasattr(replacement, "__self__") + ): + replacement_base_class = replacement.__self__ # Verify that the base class inherits from VersionableInterface if inspect.isclass(replacement_base_class) and issubclass( @@ -99,7 +97,7 @@ class Deprecation: deprecated_func, replacement_base_class, replacement_version, - "A deprecated method was unable to proxy the call to its replacement", + "This is a bug, the deprecated call needs to be removed and the caller needs to update their code to use the new method.", ) deprecation_msg = f"Method \"{deprecated_func.__module__ + '.' + deprecated_func.__qualname__}\" is deprecated, use \"{replacement.__module__ + '.' + replacement.__qualname__}\" instead. {additional_information}" From a25165d935c02c07510b8b4721b074d7bed21258 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 26 Jan 2025 13:44:23 +0100 Subject: [PATCH 256/268] 2.18.1 -> 2.19.0 bump --- volatility3/framework/constants/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 689e39664..f2403cf4a 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 18 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 19 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From 1ac2dbc49c10552d1d26debe2eb54e0f6922c1c8 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 26 Jan 2025 13:44:33 +0000 Subject: [PATCH 257/268] Shift exposed staticmethods to classmethods --- .../framework/plugins/linux/capabilities.py | 4 ++- volatility3/framework/plugins/linux/envars.py | 5 +-- .../framework/plugins/linux/hidden_modules.py | 6 ++-- .../framework/plugins/linux/pagecache.py | 11 +++--- .../framework/plugins/linux/vmayarascan.py | 5 +-- .../framework/plugins/windows/cachedump.py | 18 +++++----- .../plugins/windows/direct_system_calls.py | 13 +++---- .../framework/plugins/windows/mftscan.py | 21 ++++++----- .../framework/plugins/windows/netscan.py | 6 ++-- .../framework/plugins/windows/pe_symbols.py | 35 +++++++++++-------- .../framework/plugins/windows/poolscanner.py | 6 ++-- .../framework/plugins/windows/shimcachemem.py | 4 ++- .../framework/plugins/windows/svcscan.py | 5 +-- .../plugins/windows/unloadedmodules.py | 5 +-- .../framework/plugins/windows/vadyarascan.py | 5 +-- volatility3/framework/plugins/yarascan.py | 14 ++++---- 16 files changed, 93 insertions(+), 70 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 1d0c60c11..b758a04b4 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -35,7 +35,9 @@ class CapabilitiesData: cap_permitted: interfaces.objects.ObjectInterface cap_effective: interfaces.objects.ObjectInterface cap_bset: interfaces.objects.ObjectInterface - cap_ambient: interfaces.objects.ObjectInterface + cap_ambient: ( + interfaces.objects.ObjectInterface | interfaces.renderers.BaseAbsentValue + ) def astuple(self) -> Tuple: """Returns a shallow copy of the capability sets in a tuple. diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 8cdbfe493..cc43c4130 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -18,7 +18,7 @@ class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" _required_framework_version = (2, 13, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls): @@ -40,8 +40,9 @@ class Envars(plugins.PluginInterface): ), ] - @staticmethod + @classmethod def get_task_env_variables( + cls, context: interfaces.context.ContextInterface, task: interfaces.objects.ObjectInterface, env_area_max_size: int = 8192, diff --git a/volatility3/framework/plugins/linux/hidden_modules.py b/volatility3/framework/plugins/linux/hidden_modules.py index fd4b28943..e1ba40926 100644 --- a/volatility3/framework/plugins/linux/hidden_modules.py +++ b/volatility3/framework/plugins/linux/hidden_modules.py @@ -16,8 +16,7 @@ class Hidden_modules(interfaces.plugins.PluginInterface): """Carves memory to find hidden kernel modules""" _required_framework_version = (2, 10, 0) - - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -32,8 +31,9 @@ class Hidden_modules(interfaces.plugins.PluginInterface): ), ] - @staticmethod + @classmethod def get_modules_memory_boundaries( + cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, ) -> Tuple[int]: diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 32b176b72..4d1250255 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -104,7 +104,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -360,8 +360,8 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): yield description, timeliner.TimeLinerType.MODIFIED, inode_out.modification_time yield description, timeliner.TimeLinerType.CHANGED, inode_out.change_time - @staticmethod - def format_fields_with_headers(headers, generator): + @classmethod + def format_fields_with_headers(cls, headers, generator): """Uses the headers type to cast the fields obtained from the generator""" for level, fields in generator: formatted_fields = [] @@ -405,7 +405,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (2, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -436,8 +436,9 @@ class InodePages(plugins.PluginInterface): ), ] - @staticmethod + @classmethod def write_inode_content_to_file( + cls, inode: interfaces.objects.ObjectInterface, filename: str, open_method: Type[interfaces.plugins.FileHandlerInterface], diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 4db23e50b..e9e56dd0f 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -18,7 +18,7 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 2) + _version = (1, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -105,8 +105,9 @@ class VmaYaraScan(interfaces.plugins.PluginInterface): value, ) - @staticmethod + @classmethod def get_vma_maps( + cls, task: interfaces.objects.ObjectInterface, ) -> Iterable[Tuple[int, int]]: """Creates a map of start/end addresses for each virtual memory area in a task. diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index 6c730e6ae..f4f2e061e 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -22,7 +22,7 @@ class Cachedump(interfaces.plugins.PluginInterface): """Dumps lsa secrets from memory""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -43,16 +43,16 @@ class Cachedump(interfaces.plugins.PluginInterface): ), ] - @staticmethod + @classmethod def get_nlkm( - sechive: registry.RegistryHive, lsakey: bytes, is_vista_or_later: bool + cls, sechive: registry.RegistryHive, lsakey: bytes, is_vista_or_later: bool ): return lsadump.Lsadump.get_secret_by_name( sechive, "NL$KM", lsakey, is_vista_or_later ) - @staticmethod - def decrypt_hash(edata: bytes, nlkm: bytes, ch, xp: bool): + @classmethod + def decrypt_hash(cls, edata: bytes, nlkm: bytes, ch, xp: bool): if xp: hmac_md5 = HMAC.new(nlkm, ch) rc4key = hmac_md5.digest() @@ -69,8 +69,8 @@ class Cachedump(interfaces.plugins.PluginInterface): data += aes.decrypt(buf) return data - @staticmethod - def parse_cache_entry(cache_data: bytes) -> Tuple[int, int, int, bytes, bytes]: + @classmethod + def parse_cache_entry(cls, cache_data: bytes) -> Tuple[int, int, int, bytes, bytes]: (uname_len, domain_len) = unpack(" Tuple[str, str, str, bytes]: """Get the data from the cache and separate it into the username, domain name, and hash data""" uname_offset = 72 diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py index 183e4095c..af626f511 100644 --- a/volatility3/framework/plugins/windows/direct_system_calls.py +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -53,7 +53,7 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): """Detects the Direct System Call technique used to bypass EDRs""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) # DLLs that are expected to host system call invocations valid_syscall_handlers = ("ntdll.dll", "win32u.dll") @@ -200,8 +200,8 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return disasm_bytes, end_inst - @staticmethod - def get_disasm_function(architecture: str) -> Callable: + @classmethod + def get_disasm_function(cls, architecture: str) -> Callable: """ Returns the disassembly handler for the given architecture .detail is used to get full instruction information @@ -284,8 +284,9 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return None - @staticmethod + @classmethod def get_vad_maps( + cls, task: interfaces.objects.ObjectInterface, ) -> List[Tuple[int, int, str]]: """Creates a map of start/end addresses within a virtual address @@ -310,9 +311,9 @@ class DirectSystemCalls(interfaces.plugins.PluginInterface): return vads - @staticmethod + @classmethod def get_range_path( - ranges: List[Tuple[int, int, str]], address: int + cls, ranges: List[Tuple[int, int, str]], address: int ) -> Optional[str]: """ Returns the path for the range holding `address`, if found diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index c4d05e634..2c5827a25 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -22,7 +22,7 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls): @@ -37,8 +37,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] - @staticmethod + @classmethod def enumerate_mft_records( + cls, context: interfaces.context.ContextInterface, config_path: str, primary_layer_name: str, @@ -128,8 +129,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): layer_name=layer.name, ) - @staticmethod + @classmethod def parse_mft_records( + cls, record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, @@ -191,8 +193,9 @@ class MFTScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_name, ) - @staticmethod + @classmethod def parse_data_record( + cls, mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, record_map: Dict[int, Tuple[str, int, int]], @@ -325,7 +328,7 @@ class ADS(interfaces.plugins.PluginInterface): _required_framework_version = (2, 7, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -343,8 +346,9 @@ class ADS(interfaces.plugins.PluginInterface): ), ] - @staticmethod + @classmethod def parse_ads_data_records( + cls, record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, @@ -394,7 +398,7 @@ class ResidentData(interfaces.plugins.PluginInterface): _required_framework_version = (2, 7, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -412,8 +416,9 @@ class ResidentData(interfaces.plugins.PluginInterface): ), ] - @staticmethod + @classmethod def parse_first_data_records( + cls, record_map: Dict[int, Tuple[str, int, int]], mft_record: interfaces.objects.ObjectInterface, attr: interfaces.objects.ObjectInterface, diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 162031104..c30792908 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -23,7 +23,7 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for network objects present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls): @@ -50,9 +50,9 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), ] - @staticmethod + @classmethod def create_netscan_constraints( - context: interfaces.context.ContextInterface, symbol_table: str + cls, context: interfaces.context.ContextInterface, symbol_table: str ) -> List[poolscanner.PoolConstraint]: """Creates a list of Pool Tag Constraints for network objects. diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 21e657ab3..88ced7e06 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -244,7 +244,7 @@ class PESymbols(interfaces.plugins.PluginInterface): _required_framework_version = (2, 7, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) # used for special handling of the kernel PDB file. See later notes os_module_name = "ntoskrnl.exe" @@ -330,9 +330,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return pe_ret - @staticmethod + @classmethod def range_info_for_address( - ranges: ranges_type, address: int + cls, ranges: ranges_type, address: int ) -> Optional[range_type]: """ Helper for getting the range information for an address. @@ -351,8 +351,8 @@ class PESymbols(interfaces.plugins.PluginInterface): return None - @staticmethod - def filepath_for_address(ranges: ranges_type, address: int) -> Optional[str]: + @classmethod + def filepath_for_address(cls, ranges: ranges_type, address: int) -> Optional[str]: """ Helper to get the file path for an address @@ -369,8 +369,8 @@ class PESymbols(interfaces.plugins.PluginInterface): return None - @staticmethod - def filename_for_path(filepath: str) -> str: + @classmethod + def filename_for_path(cls, filepath: str) -> str: """ Consistent way to get the filename regardless of platform @@ -382,8 +382,9 @@ class PESymbols(interfaces.plugins.PluginInterface): """ return ntpath.basename(filepath).lower() - @staticmethod + @classmethod def addresses_for_process_symbols( + cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, @@ -416,8 +417,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_symbols - @staticmethod + @classmethod def path_and_symbol_for_address( + cls, context: interfaces.context.ContextInterface, config_path: str, collected_modules: collected_modules_type, @@ -733,8 +735,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return found, remaining - @staticmethod + @classmethod def find_symbols( + cls, context: interfaces.context.ContextInterface, config_path: str, wanted_modules: PESymbolFinder.cached_value_dict, @@ -775,8 +778,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_symbols, missing_symbols - @staticmethod + @classmethod def get_kernel_modules( + cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, @@ -837,8 +841,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return found_modules - @staticmethod + @classmethod def get_vads_for_process_cache( + cls, vads_cache: Dict[int, ranges_type], owner_proc: interfaces.objects.ObjectInterface, ) -> Optional[ranges_type]: @@ -865,8 +870,9 @@ class PESymbols(interfaces.plugins.PluginInterface): return vads - @staticmethod + @classmethod def get_proc_vads_with_file_paths( + cls, proc: interfaces.objects.ObjectInterface, ) -> ranges_type: """ @@ -928,8 +934,9 @@ class PESymbols(interfaces.plugins.PluginInterface): yield proc, proc_layer_name, vads - @staticmethod + @classmethod def get_process_modules( + cls, context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index efde09638..5be0e7fa8 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -127,8 +127,8 @@ class PoolHeaderScanner(interfaces.layers.ScannerInterface): class PoolScanner(plugins.PluginInterface): """A generic pool scanner plugin.""" - _version = (1, 0, 0) _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -181,9 +181,9 @@ class PoolScanner(plugins.PluginInterface): ), ) - @staticmethod + @classmethod def builtin_constraints( - symbol_table: str, tags_filter: Optional[List[bytes]] = None + cls, symbol_table: str, tags_filter: Optional[List[bytes]] = None ) -> List[PoolConstraint]: """Get built-in PoolConstraints given a list of pool tags. diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index 9d968c30a..1e1024656 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -24,6 +24,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf """Reads Shimcache entries from the ahcache.sys AVL tree""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 1) # These checks must be completed from newest -> oldest OS version. _win_version_file_map: List[Tuple[versions.OsDistinguisher, bool, str]] = [ @@ -74,8 +75,9 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf ), ] - @staticmethod + @classmethod def create_shimcache_table( + cls, context: interfaces.context.ContextInterface, symbol_table: str, config_path: str, diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 17baac5b0..6645fa6a3 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -35,7 +35,7 @@ class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 1) + _version = (3, 0, 2) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -61,8 +61,9 @@ class SvcScan(interfaces.plugins.PluginInterface): ), ] - @staticmethod + @classmethod def get_record_tuple( + cls, service_record: interfaces.objects.ObjectInterface, binary_info: ServiceBinaryInfo, ): diff --git a/volatility3/framework/plugins/windows/unloadedmodules.py b/volatility3/framework/plugins/windows/unloadedmodules.py index 077fe33cb..d9f104ae8 100644 --- a/volatility3/framework/plugins/windows/unloadedmodules.py +++ b/volatility3/framework/plugins/windows/unloadedmodules.py @@ -22,7 +22,7 @@ class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInt """Lists the unloaded kernel modules.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -34,8 +34,9 @@ class UnloadedModules(interfaces.plugins.PluginInterface, timeliner.TimeLinerInt ), ] - @staticmethod + @classmethod def create_unloadedmodules_table( + cls, context: interfaces.context.ContextInterface, symbol_table: str, config_path: str, diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 2e9cc44ea..11ddc3716 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -18,7 +18,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 1, 1) + _version = (1, 1, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -104,8 +104,9 @@ class VadYaraScan(interfaces.plugins.PluginInterface): value, ) - @staticmethod + @classmethod def get_vad_maps( + cls, task: interfaces.objects.ObjectInterface, ) -> Iterable[Tuple[int, int]]: """Creates a map of start/end addresses within a virtual address diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 310bbd072..38c8b6085 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -37,7 +37,7 @@ except ImportError: class YaraScanner(interfaces.layers.ScannerInterface): - _version = (2, 1, 0) + _version = (2, 1, 1) # yara.Rules isn't exposed, so we can't type this properly def __init__(self, rules) -> None: @@ -79,23 +79,23 @@ class YaraScanner(interfaces.layers.ScannerInterface): for offset, name, value in match.strings: yield (offset + data_offset, match.rule, name, value) - @staticmethod - def get_rule(rule): + @classmethod + def get_rule(cls, rule): if USE_YARA_X: return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}") return yara.compile( sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"} ) - @staticmethod - def from_compiled_file(filepath): + @classmethod + def from_compiled_file(cls, filepath): with resources.ResourceAccessor().open(filepath, "rb") as fp: if USE_YARA_X: return yara_x.Rules.deserialize_from(file=fp) return yara.load(file=fp) - @staticmethod - def from_file(filepath): + @classmethod + def from_file(cls, filepath): with resources.ResourceAccessor().open(filepath, "rb") as fp: if USE_YARA_X: return yara_x.compile(fp.read().decode()) From e4e54a5c58e24b053b0509de952ea1647e07877f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 26 Jan 2025 13:48:00 +0000 Subject: [PATCH 258/268] Don't fix the type error as part of the shift. --- volatility3/framework/plugins/linux/capabilities.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index b758a04b4..1d0c60c11 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -35,9 +35,7 @@ class CapabilitiesData: cap_permitted: interfaces.objects.ObjectInterface cap_effective: interfaces.objects.ObjectInterface cap_bset: interfaces.objects.ObjectInterface - cap_ambient: ( - interfaces.objects.ObjectInterface | interfaces.renderers.BaseAbsentValue - ) + cap_ambient: interfaces.objects.ObjectInterface def astuple(self) -> Tuple: """Returns a shallow copy of the capability sets in a tuple. From 8b31ae612f8458e349cf0a063a578b97ec7ef8e9 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 27 Jan 2025 09:32:33 +0000 Subject: [PATCH 259/268] Layers: Make the low-stub method less brittle --- volatility3/framework/automagic/pdbscan.py | 64 ++++++++++++---------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index f9c0d853d..dd2ad0683 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -392,38 +392,42 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): # Try locating kernel base via x64 Low Stub in lower 1MB starting from second page (4KB) # If "Discard Low Memory" setting is disabled in BIOS, the Low Stub may be at the third/fourth or further pages for offset in range(0x1000, 0x100000, 0x1000): - jmp_and_completion_values = int.from_bytes( - physical_layer.read(offset, 0x8), "little" - ) - if ( - 0xFFFFFFFFFFFF00FF & jmp_and_completion_values - != constants.windows.JMP_AND_COMPLETION_SIGNATURE - ): - continue - cr3_value = int.from_bytes( - physical_layer.read( - offset + constants.windows.PROCESSOR_START_BLOCK_CR3_OFFSET, 0x8 - ), - "little", - ) + try: + jmp_and_completion_values = int.from_bytes( + physical_layer.read(offset, 0x8), "little" + ) + if ( + 0xFFFFFFFFFFFF00FF & jmp_and_completion_values + != constants.windows.JMP_AND_COMPLETION_SIGNATURE + ): + continue + cr3_value = int.from_bytes( + physical_layer.read( + offset + constants.windows.PROCESSOR_START_BLOCK_CR3_OFFSET, 0x8 + ), + "little", + ) - # Compare previously observed valid page table address that's stored in vlayer._initial_entry - # with PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3 - # which was observed to be an invalid page address, so add 1 (to make it valid too) - if (cr3_value + 1) != vlayer._initial_entry: + # Compare previously observed valid page table address that's stored in vlayer._initial_entry + # with PROCESSOR_START_BLOCK->ProcessorState->SpecialRegisters->Cr3 + # which was observed to be an invalid page address, so add 1 (to make it valid too) + if (cr3_value + 1) != vlayer._initial_entry: + continue + potential_kernel_hint = int.from_bytes( + physical_layer.read( + offset + + constants.windows.PROCESSOR_START_BLOCK_LM_TARGET_OFFSET, + 0x8, + ), + "little", + ) + if 0x3 & potential_kernel_hint: + continue + kernel_hint = potential_kernel_hint & 0xFFFFFFFFFFFF + kernel_base = kernel_hint & (~0x1FFFFF) & 0xFFFFFFFFFFFF + break + except exceptions.InvalidAddressException: continue - potential_kernel_hint = int.from_bytes( - physical_layer.read( - offset + constants.windows.PROCESSOR_START_BLOCK_LM_TARGET_OFFSET, - 0x8, - ), - "little", - ) - if 0x3 & potential_kernel_hint: - continue - kernel_hint = potential_kernel_hint & 0xFFFFFFFFFFFF - kernel_base = kernel_hint & (~0x1FFFFF) & 0xFFFFFFFFFFFF - break if kernel_base: # Scanning 32mb in 2mb chunks for the 'ntoskrnl' base address From 7d14f4c778d1f624a35a6db103f99b9c00cfd8b0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 27 Jan 2025 14:57:39 +0100 Subject: [PATCH 260/268] remove additional_description in favor of the docstring implementation --- volatility3/framework/interfaces/plugins.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 7ad78d0ba..f763815a6 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -112,8 +112,6 @@ class PluginInterface( # Be careful with inheritance around this (We default to requiring a version which doesn't exist, so it must be set) _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" - additional_description: str = None - """Display additional description of the plugin after the description of the arguments. See: https://docs.python.org/3/library/argparse.html#epilog""" def __init__( self, From d2859f2390070111f42e470dc9e1c6d6ad38a802 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 27 Jan 2025 15:00:03 +0100 Subject: [PATCH 261/268] remove additional_description in favor of the docstring implementation --- volatility3/cli/__init__.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index b57d9a3f3..020dac2d2 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -363,11 +363,20 @@ class CommandLine: metavar="PLUGIN", ) for plugin in sorted(plugin_list): + # First line of a plugin docstring will be the short description for -h + # Following lines will be the additional description (argparse epilog) + short_help = additional_help = None + if plugin_list[plugin].__doc__ is not None: + doc_split = plugin_list[plugin].__doc__.strip().split("\n", 1) + short_help = doc_split[0] + if len(doc_split) > 1: + additional_help = doc_split[1].strip() + plugin_parser = subparser.add_parser( plugin, - help=plugin_list[plugin].__doc__, - description=plugin_list[plugin].__doc__, - epilog=plugin_list[plugin].additional_description, + help=short_help, + description=short_help, + epilog=additional_help, ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) From 08f03642807056c0f3857eaa78cd4053fc7f391b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 27 Jan 2025 15:02:02 +0100 Subject: [PATCH 262/268] adapt plugins docstring to fit short and additional help format --- volatility3/framework/plugins/configwriter.py | 4 ++-- volatility3/framework/plugins/linux/modxview.py | 4 ++-- volatility3/framework/plugins/linux/pstree.py | 3 +-- volatility3/framework/plugins/mac/mount.py | 4 ++-- volatility3/framework/plugins/mac/pstree.py | 3 +-- volatility3/framework/plugins/timeliner.py | 4 ++-- volatility3/framework/plugins/windows/pstree.py | 3 +-- volatility3/framework/plugins/windows/psxview.py | 7 ++++--- volatility3/framework/plugins/windows/registry/hivescan.py | 3 +-- volatility3/framework/plugins/windows/scheduled_tasks.py | 5 ++--- 10 files changed, 18 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/configwriter.py b/volatility3/framework/plugins/configwriter.py index eca01a84a..a567a6acd 100644 --- a/volatility3/framework/plugins/configwriter.py +++ b/volatility3/framework/plugins/configwriter.py @@ -14,8 +14,8 @@ vollog = logging.getLogger(__name__) class ConfigWriter(plugins.PluginInterface): - """Runs the automagics and both prints and outputs configuration in the - output directory.""" + """Runs the automagics and both prints and outputs configuration in the \ +output directory.""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index c74bf28e8..0dd503829 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -15,8 +15,8 @@ vollog = logging.getLogger(__name__) class Modxview(interfaces.plugins.PluginInterface): - """Centralize lsmod, check_modules and hidden_modules results to efficiently - spot modules presence and taints.""" + """Centralize lsmod, check_modules and hidden_modules results to efficiently \ +spot modules presence and taints.""" _version = (1, 0, 0) _required_framework_version = (2, 17, 0) diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index 74e172139..fd28fcbbd 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -9,8 +9,7 @@ from volatility3.plugins.linux import pslist class PsTree(interfaces.plugins.PluginInterface): - """Plugin for listing processes in a tree based on their parent process - ID.""" + """Plugin for listing processes in a tree based on their parent process ID.""" _required_framework_version = (2, 13, 0) _version = (1, 1, 1) diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index 1a1e33571..0f3aa745c 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -11,8 +11,8 @@ from volatility3.framework.symbols import mac class Mount(plugins.PluginInterface): - """A module containing a collection of plugins that produce data typically - found in Mac's mount command""" + """A module containing a collection of plugins that produce data typically \ +found in Mac's mount command""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/mac/pstree.py b/volatility3/framework/plugins/mac/pstree.py index e62d5eb72..ad5bb309b 100644 --- a/volatility3/framework/plugins/mac/pstree.py +++ b/volatility3/framework/plugins/mac/pstree.py @@ -10,8 +10,7 @@ from volatility3.plugins.mac import pslist class PsTree(plugins.PluginInterface): - """Plugin for listing processes in a tree based on their parent process - ID.""" + """Plugin for listing processes in a tree based on their parent process ID.""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index 0f4064d79..6000704eb 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -41,8 +41,8 @@ class TimeLinerInterface(metaclass=abc.ABCMeta): class Timeliner(interfaces.plugins.PluginInterface): - """Runs all relevant plugins that provide time related information and - orders the results by time.""" + """Runs all relevant plugins that provide time related information and \ +orders the results by time.""" _required_framework_version = (2, 0, 0) _version = (1, 1, 0) diff --git a/volatility3/framework/plugins/windows/pstree.py b/volatility3/framework/plugins/windows/pstree.py index 2be96277c..4f3fe0455 100644 --- a/volatility3/framework/plugins/windows/pstree.py +++ b/volatility3/framework/plugins/windows/pstree.py @@ -14,8 +14,7 @@ vollog = logging.getLogger(__name__) class PsTree(interfaces.plugins.PluginInterface): - """Plugin for listing processes in a tree based on their parent process - ID.""" + """Plugin for listing processes in a tree based on their parent process ID.""" _required_framework_version = (2, 0, 0) diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index b5ddd2ee5..aa379bdc5 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -21,9 +21,10 @@ vollog = logging.getLogger(__name__) class PsXView(plugins.PluginInterface): - """Lists all processes found via four of the methods described in \"The Art of Memory Forensics,\" which may help - identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this - plugin's output in a terminal.""" + """Lists all processes found via four of the methods described in \"The Art of Memory Forensics\" which may help \ +identify processes that are trying to hide themselves. + +We recommend using -r pretty if you are looking at this plugin's output in a terminal.""" # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the functionality # which the original plugin used to do it. diff --git a/volatility3/framework/plugins/windows/registry/hivescan.py b/volatility3/framework/plugins/windows/registry/hivescan.py index 7b3c0b622..6e0171a78 100644 --- a/volatility3/framework/plugins/windows/registry/hivescan.py +++ b/volatility3/framework/plugins/windows/registry/hivescan.py @@ -12,8 +12,7 @@ from volatility3.plugins.windows import poolscanner, bigpools class HiveScan(interfaces.plugins.PluginInterface): - """Scans for registry hives present in a particular windows memory - image.""" + """Scans for registry hives present in a particular windows memory image.""" _required_framework_version = (2, 0, 0) _version = (1, 0, 0) diff --git a/volatility3/framework/plugins/windows/scheduled_tasks.py b/volatility3/framework/plugins/windows/scheduled_tasks.py index 6dd5613c4..31aaec4f0 100644 --- a/volatility3/framework/plugins/windows/scheduled_tasks.py +++ b/volatility3/framework/plugins/windows/scheduled_tasks.py @@ -1099,9 +1099,8 @@ class DynamicInfo: class ScheduledTasks(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): - """Decodes scheduled task information from the Windows registry, including - information about triggers, actions, run times, and creation times. - """ + """Decodes scheduled task information from the Windows registry, including \ +information about triggers, actions, run times, and creation times.""" _required_framework_version = (2, 11, 0) _version = (1, 0, 0) From f64291faec4991bbe13f0e5cda655ae463b1f3bb Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 27 Jan 2025 17:44:18 +0100 Subject: [PATCH 263/268] split help on two consecutive newlines --- volatility3/cli/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 020dac2d2..a41cf95a3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -363,12 +363,13 @@ class CommandLine: metavar="PLUGIN", ) for plugin in sorted(plugin_list): - # First line of a plugin docstring will be the short description for -h - # Following lines will be the additional description (argparse epilog) + # First line of a plugin docstring will be the short description for -h. + # Text after the first two consecutive new lines will be + # the additional description (argparse epilog). short_help = additional_help = None if plugin_list[plugin].__doc__ is not None: - doc_split = plugin_list[plugin].__doc__.strip().split("\n", 1) - short_help = doc_split[0] + doc_split = plugin_list[plugin].__doc__.split("\n\n", 1) + short_help = doc_split[0].strip() if len(doc_split) > 1: additional_help = doc_split[1].strip() From 105df4e140767045e51c69c9a24ad17d231564bc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 27 Jan 2025 22:29:59 +0000 Subject: [PATCH 264/268] Windows: Fix vadyarascan typo --- volatility3/framework/plugins/windows/vadyarascan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 2e9cc44ea..9758c5994 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -84,7 +84,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface): if not vad_maps_to_scan: vollog.warning( - f"No VADs were found for task {task.UniqueProcessID}, not scanning" + f"No VADs were found for task {task.UniqueProcessId}, not scanning" ) continue From 74a834b6de089a0ba8cca62d9e86314996faa9fb Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 09:22:58 +1100 Subject: [PATCH 265/268] linux: vfsmount: improve kernel implementation detection --- volatility3/framework/symbols/linux/extensions/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f612dfc3b..ec79b0203 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1575,13 +1575,11 @@ class vfsmount(objects.StructType): # the 'mnt_parent' member was relocated from the 'vfsmount' struct to the newly # introduced 'mount' struct. - Alternatively, vmlinux.has_type('mount') can be used here but it is faster. - Returns: 'True' if the kernel lacks the 'mount' struct, typically indicating kernel < 3.3. """ - return self.has_member("mnt_parent") + return not self._context.symbol_space.has_type("mount") def is_equal(self, vfsmount_ptr) -> bool: """Helper to make sure it is comparing two pointers to 'vfsmount'. From 53364f7dab197b4d3b83de259cc7ff632016e8ad Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 10:41:00 +1100 Subject: [PATCH 266/268] linux: LinuxUtilities: Revert do_get_path() typing --- volatility3/framework/symbols/linux/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 4634a2bfb..af0697c10 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -6,7 +6,7 @@ import contextlib import functools import logging from abc import ABC, abstractmethod -from typing import Iterator, List, Tuple, Optional +from typing import Iterator, List, Tuple, Optional, Union import volatility3.framework.symbols.linux.utilities.modules as linux_utilities_modules from volatility3 import framework @@ -133,7 +133,7 @@ class LinuxUtilities(interfaces.configuration.VersionableInterface): return cls.do_get_path(rdentry, rmnt, dentry, vfsmnt) @classmethod - def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> str: + def do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> Union[None, str]: """Returns a pathname of the mount point or file It mimics the Linux kernel prepend_path function. From 3bd1c70a9bc3732bf4ab4131c4ee5170cffaa945 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 11:31:06 +1100 Subject: [PATCH 267/268] linux: kmsg plugin: improve return types --- volatility3/framework/plugins/linux/kmsg.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 248e37dd8..a5ce84ae9 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -5,7 +5,7 @@ import re import logging from abc import ABC, abstractmethod from enum import Enum -from typing import Generator, Iterator, List, Tuple +from typing import Generator, Iterator, List, Tuple, Union from volatility3.framework import ( class_subclasses, @@ -135,10 +135,11 @@ class ABCKmsg(ABC): bool: True if the kernel being analyzed fulfill the class requirements. """ - def get_string(self, addr: int, length: int) -> str: + def get_string(self, addr: int, length: int) -> Union[str, None]: layer = self._context.layers[self.layer_name] if not layer.is_valid(addr, length): - return "" + vollog.error("Failed to read log record at address 0x%x", addr) + return None txt = layer.read(addr, length) @@ -268,7 +269,7 @@ class Kmsg_3_5_to_3_11(ABCKmsg): def _get_log_struct_name(self): return "log" - def get_text_from_log(self, msg) -> str: + def get_text_from_log(self, msg) -> Union[str, None]: log_struct_name = self._get_log_struct_name() log_struct_size = self.vmlinux.get_type(log_struct_name).size msg_offset = msg.vol.offset + log_struct_size @@ -277,7 +278,8 @@ class Kmsg_3_5_to_3_11(ABCKmsg): def get_log_lines(self, msg) -> Generator[str, None, None]: if msg.text_len > 0: text = self.get_text_from_log(msg) - yield from text.splitlines() + if text: + yield from text.splitlines() def get_dict_lines(self, msg) -> Generator[str, None, None]: if msg.dict_len == 0: @@ -412,7 +414,7 @@ class Kmsg_5_10_to_(ABCKmsg): def symtab_checks(cls, vmlinux) -> bool: return vmlinux.has_symbol("prb") - def get_text_from_data_ring(self, text_data_ring, desc, info) -> str: + def get_text_from_data_ring(self, text_data_ring, desc, info) -> Union[str, None]: text_data_sz = text_data_ring.size_bits text_data_mask = 1 << text_data_sz @@ -440,7 +442,8 @@ class Kmsg_5_10_to_(ABCKmsg): def get_log_lines(self, text_data_ring, desc, info) -> Generator[str, None, None]: text = self.get_text_from_data_ring(text_data_ring, desc, info) - yield from text.splitlines() + if text: + yield from text.splitlines() def get_dict_lines(self, info) -> Generator[str, None, None]: dict_text = utility.array_to_string(info.dev_info.subsystem) From 4466d9accda1fbb125a34292422b3ad046e5bd2f Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 30 Jan 2025 11:47:01 +1100 Subject: [PATCH 268/268] linux: kmsg plugin: log warning instead of error --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index a5ce84ae9..849060d3c 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -138,7 +138,7 @@ class ABCKmsg(ABC): def get_string(self, addr: int, length: int) -> Union[str, None]: layer = self._context.layers[self.layer_name] if not layer.is_valid(addr, length): - vollog.error("Failed to read log record at address 0x%x", addr) + vollog.warning("Failed to read log record at address 0x%x", addr) return None txt = layer.read(addr, length)