From d5a0b93383fda59267bdd9b42e716b70ad66595c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:28:40 +0100 Subject: [PATCH 001/137] add TAINT_FLAGS constant --- .../framework/constants/linux/__init__.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 7c485d3c3..9f25c9225 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -347,3 +347,88 @@ class PT_FLAGS(Flag): MODULE_MAXIMUM_CORE_SIZE = 20000000 MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 MODULE_MINIMUM_SIZE = 4096 + + +TAINT_FLAGS = { + "P": { + "shift": 1 << 0, + "desc": "PROPRIETARY_MODULE", + "when_present": True, + "module": True, + }, + "G": { + "shift": 1 << 0, + "desc": "PROPRIETARY_MODULE", + "when_present": False, + "module": True, + }, + "F": { + "shift": 1 << 1, + "desc": "FORCED_MODULE", + "when_present": True, + "module": False, + }, + # CPU_OUT_OF_SPEC was TAINT_UNSAFE_SMP on < 3.15-rc1 : https://lore.kernel.org/linux-kernel//20140303080432.GA25489@localhost/t/#:~:text=liked%20your%20proposal%3A-,%3E%20Right,-%2C%20I%20was%20about + "S": { + "shift": 1 << 2, + "desc": "CPU_OUT_OF_SPEC", + "when_present": True, + "module": False, + }, + "R": { + "shift": 1 << 3, + "desc": "FORCED_RMMOD", + "when_present": True, + "module": False, + }, + "M": { + "shift": 1 << 4, + "desc": "MACHINE_CHECK", + "when_present": True, + "module": False, + }, + "B": {"shift": 1 << 5, "desc": "BAD_PAGE", "when_present": True, "module": False}, + "U": {"shift": 1 << 6, "desc": "USER", "when_present": True, "module": False}, + "D": {"shift": 1 << 7, "desc": "DIE", "when_present": True, "module": False}, + "A": { + "shift": 1 << 8, + "desc": "OVERRIDDEN_ACPI_TABLE", + "when_present": True, + "module": False, + }, + "W": {"shift": 1 << 9, "desc": "WARN", "when_present": True, "module": False}, + "C": {"shift": 1 << 10, "desc": "CRAP", "when_present": True, "module": True}, + "I": { + "shift": 1 << 11, + "desc": "FIRMWARE_WORKAROUND", + "when_present": True, + "module": False, + }, + "O": {"shift": 1 << 12, "desc": "OOT_MODULE", "when_present": True, "module": True}, + "E": { + "shift": 1 << 13, + "desc": "UNSIGNED_MODULE", + "when_present": True, + "module": True, + }, + "L": { + "shift": 1 << 14, + "desc": "SOFTLOCKUP", + "when_present": True, + "module": False, + }, + "K": {"shift": 1 << 15, "desc": "LIVEPATCH", "when_present": True, "module": True}, + "X": {"shift": 1 << 16, "desc": "AUX", "when_present": True, "module": True}, + "T": {"shift": 1 << 17, "desc": "RANDSTRUCT", "when_present": True, "module": True}, + "N": {"shift": 1 << 18, "desc": "TEST", "when_present": True, "module": True}, +} +"""Flags used to taint kernel and modules, for debugging purposes. + +Map based on 6.12-rc5. + +Documentation : + - https://www.kernel.org/doc/Documentation/admin-guide/sysctl/kernel.rst#:~:text=guide/sysrq.rst.-,tainted,-%3D%3D%3D%3D%3D%3D%3D%0A%0ANon%2Dzero%20if + - https://www.kernel.org/doc/Documentation/admin-guide/tainted-kernels.rst#:~:text=More%20detailed%20explanation%20for%20tainting + - taint_flag kernel struct + - taint_flags kernel constant +""" From e1b343a284436ee4d92a7b8a6daf0a98dd01fdeb Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:33:54 +0100 Subject: [PATCH 002/137] add module taints parsing apis --- .../symbols/linux/extensions/__init__.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index aa3e8c675..89e2cde27 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -279,6 +279,77 @@ class module(generic.GenericIntelProcess): return None + def _module_flags_taints_pre_4_10_rc1(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Returns: + The raw taints string. + """ + taints_string = "" + for char, infos in linux_constants.TAINT_FLAGS.items(): + if infos["module"] and self.taints_value & infos["shift"]: + taints_string += char + + return taints_string + + def _module_flags_taints_post_4_10_rc1(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self.taint_flags_list): + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taint_flag.module and (self.taints_value & (1 << i)): + taints_string += c_true + elif taint_flag.module and c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self) -> str: + """Convert the module's taints value to a 1-1 character mapping. + + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self.taint_flags_list: + return self._module_flags_taints_post_4_10_rc1() + return self._module_flags_taints_pre_4_10_rc1() + + def get_taints_parsed(self) -> List[str]: + """Convert the module's taints string to a 1-1 descriptor mapping. + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for c in self.get_taints_as_plain_string(): + infos = linux_constants.TAINT_FLAGS.get(c) + if not infos: + comprehensive_taints.append(f"") + elif infos["when_present"]: + comprehensive_taints.append(infos["desc"]) + + return comprehensive_taints + @property def section_symtab(self): if self.has_member("kallsyms"): @@ -307,6 +378,17 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("module -> strtab: Unable to get strtab") + @property + def taints_value(self) -> int: + return self.taints + + @property + def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: + kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + if kernel.has_symbol("taint_flags"): + return list(kernel.object_from_symbol("taint_flags")) + return None + class task_struct(generic.GenericIntelProcess): def add_process_layer( From c88ebe89270355d188770c74b39eb8acef1f3549 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 31 Oct 2024 12:35:54 +0100 Subject: [PATCH 003/137] introduce modxview linux plugin --- .../framework/plugins/linux/modxview.py | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 volatility3/framework/plugins/linux/modxview.py diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py new file mode 100644 index 000000000..66b644164 --- /dev/null +++ b/volatility3/framework/plugins/linux/modxview.py @@ -0,0 +1,195 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from typing import List, Dict, Set, Iterator +from volatility3.plugins.linux import lsmod, check_modules, hidden_modules +from volatility3.framework import interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue +from volatility3.framework.symbols.linux import extensions +from volatility3.framework.constants import architectures + +vollog = logging.getLogger(__name__) + + +class Modxview(interfaces.plugins.PluginInterface): + """Centralize lsmod, check_modules and hidden_modules results to efficiently + spot modules presence and taints.""" + + _version = (1, 0, 0) + _required_framework_version = (2, 11, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=architectures.LINUX_ARCHS, + ), + requirements.PluginRequirement( + name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="check_modules", + plugin=check_modules.Check_modules, + version=(0, 0, 0), + ), + requirements.PluginRequirement( + name="hidden_modules", + plugin=hidden_modules.Hidden_modules, + version=(1, 0, 0), + ), + requirements.BooleanRequirement( + name="plain_taints", + description="Display the plain taints string for each module.", + optional=True, + default=False, + ), + ] + + @classmethod + def run_lsmod( + cls, context: interfaces.context.ContextInterface, kernel_name: str + ) -> List[extensions.module]: + """Wrapper for the lsmod plugin.""" + return list(lsmod.Lsmod.list_modules(context, kernel_name)) + + @classmethod + def run_check_modules( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + ) -> List[extensions.module]: + """Wrapper for the check_modules plugin. + Here, we extract the /sys/module/ list.""" + kernel = context.modules[kernel_name] + sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( + context, kernel_name + ) + + # Convert get_kset_modules() offsets back to module objects + return [ + kernel.object(object_type="module", offset=m_offset, absolute=True) + for m_offset in sysfs_modules.values() + ] + + @classmethod + def run_hidden_modules( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + known_modules_addresses: Set[int], + ) -> List[extensions.module]: + """Wrapper for the hidden_modules plugin.""" + modules_memory_boundaries = ( + hidden_modules.Hidden_modules.get_modules_memory_boundaries( + context, kernel_name + ) + ) + return list( + hidden_modules.Hidden_modules.get_hidden_modules( + context, + kernel_name, + known_modules_addresses, + modules_memory_boundaries, + ) + ) + + @classmethod + def flatten_run_modules_results( + cls, run_results: Dict[str, List[extensions.module]], deduplicate: bool = True + ) -> Iterator[extensions.module]: + """Flatten a dictionary mapping plugin names and modules list, to a single merged list. + This is useful to get a generic lookup list of all the detected modules. + + Args: + run_results: dictionary of plugin names mapping a list of detected modules + deduplicate: remove duplicate modules, based on their offsets + + Returns: + Iterator of modules objects + """ + seen_addresses = set() + for modules in run_results.values(): + for module in modules: + if deduplicate and module.vol.offset in seen_addresses: + continue + yield module + + @classmethod + def run_modules_scanners( + cls, + context: interfaces.context.ContextInterface, + kernel_name: str, + run_hidden_modules: bool = True, + ) -> Dict[str, List[extensions.module]]: + """Run module scanning plugins and aggregate the results. + + Args: + run_hidden_modules: specify if the hidden_modules plugin should be run + Returns: + Dictionary mapping each plugin to its corresponding result + """ + + kernel = context.modules[kernel_name] + run_results = {} + run_results["lsmod"] = cls.run_lsmod(context, kernel_name) + run_results["check_modules"] = cls.run_check_modules(context, kernel_name) + if run_hidden_modules: + known_module_addresses = set( + context.layers[kernel.layer_name].canonicalize(module.vol.offset) + for module in run_results["lsmod"] + run_results["check_modules"] + ) + run_results["hidden_modules"] = cls.run_hidden_modules( + context, kernel_name, known_module_addresses + ) + + return run_results + + def _generator(self): + kernel_name = self.config["kernel"] + run_results = self.run_modules_scanners(self.context, kernel_name) + modules_offsets = {} + for key in ["lsmod", "check_modules", "hidden_modules"]: + modules_offsets[key] = set(module.vol.offset for module in run_results[key]) + + seen_addresses = set() + for modules_list in run_results.values(): + for module in modules_list: + if module.vol.offset in seen_addresses: + continue + seen_addresses.add(module.vol.offset) + + if self.config.get("plain_taints"): + taints = module.get_taints_as_plain_string() + else: + taints = ",".join(module.get_taints_parsed()) + + yield ( + 0, + ( + module.get_name() or NotAvailableValue(), + format_hints.Hex(module.vol.offset), + module.vol.offset in modules_offsets["lsmod"], + module.vol.offset in modules_offsets["check_modules"], + module.vol.offset in modules_offsets["hidden_modules"], + taints or NotAvailableValue(), + ), + ) + + def run(self): + columns = [ + ("Name", str), + ("Address", format_hints.Hex), + ("In /proc/modules", bool), + ("In /sys/module/", bool), + ("Hidden", bool), + ("Taints", str), + ] + + return TreeGrid( + columns, + self._generator(), + ) From 9440f53429a1f9c7d77d51eeb75c2b5938da040f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:05:25 +0100 Subject: [PATCH 004/137] use a dict of dataclasses for taint_flags --- .../framework/constants/linux/__init__.py | 112 +++++++----------- .../symbols/linux/extensions/__init__.py | 12 +- 2 files changed, 47 insertions(+), 77 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 9f25c9225..6cf8585f5 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -6,6 +6,7 @@ Linux-specific values that aren't found in debug symbols """ from enum import IntEnum, Flag +from dataclasses import dataclass KERNEL_NAME = "__kernel__" @@ -349,78 +350,47 @@ MODULE_MAXIMUM_CORE_TEXT_SIZE = 20000000 MODULE_MINIMUM_SIZE = 4096 +@dataclass +class TaintFlag: + shift: int + desc: str + when_present: bool + module: bool + + TAINT_FLAGS = { - "P": { - "shift": 1 << 0, - "desc": "PROPRIETARY_MODULE", - "when_present": True, - "module": True, - }, - "G": { - "shift": 1 << 0, - "desc": "PROPRIETARY_MODULE", - "when_present": False, - "module": True, - }, - "F": { - "shift": 1 << 1, - "desc": "FORCED_MODULE", - "when_present": True, - "module": False, - }, - # CPU_OUT_OF_SPEC was TAINT_UNSAFE_SMP on < 3.15-rc1 : https://lore.kernel.org/linux-kernel//20140303080432.GA25489@localhost/t/#:~:text=liked%20your%20proposal%3A-,%3E%20Right,-%2C%20I%20was%20about - "S": { - "shift": 1 << 2, - "desc": "CPU_OUT_OF_SPEC", - "when_present": True, - "module": False, - }, - "R": { - "shift": 1 << 3, - "desc": "FORCED_RMMOD", - "when_present": True, - "module": False, - }, - "M": { - "shift": 1 << 4, - "desc": "MACHINE_CHECK", - "when_present": True, - "module": False, - }, - "B": {"shift": 1 << 5, "desc": "BAD_PAGE", "when_present": True, "module": False}, - "U": {"shift": 1 << 6, "desc": "USER", "when_present": True, "module": False}, - "D": {"shift": 1 << 7, "desc": "DIE", "when_present": True, "module": False}, - "A": { - "shift": 1 << 8, - "desc": "OVERRIDDEN_ACPI_TABLE", - "when_present": True, - "module": False, - }, - "W": {"shift": 1 << 9, "desc": "WARN", "when_present": True, "module": False}, - "C": {"shift": 1 << 10, "desc": "CRAP", "when_present": True, "module": True}, - "I": { - "shift": 1 << 11, - "desc": "FIRMWARE_WORKAROUND", - "when_present": True, - "module": False, - }, - "O": {"shift": 1 << 12, "desc": "OOT_MODULE", "when_present": True, "module": True}, - "E": { - "shift": 1 << 13, - "desc": "UNSIGNED_MODULE", - "when_present": True, - "module": True, - }, - "L": { - "shift": 1 << 14, - "desc": "SOFTLOCKUP", - "when_present": True, - "module": False, - }, - "K": {"shift": 1 << 15, "desc": "LIVEPATCH", "when_present": True, "module": True}, - "X": {"shift": 1 << 16, "desc": "AUX", "when_present": True, "module": True}, - "T": {"shift": 1 << 17, "desc": "RANDSTRUCT", "when_present": True, "module": True}, - "N": {"shift": 1 << 18, "desc": "TEST", "when_present": True, "module": True}, + "P": TaintFlag( + shift=1 << 0, desc="PROPRIETARY_MODULE", when_present=True, module=True + ), + "G": TaintFlag( + shift=1 << 0, desc="PROPRIETARY_MODULE", when_present=False, module=True + ), + "F": TaintFlag(shift=1 << 1, desc="FORCED_MODULE", when_present=True, module=False), + "S": TaintFlag( + shift=1 << 2, desc="CPU_OUT_OF_SPEC", when_present=True, module=False + ), + "R": TaintFlag(shift=1 << 3, desc="FORCED_RMMOD", when_present=True, module=False), + "M": TaintFlag(shift=1 << 4, desc="MACHINE_CHECK", when_present=True, module=False), + "B": TaintFlag(shift=1 << 5, desc="BAD_PAGE", when_present=True, module=False), + "U": TaintFlag(shift=1 << 6, desc="USER", when_present=True, module=False), + "D": TaintFlag(shift=1 << 7, desc="DIE", when_present=True, module=False), + "A": TaintFlag( + shift=1 << 8, desc="OVERRIDDEN_ACPI_TABLE", when_present=True, module=False + ), + "W": TaintFlag(shift=1 << 9, desc="WARN", when_present=True, module=False), + "C": TaintFlag(shift=1 << 10, desc="CRAP", when_present=True, module=True), + "I": TaintFlag( + shift=1 << 11, desc="FIRMWARE_WORKAROUND", when_present=True, module=False + ), + "O": TaintFlag(shift=1 << 12, desc="OOT_MODULE", when_present=True, module=True), + "E": TaintFlag( + shift=1 << 13, desc="UNSIGNED_MODULE", when_present=True, module=True + ), + "L": TaintFlag(shift=1 << 14, desc="SOFTLOCKUP", when_present=True, module=False), + "K": TaintFlag(shift=1 << 15, desc="LIVEPATCH", when_present=True, module=True), + "X": TaintFlag(shift=1 << 16, desc="AUX", when_present=True, module=True), + "T": TaintFlag(shift=1 << 17, desc="RANDSTRUCT", when_present=True, module=True), + "N": TaintFlag(shift=1 << 18, desc="TEST", when_present=True, module=True), } """Flags used to taint kernel and modules, for debugging purposes. diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 89e2cde27..42c1a470d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -287,8 +287,8 @@ class module(generic.GenericIntelProcess): The raw taints string. """ taints_string = "" - for char, infos in linux_constants.TAINT_FLAGS.items(): - if infos["module"] and self.taints_value & infos["shift"]: + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if taint_flag.module and self.taints_value & taint_flag.shift: taints_string += char return taints_string @@ -342,11 +342,11 @@ class module(generic.GenericIntelProcess): """ comprehensive_taints = [] for c in self.get_taints_as_plain_string(): - infos = linux_constants.TAINT_FLAGS.get(c) - if not infos: + taint_flag = linux_constants.TAINT_FLAGS.get(c) + if not taint_flag: comprehensive_taints.append(f"") - elif infos["when_present"]: - comprehensive_taints.append(infos["desc"]) + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) return comprehensive_taints From 9d08c4681ae1cf18ddf4bd53ff970f6a9bc26573 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:07:23 +0100 Subject: [PATCH 005/137] add module offset to seen_addresses --- volatility3/framework/plugins/linux/modxview.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 66b644164..f44984926 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -116,6 +116,7 @@ class Modxview(interfaces.plugins.PluginInterface): for module in modules: if deduplicate and module.vol.offset in seen_addresses: continue + seen_addresses.add(module.vol.offset) yield module @classmethod From b209ea36a284ae1a75ba18222cbe8db1c1eede4f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 1 Nov 2024 15:12:52 +0100 Subject: [PATCH 006/137] remove slashes in columns --- volatility3/framework/plugins/linux/modxview.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index f44984926..d79f5e7a9 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -184,8 +184,8 @@ class Modxview(interfaces.plugins.PluginInterface): columns = [ ("Name", str), ("Address", format_hints.Hex), - ("In /proc/modules", bool), - ("In /sys/module/", bool), + ("In procfs", bool), + ("In sysfs", bool), ("Hidden", bool), ("Taints", str), ] From c7259037356fde4cf6120acddecbae4af16c9ea0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 7 Nov 2024 18:08:58 +0100 Subject: [PATCH 007/137] introduce scatter-gather scatterlists --- .../framework/symbols/linux/__init__.py | 1 + .../symbols/linux/extensions/__init__.py | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3289775b6..e8a3d5240 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -43,6 +43,7 @@ class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): self.optional_set_type_class("bpf_prog_aux", extensions.bpf_prog_aux) self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) + self.optional_set_type_class("scatterlist", extensions.scatterlist) # kernels >= 4.18 self.optional_set_type_class("timespec64", extensions.timespec64) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index aa3e8c675..0dd657372 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2410,3 +2410,107 @@ class rb_root(objects.StructType): """ yield from self._walk_nodes(root_node=self.rb_node) + + +class scatterlist(objects.StructType): + SG_CHAIN = 0x01 + SG_END = 0x02 + SG_PAGE_LINK_MASK = SG_CHAIN | SG_END + + def _sg_flags(self) -> int: + return self.page_link & self.SG_PAGE_LINK_MASK + + def _sg_is_chain(self) -> int: + return self._sg_flags() & self.SG_CHAIN + + def _sg_is_last(self) -> int: + return self._sg_flags() & self.SG_END + + def _sg_chain_ptr(self) -> int: + """Clears the last two bits basically.""" + return self.page_link & ~self.SG_PAGE_LINK_MASK + + def _sg_dma_len(self) -> int: + # Depends on CONFIG_NEED_SG_DMA_LENGTH + if self.has_member("dma_length"): + return self.dma_length + return self.length + + def _get_sg_max_single_alloc(self) -> int: + """Based on kernel's SG_MAX_SINGLE_ALLOC. + + Doc. from kernel source : + * Maximum number of entries that will be allocated in one piece, if + * a list larger than this is required then chaining will be utilized. + """ + return self._context.layers[self.vol.layer_name].page_size // self.vol.size + + def _sg_next(self) -> interfaces.objects.ObjectInterface: + """Get the next scatterlist struct from the list. + Based on kernel's sg_next. + + Doc. from kernel source : + * Notes on SG table design. + * + * We use the unsigned long page_link field in the scatterlist struct to place + * the page pointer AND encode information about the sg table as well. The two + * lower bits are reserved for this information. + * + * If bit 0 is set, then the page_link contains a pointer to the next sg + * table list. Otherwise the next entry is at sg + 1. + * + * If bit 1 is set, then this sg entry is the last element in a list. + """ + if self._sg_is_last(): + return None + + if self._sg_is_chain(): + next_address = self._sg_chain_ptr() + else: + next_address = self.vol.offset + self.vol.size + + sg = self._context.object( + self.get_symbol_table_name() + constants.BANG + "scatterlist", + self.vol.layer_name, + next_address, + ) + return sg + + def for_each_sg(self) -> Iterator[interfaces.objects.ObjectInterface]: + """Iterate over each struct in the scatterlist.""" + sg = self + sg_max_single_alloc = self._get_sg_max_single_alloc() + + # Empty scatterlists protection + if sg.page_link == 0 and sg._sg_dma_len() == 0 and sg.dma_address == 0: + return None + else: + # Yield itself first + yield sg + + entries_count = 1 + # entries_count <= sg_max_single_alloc should always be true if the + # scatterlists were correctly chained. + while entries_count <= sg_max_single_alloc: + sg = sg._sg_next() + if sg is None: + break + # Points to a new scatterlist + elif sg._sg_is_chain(): + entries_count = 0 + else: + entries_count += 1 + yield sg + + def get_content( + self, + ) -> Iterator[bytes]: + """Traverse a scatterlist to gather content located at each + dma_address position. + + Returns: + An iterator of bytes + """ + physical_layer = self._context.layers["memory_layer"] + for sg in self.for_each_sg(): + yield from physical_layer.read(sg.dma_address, sg._sg_dma_len()) From 8f33aaf5b4ee859b12ca35347144597bec59ee1b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Nov 2024 17:45:53 +0100 Subject: [PATCH 008/137] Optional type hints --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0dd657372..e5a074a49 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2445,7 +2445,7 @@ class scatterlist(objects.StructType): """ return self._context.layers[self.vol.layer_name].page_size // self.vol.size - def _sg_next(self) -> interfaces.objects.ObjectInterface: + def _sg_next(self) -> Optional[interfaces.objects.ObjectInterface]: """Get the next scatterlist struct from the list. Based on kernel's sg_next. @@ -2476,7 +2476,7 @@ class scatterlist(objects.StructType): ) return sg - def for_each_sg(self) -> Iterator[interfaces.objects.ObjectInterface]: + def for_each_sg(self) -> Optional[Iterator[interfaces.objects.ObjectInterface]]: """Iterate over each struct in the scatterlist.""" sg = self sg_max_single_alloc = self._get_sg_max_single_alloc() @@ -2504,7 +2504,7 @@ class scatterlist(objects.StructType): def get_content( self, - ) -> Iterator[bytes]: + ) -> Optional[Iterator[bytes]]: """Traverse a scatterlist to gather content located at each dma_address position. From 485ef894e113cf68eb1acaef3c679b675639281d Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Nov 2024 17:49:08 +0100 Subject: [PATCH 009/137] remove taints_value overload attr --- .../framework/symbols/linux/extensions/__init__.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 42c1a470d..f9f72c161 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -288,7 +288,7 @@ class module(generic.GenericIntelProcess): """ taints_string = "" for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if taint_flag.module and self.taints_value & taint_flag.shift: + if taint_flag.module and self.taints & taint_flag.shift: taints_string += char return taints_string @@ -310,7 +310,7 @@ class module(generic.GenericIntelProcess): for i, taint_flag in enumerate(self.taint_flags_list): c_true = chr(taint_flag.c_true) c_false = chr(taint_flag.c_false) - if taint_flag.module and (self.taints_value & (1 << i)): + if taint_flag.module and (self.taints & (1 << i)): taints_string += c_true elif taint_flag.module and c_false != " ": taints_string += c_false @@ -378,10 +378,6 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("module -> strtab: Unable to get strtab") - @property - def taints_value(self) -> int: - return self.taints - @property def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) From dd3542b127b751ad083c91be4e8ffd373a1c74f7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Nov 2024 17:51:48 +0100 Subject: [PATCH 010/137] explicit loop iterator --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index f9f72c161..402f8c9c6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -341,10 +341,10 @@ class module(generic.GenericIntelProcess): - module_flags_taint kernel function """ comprehensive_taints = [] - for c in self.get_taints_as_plain_string(): - taint_flag = linux_constants.TAINT_FLAGS.get(c) + for character in self.get_taints_as_plain_string(): + taint_flag = linux_constants.TAINT_FLAGS.get(character) if not taint_flag: - comprehensive_taints.append(f"") + comprehensive_taints.append(f"") elif taint_flag.when_present: comprehensive_taints.append(taint_flag.desc) From 20f15d3591a5e6340ecd2d3628406c01aef71924 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 2 Dec 2024 11:34:49 +0100 Subject: [PATCH 011/137] modular physical_layer access --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index e5a074a49..2d77b8562 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2511,6 +2511,10 @@ class scatterlist(objects.StructType): Returns: An iterator of bytes """ - physical_layer = self._context.layers["memory_layer"] + # Either "physical" is layer-1 because this is a module layer, either "physical" is the current layer + physical_layer_name = self._context.layers[self.vol.layer_name].config.get( + "memory_layer", self.vol.layer_name + ) + physical_layer = self._context.layers[physical_layer_name] for sg in self.for_each_sg(): yield from physical_layer.read(sg.dma_address, sg._sg_dma_len()) From c45beb3ebe7feaec42567eb8ef3dd665e15db3ae Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 17 Dec 2024 18:12:19 +0000 Subject: [PATCH 012/137] Automagic: Fixes #1417 --- volatility3/framework/automagic/symbol_cache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index e38771f79..2c9883c7d 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -299,6 +299,13 @@ class SqliteCache(CacheManagerInterface): This also updates remote locations based on a cache timeout. """ + if progress_callback is None: + + def dummy_progress(*args, **kargs) -> None: + return None + + progress_callback = dummy_progress + on_disk_locations = set( [ filename From f3d7647433a727a5bb7bc8c91fa3803ad44a6bf4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 22 Dec 2024 15:48:29 +0100 Subject: [PATCH 013/137] unify Tainting parsing capabilities --- .../framework/symbols/linux/__init__.py | 121 ++++++++++++++++++ .../symbols/linux/extensions/__init__.py | 74 ++--------- 2 files changed, 131 insertions(+), 64 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0230a9c48..832b1de9b 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -11,6 +11,7 @@ from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions +from volatility3.framework.constants import linux as linux_constants class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -830,3 +831,123 @@ class PageCache: page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page + + +class Tainting: + """Tainted kernel and modules parsing capabilities. + + Relevant kernel functions: + - modules: module_flags_taint + - kernel: print_tainted + """ + + def __init__( + self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ): + self.kernel = context.modules[kernel_module_name] + + @property + def kernel_taint_flags_list( + self, + ) -> Optional[List[interfaces.objects.ObjectInterface]]: + if self.kernel.has_symbol("taint_flags"): + return list(self.kernel.object_from_symbol("taint_flags")) + return None + + def _module_flags_taint_pre_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if is_module and is_module != taint_flag.module: + continue + + if taints & taint_flag.shift: + taints_string += char + + return taints_string + + def _module_flags_taint_post_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self.kernel_taint_flags_list): + if is_module and is_module != taint_flag.module: + continue + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taints & (1 << i): + taints_string += c_true + elif c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + """Convert the taints value to a 1-1 character mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + s + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self.kernel_taint_flags_list: + return self._module_flags_taint_post_4_10_rc1(taints, is_module) + return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + + def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + """Convert the taints string to a 1-1 descriptor mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for character in self.get_taints_as_plain_string(taints, is_module): + taint_flag = linux_constants.TAINT_FLAGS.get(character) + if not taint_flag: + comprehensive_taints.append(f"") + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) + + return comprehensive_taints diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 0ecf731f4..075a83ae8 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -279,76 +279,29 @@ class module(generic.GenericIntelProcess): return None - def _module_flags_taints_pre_4_10_rc1(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on statically defined taints mappings in the framework. - - Returns: - The raw taints string. - """ - taints_string = "" - for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if taint_flag.module and self.taints & taint_flag.shift: - taints_string += char - - return taints_string - - def _module_flags_taints_post_4_10_rc1(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on kernel symbol embedded taints definitions. - - struct taint_flag { - char c_true; /* character printed when tainted */ - char c_false; /* character printed when not tainted */ - bool module; /* also show as a per-module taint flag */ - }; - - Returns: - The raw taints string. - """ - taints_string = "" - for i, taint_flag in enumerate(self.taint_flags_list): - c_true = chr(taint_flag.c_true) - c_false = chr(taint_flag.c_false) - if taint_flag.module and (self.taints & (1 << i)): - taints_string += c_true - elif taint_flag.module and c_false != " ": - taints_string += c_false - - return taints_string - def get_taints_as_plain_string(self) -> str: """Convert the module's taints value to a 1-1 character mapping. + Convenient wrapper around framework's Tainting capabilities. Returns: The raw taints string. - - Documentation: - - module_flags_taint kernel function """ - - if self.taint_flags_list: - return self._module_flags_taints_post_4_10_rc1() - return self._module_flags_taints_pre_4_10_rc1() + return linux.Tainting( + self._context, + linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, + ).get_taints_as_plain_string(self.taints, True) def get_taints_parsed(self) -> List[str]: """Convert the module's taints string to a 1-1 descriptor mapping. + Convenient wrapper around framework's Tainting capabilities. Returns: A comprehensive (user-friendly) taint descriptor list. - - Documentation: - - module_flags_taint kernel function """ - comprehensive_taints = [] - for character in self.get_taints_as_plain_string(): - taint_flag = linux_constants.TAINT_FLAGS.get(character) - if not taint_flag: - comprehensive_taints.append(f"") - elif taint_flag.when_present: - comprehensive_taints.append(taint_flag.desc) - - return comprehensive_taints + return linux.Tainting( + self._context, + linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, + ).get_taints_parsed(self.taints, True) @property def section_symtab(self): @@ -376,13 +329,6 @@ class module(generic.GenericIntelProcess): return self.strtab raise AttributeError("Unable to get strtab") - @property - def taint_flags_list(self) -> Optional[List[interfaces.objects.ObjectInterface]]: - kernel = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) - if kernel.has_symbol("taint_flags"): - return list(kernel.object_from_symbol("taint_flags")) - return None - class task_struct(generic.GenericIntelProcess): def add_process_layer( From e9d9345cef488067e7035aa485ff11ed665c4414 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 09:37:40 -0600 Subject: [PATCH 014/137] Windows Cachedump: Handle uncaught InvalidAddressException --- volatility3/framework/plugins/windows/cachedump.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index 6e667984a..6c730e6ae 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -8,7 +8,7 @@ from typing import Tuple from Crypto.Cipher import ARC4, AES from Crypto.Hash import HMAC -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions @@ -140,9 +140,14 @@ class Cachedump(interfaces.plugins.PluginInterface): if cache_item.Name == "NL$Control": continue - data = sechive.read(cache_item.Data + 4, cache_item.DataLength) - if data is None: + try: + data = sechive.read(cache_item.Data + 4, cache_item.DataLength) + except exceptions.InvalidAddressException: continue + + if not data: + continue + ( uname_len, domain_len, From 3eeb10be2916bb7988d296c7a85785ffb5a7f25e Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 10:10:03 -0600 Subject: [PATCH 015/137] Windows Registry: Handle uncaught exceptions A number of calls to `get_key` across multiple plugins are not made within a `try/except` block that handles `registry.RegistryFormatException` - the calls are either unprotected or only check for `KeyError`. This adds the required `try/except` blocks, or updates the existing ones as needed. --- volatility3/framework/plugins/windows/amcache.py | 10 +++++----- volatility3/framework/plugins/windows/hashdump.py | 2 +- volatility3/framework/plugins/windows/lsadump.py | 7 +++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/windows/amcache.py b/volatility3/framework/plugins/windows/amcache.py index 1e918d61c..46a742233 100644 --- a/volatility3/framework/plugins/windows/amcache.py +++ b/volatility3/framework/plugins/windows/amcache.py @@ -543,7 +543,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\InventoryDriverBinary") # type: ignore ) ) - except KeyError: + except (KeyError, registry.RegistryFormatException): # Registry key not found pass @@ -554,7 +554,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\Programs") ) # type: ignore } - except KeyError: + except (KeyError, registry.RegistryFormatException): programs = {} try: @@ -564,7 +564,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), key=_entry_sort_key, ) - except KeyError: + except (KeyError, registry.RegistryFormatException): files = [] for program_id, file_entries in itertools.groupby( @@ -593,7 +593,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): amcache.get_key("Root\\InventoryApplication") # type: ignore ) ) - except KeyError: + except (KeyError, registry.RegistryFormatException): programs = {} try: @@ -603,7 +603,7 @@ class Amcache(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): ), key=_entry_sort_key, ) - except KeyError: + except (KeyError, registry.RegistryFormatException): files = [] for program_id, file_entries in itertools.groupby( diff --git a/volatility3/framework/plugins/windows/hashdump.py b/volatility3/framework/plugins/windows/hashdump.py index 0c98ab8ca..621b0ae53 100644 --- a/volatility3/framework/plugins/windows/hashdump.py +++ b/volatility3/framework/plugins/windows/hashdump.py @@ -332,7 +332,7 @@ class Hashdump(interfaces.plugins.PluginInterface): try: if hive: result = hive.get_key(key) - except KeyError: + except (KeyError, registry.RegistryFormatException): vollog.info( f"Unable to load the required registry key {hive.get_name()}\\{key} from this memory image" ) diff --git a/volatility3/framework/plugins/windows/lsadump.py b/volatility3/framework/plugins/windows/lsadump.py index da8dee325..f3925f2a2 100644 --- a/volatility3/framework/plugins/windows/lsadump.py +++ b/volatility3/framework/plugins/windows/lsadump.py @@ -8,7 +8,7 @@ from typing import Optional from Crypto.Cipher import ARC4, DES, AES from Crypto.Hash import MD5, SHA256 -from volatility3.framework import interfaces, renderers +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions @@ -81,7 +81,10 @@ class Lsadump(interfaces.plugins.PluginInterface): if not enc_reg_value: return None - obf_lsa_key = sechive.read(enc_reg_value.Data + 4, enc_reg_value.DataLength) + try: + obf_lsa_key = sechive.read(enc_reg_value.Data + 4, enc_reg_value.DataLength) + except exceptions.InvalidAddressException: + return None if not obf_lsa_key: return None From f3294ef5f12a6b036989585b88105fde62634ce2 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 30 Dec 2024 12:58:44 -0600 Subject: [PATCH 016/137] Windows Registry: Handle possible exception in get_node Encountered a `SwappedInvalidAddressException` within the call to `cast` due to an underlying call to `read`. --- volatility3/framework/layers/registry.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index cc364ad50..ee7286e1e 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -140,7 +140,14 @@ class RegistryHive(linear.LinearlyMappedLayer): """Returns the appropriate Node, interpreted from the Cell based on its Signature.""" cell = self.get_cell(cell_offset) - signature = cell.cast("string", max_length=2, encoding="latin-1") + try: + signature = cell.cast("string", max_length=2, encoding="latin-1") + except exceptions.InvalidAddressException: + vollog.debug( + f"Unable to read cell signature for cell at {cell.vol.offset:x}" + ) + return cell + if signature == "nk": return cell.u.KeyNode elif signature == "sk": From 263c87611b51f1cb9710c2ec71d1f41eb98e9c77 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 31 Dec 2024 10:43:27 -0600 Subject: [PATCH 017/137] Windows Registry: Handle exceptions in read calls These calls to `.read()` can raise an `InvalidAddressException`. Instead of propagating this exception to the caller, this adds debug logging, and pads the data will null bytes. Also updates the docstring for `decode_data()` to indicate that it can raise `TypeError` and `ValueError`. --- .../symbols/windows/extensions/registry.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 9e2f8df3b..97dd7390d 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -276,7 +276,16 @@ class CM_KEY_VALUE(objects.StructType): return RegValueTypes(self.Type) def decode_data(self) -> Union[int, bytes]: - """Properly decodes the data associated with the value node""" + """ + Properly decodes the data associated with the value node. + + If an InvalidAddressException occurs when reading data from the + underlying RegistryHive layer, the data will be padded with null bytes + of the same length. + + Raises ValueError if the data cannot be read + Raises TypeError if the class was not instantiated on a RegistryHive layer + """ # Determine if the data is stored inline datalen = self.DataLength data = b"" @@ -310,14 +319,26 @@ class CM_KEY_VALUE(objects.StructType): and block_offset < layer.maximum_address ): amount = min(BIG_DATA_MAXLEN, datalen) - data += layer.read( - offset=layer.get_cell(block_offset).vol.offset, length=amount - ) + try: + data += layer.read( + offset=layer.get_cell(block_offset).vol.offset, + length=amount, + ) + except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {amount:x} bytes of data, padding with {amount:x}" + ) datalen -= amount else: # Suspect Data actually points to a Cell, # but the length at the start could be negative so just adding 4 to jump past it - data = layer.read(self.Data + 4, datalen) + try: + data = layer.read(self.Data + 4, datalen) + except exceptions.InvalidAddressException: + vollog.debug( + f"Failed to read {datalen:x} bytes of data, returning {datalen:x} null bytes" + ) + data = b"\x00" * datalen if self.get_type() == RegValueTypes.REG_DWORD: if len(data) != struct.calcsize(" Date: Tue, 31 Dec 2024 11:11:57 -0600 Subject: [PATCH 018/137] Windows Registry: Update docstrings + exceptions This updates the docstrings on several methods to indicate that they may raise an exception. --- .../symbols/windows/extensions/registry.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 97dd7390d..e53338855 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -159,6 +159,11 @@ class CM_KEY_NODE(objects.StructType): """Extension to allow traversal of registry keys.""" def get_volatile(self) -> bool: + """ + Returns a bool indicating whether or not the key is volatile. + + Raises ValueError if the key was not instantiated on a RegistryHive layer + """ if not isinstance(self._context.layers[self.vol.layer_name], RegistryHive): raise ValueError( "Cannot determine volatility of registry key without an offset in a RegistryHive layer" @@ -166,7 +171,10 @@ class CM_KEY_NODE(objects.StructType): return bool(self.vol.offset & 0x80000000) def get_subkeys(self) -> Iterator["CM_KEY_NODE"]: - """Returns a list of the key nodes.""" + """Returns a list of the key nodes. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") @@ -222,7 +230,10 @@ class CM_KEY_NODE(objects.StructType): yield from self._get_subkeys_recursive(hive, subnode) def get_values(self) -> Iterator["CM_KEY_VALUE"]: - """Returns a list of the Value nodes for a key.""" + """Returns a list of the Value nodes for a key. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ hive = self._context.layers[self.vol.layer_name] if not isinstance(hive, RegistryHive): raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") @@ -251,6 +262,11 @@ class CM_KEY_NODE(objects.StructType): return self.Name.cast("string", max_length=namelength, encoding="latin-1") def get_key_path(self) -> str: + """ + Returns the full path to this registry key. + + Raises TypeError if the key was not instantiated on a RegistryHive layer + """ reg = self._context.layers[self.vol.layer_name] if not isinstance(reg, RegistryHive): raise TypeError("Key was not instantiated on a RegistryHive layer") From 9c02f0d12a13fb77db8fb326f6f68dc31fceec1f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 15:22:09 +0000 Subject: [PATCH 019/137] Linux: Fix kmsf f-strings Closes #1496 --- volatility3/framework/plugins/linux/kmsg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d66e3b9ca..c1d09aff8 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return f"{nsec / 1000000000:lu}.{(nsec % 1000000000) / 1000:06lu}" + return f"{nsec / 1000000000}.{(nsec % 1000000000) / 1000:06}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = f"{caller_name}({caller_id & ~0x80000000:u})" + caller = f"{caller_name}({int(caller_id & ~0x80000000)})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: From ac3e76665b7a44b6c5dbc18e633814bd2371ff75 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 1 Jan 2025 15:29:07 +0000 Subject: [PATCH 020/137] Linux: Fix kmsg unguarded read of msg.len --- volatility3/framework/plugins/linux/kmsg.py | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index d66e3b9ca..67114c087 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -317,23 +317,27 @@ class Kmsg_3_5_to_3_11(ABCKmsg): while cur_idx < end_idx: msg_offset = log_buf_ptr + cur_idx # type: ignore msg = self.vmlinux.object(object_type=log_struct_name, offset=msg_offset) - if msg.len == 0: - # As per kernel/printk.c: - # A length == 0 for the next message indicates a wrap-around to - # the beginning of the buffer. - cur_idx = 0 - end_idx = log_next_idx - else: - facility, level, timestamp, caller = self.get_prefix(msg) - level_txt = self.get_level_text(level) - facility_txt = self.get_facility_text(facility) + try: + if msg.len == 0: + # As per kernel/printk.c: + # A length == 0 for the next message indicates a wrap-around to + # the beginning of the buffer. + cur_idx = 0 + end_idx = log_next_idx + else: + facility, level, timestamp, caller = self.get_prefix(msg) + level_txt = self.get_level_text(level) + facility_txt = self.get_facility_text(facility) - for line in self.get_log_lines(msg): - yield facility_txt, level_txt, timestamp, caller, line - for line in self.get_dict_lines(msg): - yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_log_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_dict_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line - cur_idx += msg.len + cur_idx += msg.len + except exceptions.InvalidAddressException: + vollog.warning("Kmsg buffer msg length could not be read") + return class Kmsg_3_11_to_5_10(Kmsg_3_5_to_3_11): From c8e67e526a831dcd05b59fa0adeeda8937c4f81a Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 1 Jan 2025 22:33:25 -0600 Subject: [PATCH 021/137] Convert ValueError to TypeError All other methods in this class raise a `TypeError` if the hive was not instantiated on a registry layer; this changes makes this method consistent with the convention used in the others. All `except` blocks checking for `ValueError` have been audited to ensure that this doesn't break exception handling in existing code within the framework. This also includes a minor version bump because: 1. RegistryHives are currently only instantiated one way, which is through the `hivelist` plugin. `hivelist` uses the correct layers when instantiating the hives. 2. Because there is currently a single source for registry hives, and it's unlikely that a hive from that source will ever be created on the wrong layer, it's unlikely that the existing `ValueError` is being raised anywhere within the framework's code. 3. It seems unlikely that consumers of this framework would be instantiating registry hives independent of the `hivelist` plugin, given that they would effectively have to duplicate the `hivelist` code to do so. For these reasons, we're going to do a minor version bump, even though an argument can be made that this warrants a major version bump according to the SemVer rules. This is a one-off and does not indicate any change in the way that we typically update version numbers. --- volatility3/framework/constants/_version.py | 2 +- .../framework/symbols/windows/extensions/registry.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 9ca2d0a5b..2f0c53093 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 14 # Number of changes that only add to the interface +VERSION_MINOR = 15 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index e53338855..c9544a8ba 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -162,12 +162,10 @@ class CM_KEY_NODE(objects.StructType): """ Returns a bool indicating whether or not the key is volatile. - Raises ValueError if the key was not instantiated on a RegistryHive layer + Raises TypeError if the key was not instantiated on a RegistryHive layer """ if not isinstance(self._context.layers[self.vol.layer_name], RegistryHive): - raise ValueError( - "Cannot determine volatility of registry key without an offset in a RegistryHive layer" - ) + raise TypeError("CM_KEY_NODE was not instantiated on a RegistryHive layer") return bool(self.vol.offset & 0x80000000) def get_subkeys(self) -> Iterator["CM_KEY_NODE"]: From 97b93abe438bf32b32067bd962a19e07d9917406 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 2 Jan 2025 11:26:21 +0000 Subject: [PATCH 022/137] Linux: Remove unnecessary int cast --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index c1d09aff8..894ca575f 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -166,7 +166,7 @@ class ABCKmsg(ABC): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = f"{caller_name}({int(caller_id & ~0x80000000)})" + caller = f"{caller_name}({caller_id & ~0x80000000})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: From 7278bb244f58f36b346d254512e393cde6f63871 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 14:47:18 +0100 Subject: [PATCH 023/137] move get_flags_list at bottom --- .../framework/symbols/linux/extensions/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 34d0fcba9..9546fcf82 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2628,6 +2628,19 @@ class page(objects.StructType): page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) return page_data + def get_flags_list(self) -> List[str]: + """Returns a list of page flags + + Returns: + List of page flags + """ + flags = [] + for name, value in self.pageflags_enum.items(): + if self.flags & (1 << value) != 0: + flags.append(name) + + return flags + class IDR(objects.StructType): IDR_BITS = 8 From b61ba66223a866a29a119eb17f44fba250ecc01e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 14:51:23 +0100 Subject: [PATCH 024/137] multi-architecture vmemmap_start calculation --- .../symbols/linux/extensions/__init__.py | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 9546fcf82..b02f80433 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -15,7 +15,7 @@ from typing import Generator, Iterable, Iterator, Optional, Tuple, List, Union, from volatility3.framework import constants, exceptions, objects, interfaces, symbols from volatility3.framework.renderers import conversion from volatility3.framework.constants import linux as linux_constants -from volatility3.framework.layers import linear +from volatility3.framework.layers import linear, intel from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf @@ -2525,16 +2525,13 @@ class address_space(objects.StructType): class page(objects.StructType): - @property - @functools.lru_cache + @functools.cached_property def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values Returns: A dictionary with the pageflags enumeration key/values """ - # FIXME: It would be even better to use @functools.cached_property instead, - # however, this requires Python +3.8 try: pageflags_enum = self._context.symbol_space.get_enumeration( self.get_symbol_table_name() + constants.BANG + "pageflags" @@ -2548,24 +2545,12 @@ class page(objects.StructType): return pageflags_enum - def get_flags_list(self) -> List[str]: - """Returns a list of page flags + @functools.cached_property + def _intel_vmemmap_start(self) -> int: + """Determine the start of the struct page array, for Intel systems. Returns: - List of page flags - """ - flags = [] - for name, value in self.pageflags_enum.items(): - if self.flags & (1 << value) != 0: - flags.append(name) - - return flags - - def to_paddr(self) -> int: - """Converts a page's virtual address to its physical address using the current physical memory model. - - Returns: - int: page physical address + int: vmemmap_start address """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] @@ -2605,13 +2590,39 @@ class page(objects.StructType): "Something went wrong, we shouldn't be here" ) - page_type_size = vmlinux.get_type("page").size + return vmemmap_start + + def _intel_to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current Intel memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] pagec = vmlinux_layer.canonicalize(self.vol.offset) - pfn = (pagec - vmemmap_start) // page_type_size + pfn = (pagec - self._intel_vmemmap_start) // vmlinux.get_type("page").size page_paddr = pfn * vmlinux_layer.page_size return page_paddr + def to_paddr(self) -> int: + """Converts a page's virtual address to its physical address using the current CPU memory model. + + Returns: + int: page physical address + """ + vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) + vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] + if isinstance(vmlinux_layer, intel.Intel): + page_paddr = self._intel_to_paddr() + else: + raise exceptions.LayerException( + f"Architecture {type(vmlinux_layer)} vmemmap_start calculation isn't currently supported." + ) + + return page_paddr + def get_content(self) -> Union[str, None]: """Returns the page content @@ -2620,7 +2631,10 @@ class page(objects.StructType): """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) vmlinux_layer = vmlinux.context.layers[vmlinux.layer_name] - physical_layer = vmlinux.context.layers["memory_layer"] + physical_layer_name = self._context.layers[self.vol.layer_name].config.get( + "memory_layer", self.vol.layer_name + ) + physical_layer = self._context.layers[physical_layer_name] page_paddr = self.to_paddr() if not page_paddr: return None From dda104bd62b9f5f7b9c0208832c6d788c0ebd2ea Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:37:14 +0100 Subject: [PATCH 025/137] move out Tainting capabilities --- .../framework/symbols/linux/__init__.py | 121 ------------------ 1 file changed, 121 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 832b1de9b..0230a9c48 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -11,7 +11,6 @@ from volatility3.framework import constants, exceptions, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions -from volatility3.framework.constants import linux as linux_constants class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -831,123 +830,3 @@ class PageCache: page = self.vmlinux.object("page", offset=page_addr, absolute=True) if page: yield page - - -class Tainting: - """Tainted kernel and modules parsing capabilities. - - Relevant kernel functions: - - modules: module_flags_taint - - kernel: print_tainted - """ - - def __init__( - self, - context: interfaces.context.ContextInterface, - kernel_module_name: str, - ): - self.kernel = context.modules[kernel_module_name] - - @property - def kernel_taint_flags_list( - self, - ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self.kernel.has_symbol("taint_flags"): - return list(self.kernel.object_from_symbol("taint_flags")) - return None - - def _module_flags_taint_pre_4_10_rc1( - self, taints: int, is_module: bool = False - ) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on statically defined taints mappings in the framework. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - The raw taints string. - """ - taints_string = "" - for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if is_module and is_module != taint_flag.module: - continue - - if taints & taint_flag.shift: - taints_string += char - - return taints_string - - def _module_flags_taint_post_4_10_rc1( - self, taints: int, is_module: bool = False - ) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Relies on kernel symbol embedded taints definitions. - - struct taint_flag { - char c_true; /* character printed when tainted */ - char c_false; /* character printed when not tainted */ - bool module; /* also show as a per-module taint flag */ - }; - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - The raw taints string. - """ - taints_string = "" - for i, taint_flag in enumerate(self.kernel_taint_flags_list): - if is_module and is_module != taint_flag.module: - continue - c_true = chr(taint_flag.c_true) - c_false = chr(taint_flag.c_false) - if taints & (1 << i): - taints_string += c_true - elif c_false != " ": - taints_string += c_false - - return taints_string - - def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: - """Convert the taints value to a 1-1 character mapping. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - s - Returns: - The raw taints string. - - Documentation: - - module_flags_taint kernel function - """ - - if self.kernel_taint_flags_list: - return self._module_flags_taint_post_4_10_rc1(taints, is_module) - return self._module_flags_taint_pre_4_10_rc1(taints, is_module) - - def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: - """Convert the taints string to a 1-1 descriptor mapping. - - Args: - taints: The taints value, represented by an integer - is_module: Indicates if the taints value is associated with a built-in/LKM module - - Returns: - A comprehensive (user-friendly) taint descriptor list. - - Documentation: - - module_flags_taint kernel function - """ - comprehensive_taints = [] - for character in self.get_taints_as_plain_string(taints, is_module): - taint_flag = linux_constants.TAINT_FLAGS.get(character) - if not taint_flag: - comprehensive_taints.append(f"") - elif taint_flag.when_present: - comprehensive_taints.append(taint_flag.desc) - - return comprehensive_taints From 2a5f38ebad48e0d729b3b22caac84bd4209f20a2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:39:07 +0100 Subject: [PATCH 026/137] introduce versioned Linux utilities --- .../framework/symbols/linux/utilities/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 volatility3/framework/symbols/linux/utilities/__init__.py diff --git a/volatility3/framework/symbols/linux/utilities/__init__.py b/volatility3/framework/symbols/linux/utilities/__init__.py new file mode 100644 index 000000000..4225d444b --- /dev/null +++ b/volatility3/framework/symbols/linux/utilities/__init__.py @@ -0,0 +1,11 @@ +from volatility3 import framework +from volatility3.framework import interfaces + + +class LinuxUtilityInterface(interfaces.configuration.VersionableInterface): + """Class with multiple useful Linux functions surrounding a specific piece of functionality.""" + + _version = (2, 1, 1) + _required_framework_version = (2, 0, 0) + + framework.require_interface_version(*_required_framework_version) From 8bc62598f4530bdf2fb99aeb725e5b8f3e0d8cd5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:39:57 +0100 Subject: [PATCH 027/137] initial tainting utilities --- .../symbols/linux/utilities/tainting.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 volatility3/framework/symbols/linux/utilities/tainting.py diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py new file mode 100644 index 000000000..e6d75a963 --- /dev/null +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -0,0 +1,130 @@ +from volatility3 import framework +from volatility3.framework import interfaces +from volatility3.framework.symbols.linux.utilities import LinuxUtilityInterface +from volatility3.framework.constants import linux as linux_constants +from typing import List, Optional + + +class Tainting(LinuxUtilityInterface): + """Tainted kernel and modules parsing capabilities. + + Relevant Linux kernel functions: + - modules: module_flags_taint + - kernel: print_tainted + """ + + _version = (1, 0, 0) + _required_framework_version = (2, 14, 0) + + framework.require_interface_version(*_required_framework_version) + + def __init__( + self, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + ): + self.kernel = context.modules[kernel_module_name] + + @property + def _kernel_taint_flags_list( + self, + ) -> Optional[List[interfaces.objects.ObjectInterface]]: + if self.kernel.has_symbol("taint_flags"): + return list(self.kernel.object_from_symbol("taint_flags")) + return None + + def _module_flags_taint_pre_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on statically defined taints mappings in the framework. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for char, taint_flag in linux_constants.TAINT_FLAGS.items(): + if is_module and is_module != taint_flag.module: + continue + + if taints & taint_flag.shift: + taints_string += char + + return taints_string + + def _module_flags_taint_post_4_10_rc1( + self, taints: int, is_module: bool = False + ) -> str: + """Convert the module's taints value to a 1-1 character mapping. + Relies on kernel symbol embedded taints definitions. + + struct taint_flag { + char c_true; /* character printed when tainted */ + char c_false; /* character printed when not tainted */ + bool module; /* also show as a per-module taint flag */ + }; + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + The raw taints string. + """ + taints_string = "" + for i, taint_flag in enumerate(self._kernel_taint_flags_list): + if is_module and is_module != taint_flag.module: + continue + c_true = chr(taint_flag.c_true) + c_false = chr(taint_flag.c_false) + if taints & (1 << i): + taints_string += c_true + elif c_false != " ": + taints_string += c_false + + return taints_string + + def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + """Convert the taints value to a 1-1 character mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + s + Returns: + The raw taints string. + + Documentation: + - module_flags_taint kernel function + """ + + if self._kernel_taint_flags_list: + return self._module_flags_taint_post_4_10_rc1(taints, is_module) + return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + + def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + """Convert the taints string to a 1-1 descriptor mapping. + + Args: + taints: The taints value, represented by an integer + is_module: Indicates if the taints value is associated with a built-in/LKM module + + Returns: + A comprehensive (user-friendly) taint descriptor list. + + Documentation: + - module_flags_taint kernel function + """ + comprehensive_taints = [] + for character in self.get_taints_as_plain_string(taints, is_module): + taint_flag = linux_constants.TAINT_FLAGS.get(character) + if not taint_flag: + comprehensive_taints.append(f"") + elif taint_flag.when_present: + comprehensive_taints.append(taint_flag.desc) + + return comprehensive_taints From 3105a31964a1a36420281bd995d983a81b161e97 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:40:49 +0100 Subject: [PATCH 028/137] leverage Tainting from separated Linux utilities --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 075a83ae8..ac07d2def 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -19,7 +19,7 @@ from volatility3.framework.layers import linear from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf - +from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -286,7 +286,7 @@ class module(generic.GenericIntelProcess): Returns: The raw taints string. """ - return linux.Tainting( + return tainting.Tainting( self._context, linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, ).get_taints_as_plain_string(self.taints, True) @@ -298,7 +298,7 @@ class module(generic.GenericIntelProcess): Returns: A comprehensive (user-friendly) taint descriptor list. """ - return linux.Tainting( + return tainting.Tainting( self._context, linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, ).get_taints_parsed(self.taints, True) From 6e4213e321b96dd8f0b35df6c87aa8426698a42c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:41:37 +0100 Subject: [PATCH 029/137] update tainting requirements to new versioned utilities --- volatility3/framework/plugins/linux/modxview.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index d79f5e7a9..c97864a87 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -9,6 +9,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints, TreeGrid, NotAvailableValue from volatility3.framework.symbols.linux import extensions from volatility3.framework.constants import architectures +from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -18,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 11, 0) + _required_framework_version = (2, 14, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -28,6 +29,9 @@ class Modxview(interfaces.plugins.PluginInterface): description="Linux kernel", architectures=architectures.LINUX_ARCHS, ), + requirements.VersionRequirement( + name="linux-tainting", component=tainting.Tainting, version=(1, 0, 0) + ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) ), From 0a3502697cca4dac2ac2f39896ecfaaef507ac9b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:42:16 +0100 Subject: [PATCH 030/137] 2.13.0 -> 2.14.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 11edc07d8..9ca2d0a5b 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 13 # Number of changes that only add to the interface +VERSION_MINOR = 14 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 89b8da8c39fe14f699d711c95a8311ec1e21331e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 2 Jan 2025 16:48:22 +0100 Subject: [PATCH 031/137] make self.kernel private and call parent __init__ --- .../framework/symbols/linux/utilities/tainting.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index e6d75a963..603215961 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -22,15 +22,18 @@ class Tainting(LinuxUtilityInterface): self, context: interfaces.context.ContextInterface, kernel_module_name: str, + *args, + **kwargs, ): - self.kernel = context.modules[kernel_module_name] + super().__init__(*args, **kwargs) + self._kernel = context.modules[kernel_module_name] @property def _kernel_taint_flags_list( self, ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self.kernel.has_symbol("taint_flags"): - return list(self.kernel.object_from_symbol("taint_flags")) + if self._kernel.has_symbol("taint_flags"): + return list(self._kernel.object_from_symbol("taint_flags")) return None def _module_flags_taint_pre_4_10_rc1( From d2bb5c9f31d7f01fe2e343867c0c7c1926b3ac50 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Fri, 3 Jan 2025 10:01:37 +1100 Subject: [PATCH 032/137] linux: fix kmsg fstring bug introduced in #1502 --- volatility3/framework/plugins/linux/kmsg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 894ca575f..30f67b319 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ class ABCKmsg(ABC): # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return f"{nsec / 1000000000}.{(nsec % 1000000000) / 1000:06}" + return f"{nsec // 1000000000}.{(nsec % 1000000000) // 1000:06}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info From 4a34b988d1e4cb02e33e555c3e2ed63d808e5028 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 3 Jan 2025 13:21:09 +0100 Subject: [PATCH 033/137] minor readability adjustments --- .../framework/symbols/linux/utilities/tainting.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 603215961..f7f6c83ec 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -51,7 +51,7 @@ class Tainting(LinuxUtilityInterface): """ taints_string = "" for char, taint_flag in linux_constants.TAINT_FLAGS.items(): - if is_module and is_module != taint_flag.module: + if is_module and not taint_flag.module: continue if taints & taint_flag.shift: @@ -79,12 +79,12 @@ class Tainting(LinuxUtilityInterface): The raw taints string. """ taints_string = "" - for i, taint_flag in enumerate(self._kernel_taint_flags_list): - if is_module and is_module != taint_flag.module: + for taint_bit, taint_flag in enumerate(self._kernel_taint_flags_list): + if is_module and not taint_flag.module: continue c_true = chr(taint_flag.c_true) c_false = chr(taint_flag.c_false) - if taints & (1 << i): + if taints & (1 << taint_bit): taints_string += c_true elif c_false != " ": taints_string += c_false @@ -97,7 +97,6 @@ class Tainting(LinuxUtilityInterface): Args: taints: The taints value, represented by an integer is_module: Indicates if the taints value is associated with a built-in/LKM module - s Returns: The raw taints string. From 38c5cc168f93a4d1a5cab2a6c9b071cf32e22fc2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 3 Jan 2025 13:23:27 +0100 Subject: [PATCH 034/137] bump framework req to 2.16.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- volatility3/framework/symbols/linux/utilities/tainting.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index c97864a87..042930740 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 14, 0) + _required_framework_version = (2, 16, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index f7f6c83ec..fc2f94109 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -14,7 +14,7 @@ class Tainting(LinuxUtilityInterface): """ _version = (1, 0, 0) - _required_framework_version = (2, 14, 0) + _required_framework_version = (2, 16, 0) framework.require_interface_version(*_required_framework_version) From 32ca62bbb1205b11be0338e741e3046d503153a8 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:20:35 +0000 Subject: [PATCH 035/137] Make f-string slightly more readable --- .../framework/plugins/windows/shimcachemem.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index b8e9b5bd7..9d968c30a 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -305,14 +305,14 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf If a number of validity checks are passed, this method will return the `SHIM_CACHE_HEAD` object. Otherwise, `None` is returned. """ - # print("checking RTL_AVL_TABLE at offset %s" % hex(offset)) + # Check RTL_AVL_TABLE at offset rtl_avl_table = context.object( symbol_table + constants.BANG + "_RTL_AVL_TABLE", layer_name, offset ) if not rtl_avl_table.is_valid(mod_page_start, mod_page_end): return None - vollog.debug(f"Candidate RTL_AVL_TABLE found at offset {hex(offset)}") + vollog.debug(f"Candidate RTL_AVL_TABLE found at offset {offset:#x}") ersrc_size = context.symbol_space.get_type( kernel_symbol_table + constants.BANG + "_ERESOURCE" @@ -324,13 +324,13 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf # 0x20 if context.symbol_space.get_type("pointer").size == 8 else 0x10 ) vollog.debug( - f"ERESOURCE size: {hex(ersrc_size)}, ERESOURCE alignment: {hex(ersrc_alignment)}" + f"ERESOURCE size: {ersrc_size:#x}, ERESOURCE alignment: {ersrc_alignment:#x}" ) eresource_rel_off = ersrc_size + ((offset - ersrc_size) % ersrc_alignment) eresource_offset = offset - eresource_rel_off - vollog.debug(f"Constructing ERESOURCE at {hex(eresource_offset)}") + vollog.debug(f"Constructing ERESOURCE at {eresource_offset:#x}") eresource = context.object( kernel_symbol_table + constants.BANG + "_ERESOURCE", layer_name, @@ -408,8 +408,8 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf # iterate over ahcache kernel module's .data section in search of *two* SHIM handles shim_heads = [] - vollog.debug(f"PAGE offset: {hex(mod_page_offset)}") - vollog.debug(f".data offset: {hex(data_sec_offset)}") + vollog.debug(f"PAGE offset: {mod_page_offset:#x}") + vollog.debug(f".data offset: {data_sec_offset:#x}") handle_type = context.symbol_space.get_type( shimcache_symbol_table + constants.BANG + "SHIM_CACHE_HANDLE" @@ -419,7 +419,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf data_sec_offset + data_sec_size, 8 if symbols.symbol_table_is_64bit(context, nt_symbol_table) else 4, ): - vollog.debug(f"Building shim handle pointer at {hex(offset)}") + vollog.debug(f"Building shim handle pointer at {offset:#x}") shim_handle = context.object( object_type=shimcache_symbol_table + constants.BANG + "pointer", layer_name=kernel_layer_name, @@ -430,7 +430,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf if shim_handle.is_valid(mod_page_offset, mod_page_offset + mod_page_size): if shim_handle.head is not None: vollog.debug( - f"Found valid shim handle @ {hex(shim_handle.vol.offset)}" + f"Found valid shim handle @ {shim_handle.vol.offset:#x}" ) shim_heads.append(shim_handle.head) if len(shim_heads) == 2: @@ -440,7 +440,7 @@ class ShimcacheMem(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterf vollog.debug("Failed to identify two valid SHIM_CACHE_HANDLE structures") return - # On Windows 8 x64, the frist cache contains the shim cache + # On Windows 8 x64, the first cache contains the shim cache. # On Windows 8 x86, 8.1 x86/x64, and 10, the second cache contains the shim cache. if ( not symbols.symbol_table_is_64bit(context, nt_symbol_table) From 03049f789559af5c4cdb56f343460178b52220f9 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 3 Jan 2025 18:40:52 +0000 Subject: [PATCH 036/137] Add missing exception handling in env var recovery. Prevent backtraces --- volatility3/framework/plugins/linux/envars.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 8cdbfe493..04b75c8a8 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -5,7 +5,7 @@ import logging from typing import Iterable, Tuple -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -58,10 +58,16 @@ class Envars(plugins.PluginInterface): Tuples of (key, value) representing each environment variable. """ - task_name = utility.array_to_string(task.comm) + # This ensures the `task` is valid as well as its + # memory mapping structures + try: + task_name = utility.array_to_string(task.comm) + env_start = task.mm.env_start + env_end = task.mm.env_end + except exceptions.InvalidAddressException: + return None + task_pid = task.pid - env_start = task.mm.env_start - env_end = task.mm.env_end env_area_size = env_end - env_start if not (0 < env_area_size <= env_area_max_size): vollog.debug( From 8ba60a2aaddf86e4cbd065c95d2553ce221db183 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 4 Jan 2025 16:49:02 +0000 Subject: [PATCH 037/137] Change add_process_layer to return None instead of throwing an exception as it was meant to be designed --- volatility3/framework/symbols/linux/extensions/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index b02f80433..df1c00e3d 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -324,9 +324,11 @@ class task_struct(generic.GenericIntelProcess): raise TypeError( "Parent layer is not a translation layer, unable to construct process layer" ) - dtb, layer_name = parent_layer.translate(pgd) - if not dtb: + try: + dtb, layer_name = parent_layer.translate(pgd) + except exceptions.InvalidAddressException: return None + if preferred_name is None: preferred_name = self.vol.layer_name + f"_Process{self.pid}" # Add the constructed layer and return the name From 5f1d318c715311ed12d67bde5a87a8a78e0d3bf0 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 13:39:00 +0000 Subject: [PATCH 038/137] Tiny comment changes --- volatility3/framework/plugins/windows/cmdscan.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 9645ee507..3dc70d649 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -67,6 +67,7 @@ class CmdScan(interfaces.plugins.PluginInterface): Args: conhost_proc: the process object for conhost.exe + size_filter: filter (keep) vads less than this size (bytes) Returns: A list of tuples of: @@ -100,7 +101,7 @@ class CmdScan(interfaces.plugins.PluginInterface): kernel_symbol_table_name: The name of the table containing the kernel symbols config_path: The config path where to find symbol files procs: list of process objects - max_history: an initial set of CommandHistorySize values + max_history: An initial set of CommandHistorySize values Returns: The conhost process object, the command history structure, a dictionary of properties for @@ -227,7 +228,6 @@ class CmdScan(interfaces.plugins.PluginInterface): "data": command_history.CommandCountMax, } ) - command_history_properties.append( { "level": 1, @@ -236,6 +236,7 @@ class CmdScan(interfaces.plugins.PluginInterface): "data": "", } ) + for ( cmd_index, bucket_cmd, @@ -352,7 +353,7 @@ class CmdScan(interfaces.plugins.PluginInterface): def _conhost_proc_filter(self, proc: interfaces.objects.ObjectInterface): """ - Used to filter to only conhost.exe processes + Used to filter only conhost.exe processes """ process_name = utility.array_to_string(proc.ImageFileName) From ab60add9933ee3863c3f2329d2c99af314b5b453 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 17:14:21 +0000 Subject: [PATCH 039/137] Update case insensitive check Update link and use casefold() instead of lower(). --- volatility3/framework/layers/registry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index c684ccd40..6d85da982 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -192,9 +192,9 @@ class RegistryHive(linear.LinearlyMappedLayer): while key_array and node_key: subkeys = node_key[-1].get_subkeys() for subkey in subkeys: - # registry keys are not case sensitive so compare lowercase - # https://msdn.microsoft.com/en-us/library/windows/desktop/ms724946(v=vs.85).aspx - if subkey.get_name().lower() == key_array[0].lower(): + # registry keys are not case sensitive so compare likewise + # https://learn.microsoft.com/en-gb/windows/win32/sysinfo/structure-of-the-registry + if subkey.get_name().casefold() == key_array[0].casefold(): node_key = node_key + [subkey] found_key, key_array = found_key + [key_array[0]], key_array[1:] break From 8f4f576e93a7594666f0e58f8ae73cce5538902c Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 21:09:15 +0000 Subject: [PATCH 040/137] Update case insensitive check Update link and use casefold() instead of lower(). --- volatility3/framework/layers/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 6d85da982..21e1a938e 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -193,7 +193,7 @@ class RegistryHive(linear.LinearlyMappedLayer): subkeys = node_key[-1].get_subkeys() for subkey in subkeys: # registry keys are not case sensitive so compare likewise - # https://learn.microsoft.com/en-gb/windows/win32/sysinfo/structure-of-the-registry + # https://learn.microsoft.com/en-us/windows/win32/sysinfo/structure-of-the-registry if subkey.get_name().casefold() == key_array[0].casefold(): node_key = node_key + [subkey] found_key, key_array = found_key + [key_array[0]], key_array[1:] From 94ec7d89c09b2a276e79fc4c7561828340d5712a Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Sun, 5 Jan 2025 21:55:58 +0000 Subject: [PATCH 041/137] Tiny comment changes --- volatility3/framework/plugins/windows/cmdscan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/windows/cmdscan.py b/volatility3/framework/plugins/windows/cmdscan.py index 3dc70d649..0cd0addb2 100644 --- a/volatility3/framework/plugins/windows/cmdscan.py +++ b/volatility3/framework/plugins/windows/cmdscan.py @@ -67,7 +67,7 @@ class CmdScan(interfaces.plugins.PluginInterface): Args: conhost_proc: the process object for conhost.exe - size_filter: filter (keep) vads less than this size (bytes) + size_filter: size above which vads will not be returned Returns: A list of tuples of: @@ -100,7 +100,7 @@ class CmdScan(interfaces.plugins.PluginInterface): kernel_layer_name: The name of the layer on which to operate kernel_symbol_table_name: The name of the table containing the kernel symbols config_path: The config path where to find symbol files - procs: list of process objects + procs: List of process objects max_history: An initial set of CommandHistorySize values Returns: From a7b4e2fb45bef981eb54c44a5e0cef87b879058f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:02:15 +0100 Subject: [PATCH 042/137] version check_modules --- volatility3/framework/plugins/linux/check_modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/check_modules.py b/volatility3/framework/plugins/linux/check_modules.py index 9b3594c5e..0ed638d9c 100644 --- a/volatility3/framework/plugins/linux/check_modules.py +++ b/volatility3/framework/plugins/linux/check_modules.py @@ -18,6 +18,7 @@ vollog = logging.getLogger(__name__) class Check_modules(plugins.PluginInterface): """Compares module list to sysfs info, if available""" + _version = (1, 0, 0) _required_framework_version = (2, 0, 0) @classmethod From 2d262e7acf5c9aabb32240c01cd57890b7d57647 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:20:56 +0100 Subject: [PATCH 043/137] cut unnecessary intermediate LinuxUtilityInterface --- .../framework/symbols/linux/utilities/__init__.py | 11 ----------- .../framework/symbols/linux/utilities/tainting.py | 5 ++--- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/__init__.py b/volatility3/framework/symbols/linux/utilities/__init__.py index 4225d444b..e69de29bb 100644 --- a/volatility3/framework/symbols/linux/utilities/__init__.py +++ b/volatility3/framework/symbols/linux/utilities/__init__.py @@ -1,11 +0,0 @@ -from volatility3 import framework -from volatility3.framework import interfaces - - -class LinuxUtilityInterface(interfaces.configuration.VersionableInterface): - """Class with multiple useful Linux functions surrounding a specific piece of functionality.""" - - _version = (2, 1, 1) - _required_framework_version = (2, 0, 0) - - framework.require_interface_version(*_required_framework_version) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index fc2f94109..29d7d2b5b 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -1,11 +1,10 @@ from volatility3 import framework from volatility3.framework import interfaces -from volatility3.framework.symbols.linux.utilities import LinuxUtilityInterface from volatility3.framework.constants import linux as linux_constants from typing import List, Optional -class Tainting(LinuxUtilityInterface): +class Tainting(interfaces.configuration.VersionableInterface): """Tainted kernel and modules parsing capabilities. Relevant Linux kernel functions: @@ -14,7 +13,7 @@ class Tainting(LinuxUtilityInterface): """ _version = (1, 0, 0) - _required_framework_version = (2, 16, 0) + _required_framework_version = (2, 0, 0) framework.require_interface_version(*_required_framework_version) From 5c70356c27aedc931c03a8e633952b126ef5254b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:23:16 +0100 Subject: [PATCH 044/137] version check_modules requirement --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 042930740..b44f84c7d 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -38,7 +38,7 @@ class Modxview(interfaces.plugins.PluginInterface): requirements.PluginRequirement( name="check_modules", plugin=check_modules.Check_modules, - version=(0, 0, 0), + version=(1, 0, 0), ), requirements.PluginRequirement( name="hidden_modules", From 302f9fdf5ba1c24d07d2fce3d0f7c87c3e6bd1f2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:24:18 +0100 Subject: [PATCH 045/137] cut unnecessary plugin runner functions --- .../framework/plugins/linux/modxview.py | 78 ++++++------------- 1 file changed, 25 insertions(+), 53 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index b44f84c7d..a247bc2cc 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -53,54 +53,6 @@ class Modxview(interfaces.plugins.PluginInterface): ), ] - @classmethod - def run_lsmod( - cls, context: interfaces.context.ContextInterface, kernel_name: str - ) -> List[extensions.module]: - """Wrapper for the lsmod plugin.""" - return list(lsmod.Lsmod.list_modules(context, kernel_name)) - - @classmethod - def run_check_modules( - cls, - context: interfaces.context.ContextInterface, - kernel_name: str, - ) -> List[extensions.module]: - """Wrapper for the check_modules plugin. - Here, we extract the /sys/module/ list.""" - kernel = context.modules[kernel_name] - sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( - context, kernel_name - ) - - # Convert get_kset_modules() offsets back to module objects - return [ - kernel.object(object_type="module", offset=m_offset, absolute=True) - for m_offset in sysfs_modules.values() - ] - - @classmethod - def run_hidden_modules( - cls, - context: interfaces.context.ContextInterface, - kernel_name: str, - known_modules_addresses: Set[int], - ) -> List[extensions.module]: - """Wrapper for the hidden_modules plugin.""" - modules_memory_boundaries = ( - hidden_modules.Hidden_modules.get_modules_memory_boundaries( - context, kernel_name - ) - ) - return list( - hidden_modules.Hidden_modules.get_hidden_modules( - context, - kernel_name, - known_modules_addresses, - modules_memory_boundaries, - ) - ) - @classmethod def flatten_run_modules_results( cls, run_results: Dict[str, List[extensions.module]], deduplicate: bool = True @@ -140,15 +92,35 @@ class Modxview(interfaces.plugins.PluginInterface): kernel = context.modules[kernel_name] run_results = {} - run_results["lsmod"] = cls.run_lsmod(context, kernel_name) - run_results["check_modules"] = cls.run_check_modules(context, kernel_name) + # lsmod + run_results["lsmod"] = list(lsmod.Lsmod.list_modules(context, kernel_name)) + # check_modules + sysfs_modules: dict = check_modules.Check_modules.get_kset_modules( + context, kernel_name + ) + ## Convert get_kset_modules() offsets back to module objects + run_results["check_modules"] = [ + kernel.object(object_type="module", offset=m_offset, absolute=True) + for m_offset in sysfs_modules.values() + ] + # hidden_modules if run_hidden_modules: - known_module_addresses = set( + known_modules_addresses = set( context.layers[kernel.layer_name].canonicalize(module.vol.offset) for module in run_results["lsmod"] + run_results["check_modules"] ) - run_results["hidden_modules"] = cls.run_hidden_modules( - context, kernel_name, known_module_addresses + modules_memory_boundaries = ( + hidden_modules.Hidden_modules.get_modules_memory_boundaries( + context, kernel_name + ) + ) + run_results["hidden_modules"] = list( + hidden_modules.Hidden_modules.get_hidden_modules( + context, + kernel_name, + known_modules_addresses, + modules_memory_boundaries, + ) ) return run_results From 4115c26e7cc119a68aa33fff7f5b8a730b5b2c69 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:24:44 +0100 Subject: [PATCH 046/137] bump framework req to 2.18.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index a247bc2cc..34f5bac8f 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 16, 0) + _required_framework_version = (2, 18, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From bd82f4f33d860cb067e379600da5bbc74f9e2247 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:25:07 +0100 Subject: [PATCH 047/137] 2.16.0 -> 2.18.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 24f96fa89..832b2a5ba 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 16 # Number of changes that only add to the interface +VERSION_MINOR = 18 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From d956742db98610dfa94678ffb98d53c5b6bcd161 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 6 Jan 2025 00:31:39 +0100 Subject: [PATCH 048/137] remove typing.Set import --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 34f5bac8f..3655200e8 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import List, Dict, Set, Iterator +from typing import List, Dict, Iterator from volatility3.plugins.linux import lsmod, check_modules, hidden_modules from volatility3.framework import interfaces from volatility3.framework.configuration import requirements From b5bc54cfaed91f4d615790305c80ce802658dafe Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:18:34 +0000 Subject: [PATCH 049/137] Use in-place subtraction Also tweak comments. --- volatility3/framework/renderers/conversion.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index e48684b31..f848b2dad 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -18,7 +18,7 @@ def wintime_to_datetime( unix_time = wintime // 10000000 if unix_time == 0: return renderers.NotApplicableValue() - unix_time = unix_time - 11644473600 + unix_time -= 11644473600 try: return datetime.datetime.fromtimestamp(unix_time, datetime.timezone.utc) # Windows sometimes throws OSErrors rather than ValueError/OverflowError when it can't convert a value @@ -71,7 +71,7 @@ def round(addr: int, align: int, up: bool = False) -> int: Args: addr: the address align: the alignment value - up: Whether to round up or not + up: whether to round up or not Returns: The aligned address @@ -122,11 +122,12 @@ def convert_port(port_as_integer): def convert_network_four_tuple(family, four_tuple): - """Converts the connection four_tuple: (source ip, source port, dest ip, - dest port) + """Converts the connection four_tuple: + + (source ip, source port, dest ip, dest port) into their string equivalents. IP addresses are expected as a tuple - of unsigned shorts Ports are converted to proper endianness as well + of unsigned shorts. Ports are converted to proper endianness as well. """ if family == socket.AF_INET: From 43ac6c4d6271c928d9bcdaf6407e01e1c96d7cf9 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 7 Jan 2025 10:24:46 -0600 Subject: [PATCH 050/137] Fix copy-pasted module docstrings This updates the module docstrings for 5 modules that duplicate the docstring from the `proc` module. This was presumably the result of using the `proc` module as a template for the others. --- volatility3/framework/plugins/linux/bash.py | 4 ++-- volatility3/framework/plugins/linux/check_afinfo.py | 4 ++-- volatility3/framework/plugins/linux/check_syscall.py | 3 +-- volatility3/framework/plugins/linux/elfs.py | 4 ++-- volatility3/framework/plugins/linux/lsmod.py | 3 +-- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index 056e3cd51..8acfeb848 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that recovers bash command history +from bash process memory.""" import datetime import struct diff --git a/volatility3/framework/plugins/linux/check_afinfo.py b/volatility3/framework/plugins/linux/check_afinfo.py index 201a443f7..7aa3cbdd2 100644 --- a/volatility3/framework/plugins/linux/check_afinfo.py +++ b/volatility3/framework/plugins/linux/check_afinfo.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that verifies the operation function +pointers of network protocols.""" import logging from typing import List diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index 3537a9fa1..13d312f2f 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -1,8 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that checks the system call table for hooks.""" import contextlib import logging from typing import List diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 2fd740941..0d1c9c2dd 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -1,8 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin for enumerating memory-mapped +ELF files across all processes.""" import logging from typing import List, Optional, Type diff --git a/volatility3/framework/plugins/linux/lsmod.py b/volatility3/framework/plugins/linux/lsmod.py index 49e990e93..e9a2a7137 100644 --- a/volatility3/framework/plugins/linux/lsmod.py +++ b/volatility3/framework/plugins/linux/lsmod.py @@ -1,8 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -"""A module containing a collection of plugins that produce data typically -found in Linux's /proc file system.""" +"""A module containing a plugin that lists loaded kernel modules.""" import logging from typing import List, Iterable From 32cb6e11f6abe86ce5284e1a618bae9ab1cd4a5f Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Tue, 7 Jan 2025 19:41:02 +0000 Subject: [PATCH 051/137] Change one letter of a typo --- volatility3/framework/plugins/windows/driverscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/driverscan.py b/volatility3/framework/plugins/windows/driverscan.py index 24d81c3d5..d388ffbb7 100644 --- a/volatility3/framework/plugins/windows/driverscan.py +++ b/volatility3/framework/plugins/windows/driverscan.py @@ -64,7 +64,7 @@ class DriverScan(interfaces.plugins.PluginInterface): names associated with a driver Args: - driver: A Eriver object + driver: A Driver object Returns: A tuple of strings of (driver name, service key, driver alt. name) From 0860441c2fc5a5a97902a7582473d8462c457bc3 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 17:11:02 +0000 Subject: [PATCH 052/137] Core: Improve speed to JSONSchema validation --- pyproject.toml | 3 +-- volatility3/framework/plugins/isfinfo.py | 2 +- volatility3/schemas/__init__.py | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86e3921d2..af22cbe0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,10 +33,9 @@ cloud = [ dev = [ "volatility3[full,cloud]", - "jsonschema>=4.23.0,<5", + "fastjsonschema>=2.21.1,<3", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", - "types-jsonschema>=4.23.0,<5", ] test = [ diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 1c2ac52e9..34b0a5653 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -97,7 +97,7 @@ class IsfInfo(plugins.PluginInterface): if filter_item in isf_file: filtered_list.append(isf_file) - if find_spec("jsonschema") and self.config["validate"]: + if find_spec("fastjsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False" diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 90cfaba48..3964e29a6 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -14,6 +14,8 @@ vollog = logging.getLogger(__name__) cached_validation_filepath = os.path.join(constants.CACHE_PATH, "valid_isf.hashcache") +validators = {} + def load_cached_validations() -> Set[str]: """Loads up the list of successfully cached json objects, so we don't need @@ -92,7 +94,12 @@ def valid( if input_hash in cached_validations and use_cache: return True try: - import jsonschema + import fastjsonschema + + schema_key = json.dumps(schema, sort_keys=True) + if schema_key not in validators: + validator = fastjsonschema.compile(schema) + validators[schema_key] = validator except ImportError: vollog.info("Dependency for validation unavailable: jsonschema") vollog.debug("All validations will report success, even with malformed input") @@ -100,10 +107,13 @@ def valid( try: vollog.debug("Validating JSON against schema...") - jsonschema.validate(input, schema) + validators[schema_key](input) + import pdb + + pdb.set_trace() cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") - except jsonschema.exceptions.SchemaError: + except fastjsonschema.JsonSchemaValueException: vollog.debug("Schema validation error", exc_info=True) return False From e676e6179a3cd9c8d6afb838560d7a9e4d3a5420 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:08:22 +0000 Subject: [PATCH 053/137] Swap fastjsonschema for jsonschema because of date-time validation issues --- volatility3/schemas/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 3964e29a6..b94e0831d 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -94,11 +94,13 @@ def valid( if input_hash in cached_validations and use_cache: return True try: - import fastjsonschema + import jsonschema schema_key = json.dumps(schema, sort_keys=True) if schema_key not in validators: - validator = fastjsonschema.compile(schema) + validator_class = jsonschema.validators.validator_for(schema) + validator_class.check_schema(schema) + validator = validator_class(schema) validators[schema_key] = validator except ImportError: vollog.info("Dependency for validation unavailable: jsonschema") @@ -107,10 +109,7 @@ def valid( try: vollog.debug("Validating JSON against schema...") - validators[schema_key](input) - import pdb - - pdb.set_trace() + validators[schema_key].validate(input) cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") except fastjsonschema.JsonSchemaValueException: From 21decf13708d882369cc1f8fa2884f5a8ae5494d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:12:36 +0000 Subject: [PATCH 054/137] Core: Put the dependencies back for jsonschema --- pyproject.toml | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af22cbe0a..fbdbf0a8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,15 @@ [project] name = "volatility3" description = "Memory forensics framework" -keywords = ["volatility", "memory", "forensics", "framework", "windows", "linux", "volshell"] +keywords = [ + "volatility", + "memory", + "forensics", + "framework", + "windows", + "linux", + "volshell", +] readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, @@ -10,9 +18,7 @@ requires-python = ">=3.8.0" license = { text = "VSL" } dynamic = ["version"] -dependencies = [ - "pefile>=2024.8.26", -] +dependencies = ["pefile>=2024.8.26"] [project.optional-dependencies] full = [ @@ -26,16 +32,14 @@ full = [ "pillow>=10.0.0,<11.0.0", ] -cloud = [ - "gcsfs>=2024.10.0", - "s3fs>=2024.10.0", -] +cloud = ["gcsfs>=2024.10.0", "s3fs>=2024.10.0"] dev = [ "volatility3[full,cloud]", - "fastjsonschema>=2.21.1,<3", + "jsonschema>=4.23.0,<5", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", + "types-jsonschema>=4.23.0,<5", ] test = [ @@ -78,16 +82,16 @@ target-version = "py38" [tool.ruff.lint] select = [ - "F", # pyflakes - "E", # pycodestyle errors - "W", # pycodestyle warnings - "G", # flake8-logging-format - "PIE", # flake8-pie - "UP", # pyupgrade + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "G", # flake8-logging-format + "PIE", # flake8-pie + "UP", # pyupgrade ] ignore = [ - "E501", # ignore due to conflict with formatter + "E501", # ignore due to conflict with formatter ] [build-system] From d07d31047b2ed49a570aa4b3698f6a64145c2d83 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:24:48 +0000 Subject: [PATCH 055/137] Core: Revert the exception catching too --- volatility3/schemas/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index b94e0831d..e894def9f 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -112,7 +112,7 @@ def valid( validators[schema_key].validate(input) cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") - except fastjsonschema.JsonSchemaValueException: + except jsonschema.exceptions.SchemaError: vollog.debug("Schema validation error", exc_info=True) return False From 585901105275a015a3c4326e486f4e2a52d8eb12 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 12:35:04 +0100 Subject: [PATCH 056/137] introduce customizable plugin arparse epilog --- volatility3/cli/__init__.py | 3 +++ volatility3/framework/interfaces/plugins.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 6172a17f3..87caaece6 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -368,6 +368,9 @@ class CommandLine: help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, ) + epilog = getattr(plugin_list[plugin], "_argparse_epilog", None) + if epilog is not None: + plugin_parser.epilog = epilog self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) ### diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index f763815a6..6cd72f02e 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -112,6 +112,8 @@ class PluginInterface( # Be careful with inheritance around this (We default to requiring a version which doesn't exist, so it must be set) _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" + _argparse_epilog: str = None + """Display additional description of the plugin after the description of the arguments. See: https://docs.python.org/3/library/argparse.html#epilog""" def __init__( self, From 530617a700e259f69d53f62f08ccc3382bcdd057 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:52:54 +0000 Subject: [PATCH 057/137] Small readability improvements --- volatility3/framework/automagic/mac.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index f3679d160..a883028d2 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -101,7 +101,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVVV, - f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}", + f"Skipping invalid idlepml4_ptr: {idlepml4_ptr:#x}", ) continue @@ -112,7 +112,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if tmp_dtb % 4096: vollog.log( constants.LOGLEVEL_VVV, - f"Skipping non-page aligned DTB: 0x{tmp_dtb:0x}", + f"Skipping non-page aligned DTB: {tmp_dtb:#x}", ) continue @@ -136,7 +136,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): new_layer.config["kernel_virtual_offset"] = kaslr_shift if new_layer and dtb: - vollog.debug(f"DTB was found at: 0x{dtb:0x}") + vollog.debug(f"DTB was found at: {dtb:#x}") return new_layer vollog.debug("No suitable mac banner could be matched") return None @@ -182,7 +182,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) tmp_aslr_shift = offset - cls.virtual_to_physical_address( version_json_address @@ -208,7 +208,6 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): continue aslr_shift = tmp_aslr_shift & 0xFFFFFFFF - break vollog.log(constants.LOGLEVEL_VVVV, f"Mac find_aslr returned: {aslr_shift:0x}") @@ -219,9 +218,9 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): """Converts a virtual mac address to a physical one (does not account of ASLR)""" if addr > 0xFFFFFF8000000000: - addr = addr - 0xFFFFFF8000000000 + addr -= 0xFFFFFF8000000000 else: - addr = addr - 0xFF8000000000 + addr -= 0xFF8000000000 return addr From a7661d45e78b10bc736946425055755c9627d111 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Fri, 10 Jan 2025 09:21:21 -0600 Subject: [PATCH 058/137] Windows: Certificates - handle uncaught RegistryFormatException Changes variable import to module import, and catches an unhandled `RegistryFormatException` in certificates.py --- .../plugins/windows/registry/certificates.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 8587b3719..a83badb90 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -1,11 +1,11 @@ import contextlib import logging import struct -from typing import List, Iterator, Optional, Tuple, Type +from typing import Iterator, List, Optional, Tuple, Type from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements -from volatility3.framework.symbols.windows.extensions.registry import RegValueTypes +from volatility3.framework.symbols.windows.extensions import registry from volatility3.plugins.windows.registry import hivelist, printkey vollog = logging.getLogger(__name__) @@ -81,7 +81,11 @@ class Certificates(interfaces.plugins.PluginInterface): "Microsoft\\SystemCertificates", "Software\\Microsoft\\SystemCertificates", ]: - with contextlib.suppress(KeyError, exceptions.InvalidAddressException): + with contextlib.suppress( + KeyError, + registry.RegistryFormatException, + exceptions.InvalidAddressException, + ): # Walk it node_path = hive.get_key(top_key, return_list=True) for ( @@ -92,7 +96,11 @@ class Certificates(interfaces.plugins.PluginInterface): _volatility, node, ) in printkey.PrintKey.key_iterator(hive, node_path, recurse=True): - if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": + if ( + not is_key + and registry.RegValueTypes(node.Type) + == registry.RegValueTypes.REG_BINARY + ): name, certificate_data = self.parse_data(node.decode_data()) unique_key_offset = ( key_path.casefold().index(top_key.casefold()) From 96eca6e0162a77699c2befcce6df16f7deac4d23 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 19:38:07 +0100 Subject: [PATCH 059/137] more compact _argparse_epilog --- volatility3/cli/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 87caaece6..37923362a 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -368,9 +368,9 @@ class CommandLine: help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, ) - epilog = getattr(plugin_list[plugin], "_argparse_epilog", None) - if epilog is not None: - plugin_parser.epilog = epilog + plugin_parser.epilog = getattr( + plugin_list[plugin], "_argparse_epilog", None + ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) ### From 615d1d5a2e85dcd2f9d65493690a474c15f691cd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 19:49:25 +0100 Subject: [PATCH 060/137] more compact _argparse_epilog --- volatility3/cli/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 37923362a..fde4fcc6d 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -367,9 +367,7 @@ class CommandLine: plugin, help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, - ) - plugin_parser.epilog = getattr( - plugin_list[plugin], "_argparse_epilog", None + epilog=getattr(plugin_list[plugin], "_argparse_epilog", None), ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) From a26ff8fa6e6ba03a6ea3ebe6c5f3b38b3a4d8851 Mon Sep 17 00:00:00 2001 From: j-t-1 <120829237+j-t-1@users.noreply.github.com> Date: Fri, 10 Jan 2025 19:03:51 +0000 Subject: [PATCH 061/137] Small readability improvements --- volatility3/framework/automagic/mac.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index a883028d2..3b16eb353 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -184,12 +184,12 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): for offset, banner in offset_generator: banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) - tmp_aslr_shift = offset - cls.virtual_to_physical_address( + aslr_shift = offset - cls.virtual_to_physical_address( version_json_address ) major_string = context.layers[layer_name].read( - version_major_phys_offset + tmp_aslr_shift, 4 + version_major_phys_offset + aslr_shift, 4 ) major = struct.unpack(" Date: Fri, 10 Jan 2025 19:08:56 +0000 Subject: [PATCH 062/137] Small readability improvements --- volatility3/framework/automagic/mac.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 3b16eb353..94c259463 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -184,9 +184,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): for offset, banner in offset_generator: banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) - aslr_shift = offset - cls.virtual_to_physical_address( - version_json_address - ) + aslr_shift = offset - cls.virtual_to_physical_address(version_json_address) major_string = context.layers[layer_name].read( version_major_phys_offset + aslr_shift, 4 From 1cf0232d25fa6dffa21ae3c281e1f568bbb280ab Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 21:19:31 +0100 Subject: [PATCH 063/137] less specific argparse epilog reference --- volatility3/cli/__init__.py | 2 +- volatility3/framework/interfaces/plugins.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index fde4fcc6d..82a2a4205 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -367,7 +367,7 @@ class CommandLine: plugin, help=plugin_list[plugin].__doc__, description=plugin_list[plugin].__doc__, - epilog=getattr(plugin_list[plugin], "_argparse_epilog", None), + epilog=plugin_list[plugin].additional_description, ) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 6cd72f02e..7ad78d0ba 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -112,7 +112,7 @@ class PluginInterface( # Be careful with inheritance around this (We default to requiring a version which doesn't exist, so it must be set) _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" - _argparse_epilog: str = None + additional_description: str = None """Display additional description of the plugin after the description of the arguments. See: https://docs.python.org/3/library/argparse.html#epilog""" def __init__( From 7913fb2bb0aac4cc390ce6e42ad6621115f0ae7c Mon Sep 17 00:00:00 2001 From: ikelos Date: Fri, 10 Jan 2025 21:07:08 +0000 Subject: [PATCH 064/137] Revert "Small readability improvements" --- volatility3/framework/automagic/mac.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 94c259463..f3679d160 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -101,7 +101,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVVV, - f"Skipping invalid idlepml4_ptr: {idlepml4_ptr:#x}", + f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}", ) continue @@ -112,7 +112,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if tmp_dtb % 4096: vollog.log( constants.LOGLEVEL_VVV, - f"Skipping non-page aligned DTB: {tmp_dtb:#x}", + f"Skipping non-page aligned DTB: 0x{tmp_dtb:0x}", ) continue @@ -136,7 +136,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): new_layer.config["kernel_virtual_offset"] = kaslr_shift if new_layer and dtb: - vollog.debug(f"DTB was found at: {dtb:#x}") + vollog.debug(f"DTB was found at: 0x{dtb:0x}") return new_layer vollog.debug("No suitable mac banner could be matched") return None @@ -182,12 +182,14 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[:2]) + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) - aslr_shift = offset - cls.virtual_to_physical_address(version_json_address) + tmp_aslr_shift = offset - cls.virtual_to_physical_address( + version_json_address + ) major_string = context.layers[layer_name].read( - version_major_phys_offset + aslr_shift, 4 + version_major_phys_offset + tmp_aslr_shift, 4 ) major = struct.unpack(" 0xFFFFFF8000000000: - addr -= 0xFFFFFF8000000000 + addr = addr - 0xFFFFFF8000000000 else: - addr -= 0xFF8000000000 + addr = addr - 0xFF8000000000 return addr From 884237534142ec10ba6e7386eedc06ef30d277d0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 10 Jan 2025 23:28:02 +0100 Subject: [PATCH 065/137] 2.15.0 -> 2.16.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 2f0c53093..24f96fa89 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 15 # Number of changes that only add to the interface +VERSION_MINOR = 16 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 0e4e7518447837b9c7f0f30203155b3a3fee0c3a Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 11 Jan 2025 14:05:36 +0100 Subject: [PATCH 066/137] stateless classmethods --- .../symbols/linux/utilities/tainting.py | 67 +++++++++++++------ 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 29d7d2b5b..552f51b98 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -17,26 +17,22 @@ class Tainting(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) - def __init__( - self, + @classmethod + def _get_kernel_taint_flags_list( + cls, context: interfaces.context.ContextInterface, kernel_module_name: str, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self._kernel = context.modules[kernel_module_name] - - @property - def _kernel_taint_flags_list( - self, ) -> Optional[List[interfaces.objects.ObjectInterface]]: - if self._kernel.has_symbol("taint_flags"): - return list(self._kernel.object_from_symbol("taint_flags")) + kernel = context.modules[kernel_module_name] + if kernel.has_symbol("taint_flags"): + return list(kernel.object_from_symbol("taint_flags")) return None + @classmethod def _module_flags_taint_pre_4_10_rc1( - self, taints: int, is_module: bool = False + cls, + taints: int, + is_module: bool = False, ) -> str: """Convert the module's taints value to a 1-1 character mapping. Relies on statically defined taints mappings in the framework. @@ -58,8 +54,13 @@ class Tainting(interfaces.configuration.VersionableInterface): return taints_string + @classmethod def _module_flags_taint_post_4_10_rc1( - self, taints: int, is_module: bool = False + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, ) -> str: """Convert the module's taints value to a 1-1 character mapping. Relies on kernel symbol embedded taints definitions. @@ -78,7 +79,9 @@ class Tainting(interfaces.configuration.VersionableInterface): The raw taints string. """ taints_string = "" - for taint_bit, taint_flag in enumerate(self._kernel_taint_flags_list): + for taint_bit, taint_flag in enumerate( + cls._get_kernel_taint_flags_list(context, kernel_module_name) + ): if is_module and not taint_flag.module: continue c_true = chr(taint_flag.c_true) @@ -90,7 +93,14 @@ class Tainting(interfaces.configuration.VersionableInterface): return taints_string - def get_taints_as_plain_string(self, taints: int, is_module: bool = False) -> str: + @classmethod + def get_taints_as_plain_string( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, + ) -> str: """Convert the taints value to a 1-1 character mapping. Args: @@ -103,11 +113,22 @@ class Tainting(interfaces.configuration.VersionableInterface): - module_flags_taint kernel function """ - if self._kernel_taint_flags_list: - return self._module_flags_taint_post_4_10_rc1(taints, is_module) - return self._module_flags_taint_pre_4_10_rc1(taints, is_module) + if cls._get_kernel_taint_flags_list(context, kernel_module_name): + return cls._module_flags_taint_post_4_10_rc1( + context, kernel_module_name, taints, is_module + ) + return cls._module_flags_taint_pre_4_10_rc1( + context, kernel_module_name, taints, is_module + ) - def get_taints_parsed(self, taints: int, is_module: bool = False) -> List[str]: + @classmethod + def get_taints_parsed( + cls, + context: interfaces.context.ContextInterface, + kernel_module_name: str, + taints: int, + is_module: bool = False, + ) -> List[str]: """Convert the taints string to a 1-1 descriptor mapping. Args: @@ -121,7 +142,9 @@ class Tainting(interfaces.configuration.VersionableInterface): - module_flags_taint kernel function """ comprehensive_taints = [] - for character in self.get_taints_as_plain_string(taints, is_module): + for character in cls.get_taints_as_plain_string( + context, kernel_module_name, taints, is_module + ): taint_flag = linux_constants.TAINT_FLAGS.get(character) if not taint_flag: comprehensive_taints.append(f"") From 6817d2c765fb5117a8ec6adb92cf343d37a92595 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 16:00:50 +1100 Subject: [PATCH 067/137] linux: ensure process listing functions yield only valid tasks --- volatility3/framework/plugins/linux/pslist.py | 5 ++- .../symbols/linux/extensions/__init__.py | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 37cf000fc..931acf29a 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -34,7 +34,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" _required_framework_version = (2, 13, 0) - _version = (4, 0, 0) + _version = (4, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -250,6 +250,9 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): # Note that the init_task itself is not yielded, since "ps" also never shows it. for task in init_task.tasks: + if not task.is_valid(): + continue + if filter_func(task): continue diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..a50b8ae09 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -307,6 +307,36 @@ class module(generic.GenericIntelProcess): class task_struct(generic.GenericIntelProcess): + def is_valid(self) -> bool: + layer = self._context.layers[self.vol.layer_name] + # Make sure the entire task content is readable + if not layer.is_valid(self.vol.offset, self.vol.size): + return False + + if self.pid < 0: + return False + + if not (self.signal and self.signal.is_readable()): + return False + + if not (self.nsproxy and self.nsproxy.is_readable()): + return False + + if not (self.real_parent and self.real_parent.is_readable()): + return False + + if self.active_mm and not self.active_mm.is_readable(): + return False + + if self.mm: + if not self.mm.is_readable(): + return False + + if self.mm != self.active_mm: + return False + + return True + def add_process_layer( self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: @@ -401,6 +431,8 @@ class task_struct(generic.GenericIntelProcess): tasks_iterable = self._get_tasks_iterable() threads_seen = set([self.vol.offset]) for task in tasks_iterable: + if not task.is_valid(): + continue if task.vol.offset not in threads_seen: threads_seen.add(task.vol.offset) yield task From 093b12b7cdf4a1623a5d534309f0673c0311cc6b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 16:52:45 +1100 Subject: [PATCH 068/137] Linux and Windows: Ensure linked list object extensions consistently yield valid entries --- .../symbols/linux/extensions/__init__.py | 42 ++++++++++------- .../symbols/windows/extensions/__init__.py | 47 +++++++++---------- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..2065b3bb4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -1209,35 +1209,43 @@ class list_head(objects.StructType, collections.abc.Iterable): Objects of the type specified via the "symbol_type" argument. """ - layer = layer or self.vol.layer_name + layer_name = layer or self.vol.layer_name + + trans_layer = self._context.layers[layer_name] + if not trans_layer.is_valid(self.vol.offset): + return None relative_offset = self._context.symbol_space.get_type( symbol_type ).relative_child_offset(member) - direction = "prev" - if forward: - direction = "next" - try: - link = getattr(self, direction).dereference() - except exceptions.InvalidAddressException: + direction = "next" if forward else "prev" + + link_ptr = getattr(self, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() + if not sentinel: - yield self._context.object( - symbol_type, layer, offset=self.vol.offset - relative_offset - ) + obj_offset = self.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + + yield self._context.object(symbol_type, layer_name, offset=obj_offset) + seen = {self.vol.offset} while link.vol.offset not in seen: - obj = self._context.object( - symbol_type, layer, offset=link.vol.offset - relative_offset - ) - yield obj + obj_offset = link.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + + yield self._context.object(symbol_type, layer_name, offset=obj_offset) seen.add(link.vol.offset) - try: - link = getattr(link, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(link, direction) + if not (link_ptr and link_ptr.is_readable()): break + link = link_ptr.dereference() def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index f12fd3f5b..214002f49 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -962,56 +962,55 @@ class LIST_ENTRY(objects.StructType, collections.abc.Iterable): ) -> Iterator[interfaces.objects.ObjectInterface]: """Returns an iterator of the entries in the list.""" - layer = layer or self.vol.layer_name + layer_name = layer or self.vol.layer_name + native_layer_name = layer_name or self.vol.native_layer_name + + trans_layer = self._context.layers[layer_name] + if not trans_layer.is_valid(self.vol.offset): + return None relative_offset = self._context.symbol_space.get_type( symbol_type ).relative_child_offset(member) - direction = "Blink" - if forward: - direction = "Flink" + direction = "Flink" if forward else "Blink" - trans_layer = self._context.layers[layer] - - try: - is_valid = trans_layer.is_valid(self.vol.offset) - if not is_valid: - return None - - link = getattr(self, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(self, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() if not sentinel: + obj_offset = self.vol.offset - relative_offset + if not trans_layer.is_valid(obj_offset): + return None + yield self._context.object( symbol_type, - layer, - offset=self.vol.offset - relative_offset, - native_layer_name=layer or self.vol.native_layer_name, + layer_name, + offset=obj_offset, + native_layer_name=native_layer_name, ) seen = {self.vol.offset} while link.vol.offset not in seen: obj_offset = link.vol.offset - relative_offset - if not trans_layer.is_valid(obj_offset): return None - obj = self._context.object( + yield self._context.object( symbol_type, - layer, + layer_name, offset=obj_offset, - native_layer_name=layer or self.vol.native_layer_name, + native_layer_name=native_layer_name, ) - yield obj seen.add(link.vol.offset) - try: - link = getattr(link, direction).dereference() - except exceptions.InvalidAddressException: + link_ptr = getattr(link, direction) + if not (link_ptr and link_ptr.is_readable()): return None + link = link_ptr.dereference() def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) From 0d9715136cc9cc96637996b2bb027a76b8b5e87a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 17:13:44 +1100 Subject: [PATCH 069/137] Linux: Ensure VMA enumration functions yield only valid objects consistently --- .../symbols/linux/extensions/__init__.py | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..61562270a 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -811,23 +811,30 @@ class mm_struct(objects.StructType): def _get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mmap list member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or - _get_mmap_iter() automatically as required.""" + _get_mmap_iter() automatically as required. + + Yields: + vm_area_struct objects + """ if not self.has_member("mmap"): raise AttributeError( "_get_mmap_iter called on mm_struct where no mmap member exists." ) - if not self.mmap: + vma_pointer = self.mmap + if not (vma_pointer and vma_pointer.is_readable()): return None - yield self.mmap + vma_object = vma_pointer.dereference() + yield vma_object - seen = {self.mmap.vol.offset} - link = self.mmap.vm_next + seen = {vma_pointer} + vma_pointer = vma_pointer.vm_next - while link != 0 and link.vol.offset not in seen: - yield link - seen.add(link.vol.offset) - link = link.vm_next + while vma_pointer and vma_pointer.is_readable() and vma_pointer not in seen: + vma_object = vma_pointer.dereference() + yield vma_object + seen.add(vma_pointer) + vma_pointer = vma_pointer.vm_next # TODO: As of version 3.0.0 this method should be removed def get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: @@ -842,7 +849,11 @@ class mm_struct(objects.StructType): def _get_maple_tree_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: """Returns an iterator for the mm_mt member of an mm_struct. Use this only if required, get_vma_iter() will choose the correct _get_maple_tree_iter() or - get_mmap_iter() automatically as required.""" + get_mmap_iter() automatically as required. + + Yields: + vm_area_struct objects + """ if not self.has_member("mm_mt"): raise AttributeError( @@ -850,20 +861,27 @@ class mm_struct(objects.StructType): ) symbol_table_name = self.get_symbol_table_name() for vma_pointer in self.mm_mt.get_slot_iter(): - # convert pointer to vm_area_struct and yield - vma = self._context.object( + # Convert pointer to vm_area_struct and yield + vma_object = self._context.object( symbol_table_name + constants.BANG + "vm_area_struct", layer_name=self.vol.native_layer_name, offset=vma_pointer, ) - yield vma + yield vma_object def get_vma_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: - """Returns an iterator for the VMAs in an mm_struct. Automatically choosing the mmap or mm_mt as required.""" + """Returns an iterator for the VMAs in an mm_struct. + Automatically choosing the mmap or mm_mt as required. + + Yields: + vm_area_struct objects + """ if self.has_member("mmap"): + # kernels < 6.1 yield from self._get_mmap_iter() elif self.has_member("mm_mt"): + # kernels >= 6.1 d4af56c5c7c6781ca6ca8075e2cf5bc119ed33d1 yield from self._get_maple_tree_iter() else: raise AttributeError("Unable to find mmap or mm_mt in mm_struct") From 8bc04529350c3ce5a927099a72bc4e419a049db5 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Jan 2025 11:21:23 +1100 Subject: [PATCH 070/137] linux: Improve compatibility with ancient kernels --- .../symbols/linux/extensions/__init__.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index a50b8ae09..4a1a263dd 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -316,16 +316,26 @@ class task_struct(generic.GenericIntelProcess): if self.pid < 0: return False - if not (self.signal and self.signal.is_readable()): + if self.has_member("signal") and not ( + self.signal and self.signal.is_readable() + ): return False - if not (self.nsproxy and self.nsproxy.is_readable()): + if self.has_member("nsproxy") and not ( + self.nsproxy and self.nsproxy.is_readable() + ): return False - if not (self.real_parent and self.real_parent.is_readable()): + if self.has_member("real_parent") and not ( + self.real_parent and self.real_parent.is_readable() + ): return False - if self.active_mm and not self.active_mm.is_readable(): + if ( + self.has_member("active_mm") + and self.active_mm + and not self.active_mm.is_readable() + ): return False if self.mm: From 7fc2af5b4ecf4b1ced5c71357d164981b05ed309 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Jan 2025 11:39:48 +1100 Subject: [PATCH 071/137] linux: Add an additional quick check before validating pointer readability --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 4a1a263dd..e2c9454f6 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -313,7 +313,7 @@ class task_struct(generic.GenericIntelProcess): if not layer.is_valid(self.vol.offset, self.vol.size): return False - if self.pid < 0: + if self.pid < 0 or self.tgid < 0: return False if self.has_member("signal") and not ( From d21fdb4211c800404acd863b71b8302022d090e4 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 13 Jan 2025 16:52:45 +1100 Subject: [PATCH 072/137] Linux: pagecache: Harden Page Cache API to consistently yield valid entries --- .../framework/plugins/linux/pagecache.py | 14 ++++++-- .../framework/symbols/linux/__init__.py | 9 +++--- .../symbols/linux/extensions/__init__.py | 32 +++++++++++++------ 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 382268515..430190970 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -104,7 +104,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -253,6 +253,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): if not root_inode.is_valid(): continue + if not (root_inode.i_mapping and root_inode.i_mapping.is_readable()): + # Retrieving data from the page cache requires a valid address space + continue + # Inode already processed? if root_inode_ptr in seen_inodes: continue @@ -284,6 +288,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): if not file_inode.is_valid(): continue + if not (file_inode.i_mapping and file_inode.i_mapping.is_readable()): + # Retrieving data from the page cache requires a valid address space + continue + # Inode already processed? if file_inode_ptr in seen_inodes: continue @@ -316,10 +324,12 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): if self.config["find"]: if inode_in.path == self.config["find"]: inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) break # Only the first match else: inode_out = inode_in.to_user(vmlinux_layer) + yield (0, astuple(inode_out)) def generate_timeline(self): @@ -389,7 +399,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 5aa27b964..3265dfd37 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -838,11 +838,12 @@ class PageCache: Yields: Page objects """ - + layer = self.vmlinux.context.layers[self.vmlinux.layer_name] for page_addr in self._idstorage.get_entries(self._page_cache.i_pages): if not page_addr: continue - page = self.vmlinux.object("page", offset=page_addr, absolute=True) - if page: - yield page + if not layer.is_valid(page_addr): + continue + + yield self.vmlinux.object("page", offset=page_addr, absolute=True) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index df1c00e3d..187b1e280 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2489,7 +2489,12 @@ class inode(objects.StructType): """ if not self.i_size: return - elif not (self.i_mapping and self.i_mapping.nrpages > 0): + + if not ( + self.i_mapping + and self.i_mapping.is_readable() + and self.i_mapping.nrpages > 0 + ): return page_cache = linux.PageCache( @@ -2497,19 +2502,21 @@ class inode(objects.StructType): kernel_module_name="kernel", page_cache=self.i_mapping.dereference(), ) + yield from page_cache.get_cached_pages() - def get_contents(self): + def get_contents(self) -> Iterable[Tuple[int, bytes]]: """Get the inode cached pages from the page cache Yields: page_index (int): The page index in the Tree. File offset is page_index * PAGE_SIZE. - page_content (str): The page content + page_content (bytes): The page content """ for page_obj in self.get_pages(): page_index = int(page_obj.index) page_content = page_obj.get_content() - yield page_index, page_content + if page_content: + yield page_index, page_content class address_space(objects.StructType): @@ -2625,7 +2632,7 @@ class page(objects.StructType): return page_paddr - def get_content(self) -> Union[str, None]: + def get_content(self) -> Union[bytes, None]: """Returns the page content Returns: @@ -2641,8 +2648,13 @@ class page(objects.StructType): if not page_paddr: return None - page_data = physical_layer.read(page_paddr, vmlinux_layer.page_size) - return page_data + if not physical_layer.is_valid(page_paddr, length=vmlinux_layer.page_size): + vollog.debug( + "Unable to read page 0x%x content at 0x%x", self.vol.offset, page_paddr + ) + return None + + return physical_layer.read(page_paddr, vmlinux_layer.page_size) def get_flags_list(self) -> List[str]: """Returns a list of page flags @@ -2755,17 +2767,17 @@ class IDR(objects.StructType): class rb_root(objects.StructType): - def _walk_nodes(self, root_node) -> Iterator[int]: + def _walk_nodes(self, root_node: int) -> Iterator[int]: """Traverses the Red-Black tree from the root node and yields a pointer to each node in this tree. Args: - root_node: A Red-Black tree node from which to start descending + root_node: A Red-Black tree node pointer from which to start descending Yields: A pointer to every node descending from the specified root node """ - if not root_node: + if not (root_node and root_node.is_readable()): return yield root_node From 529b67fc96d7e975bdff83edd7096c4c0cd8db80 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Tue, 14 Jan 2025 12:31:25 +1100 Subject: [PATCH 073/137] Linux: pagecache: Fix issue reported in #1527 --- volatility3/framework/plugins/linux/pagecache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 430190970..9aff0e4f9 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -469,6 +469,8 @@ class InodePages(plugins.PluginInterface): inode_size, page_idx, ) + continue + f.seek(current_fp) f.write(page_bytes) From 1a84f96c70060bc09aab3c1b3348d980f8b9bc0e Mon Sep 17 00:00:00 2001 From: Kerry Goodwine Date: Thu, 9 Jan 2025 15:49:24 -0500 Subject: [PATCH 074/137] Actions: Add new workflow for generating windows EXEs with pyinstaller --- .github/workflows/build-pyinstaller.yml | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/build-pyinstaller.yml diff --git a/.github/workflows/build-pyinstaller.yml b/.github/workflows/build-pyinstaller.yml new file mode 100644 index 000000000..bcba95403 --- /dev/null +++ b/.github/workflows/build-pyinstaller.yml @@ -0,0 +1,50 @@ +name: build-pyinstaller +on: + push: + branches: + - stable + - develop + - 'release/**' + pull_request: + branches: + - stable + - 'release/**' + +jobs: + + exe: + runs-on: windows-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pyinstaller + + - name: Pyinstall executable + run: | + pyinstaller --clean -y vol.spec + pyinstaller --clean -y volshell.spec + + - name: Move files + run: | + mv dist/vol.exe vol.exe + mv dist/volshell.exe volshell.exe + + - name: Archive + uses: actions/upload-artifact@v4 + with: + name: volatility3-pyinstaller + path: | + vol.exe + volshell.exe + README.md + LICENSE.txt From 9f08af47b161579bf31f9d45c8b248c23861388a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 12:51:52 +1100 Subject: [PATCH 075/137] Linux: Add support for Intel 32bit with PAE --- volatility3/framework/automagic/linux.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f22cae012..542d26a8d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -71,6 +71,12 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): elif "init_level4_pgt" in table.symbols: layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" + elif ( + "pkmap_count" in table.symbols + and table.get_symbol("pkmap_count").type.count == 512 + ): + layer_class = intel.LinuxIntelPAE + dtb_symbol_name = "swapper_pg_dir" else: layer_class = intel.LinuxIntel dtb_symbol_name = "swapper_pg_dir" From 28c74f8c1b853df3680de14f6fdc22958516a9c2 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 13:23:39 +1100 Subject: [PATCH 076/137] Linux: Add support for Intel 32bit with PAE in early kernels, including versions 2.3.27 and 2.3.28. --- volatility3/framework/automagic/linux.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 542d26a8d..cb4f3cc64 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -71,10 +71,9 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): elif "init_level4_pgt" in table.symbols: layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" - elif ( - "pkmap_count" in table.symbols - and table.get_symbol("pkmap_count").type.count == 512 - ): + elif "pkmap_count" in table.symbols and table.get_symbol( + "pkmap_count" + ).type.count in (512, 2048): layer_class = intel.LinuxIntelPAE dtb_symbol_name = "swapper_pg_dir" else: From b27f98fed258b597e043a5c80304d8489da27b32 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 14:37:35 +1100 Subject: [PATCH 077/137] linux: pslist: fix task credentials rendering --- volatility3/framework/plugins/linux/pslist.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 37cf000fc..77b57e000 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -179,6 +179,10 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): file_output = "VMA start matching task start_code not found" return file_output + @staticmethod + def _format_cred(cred): + return renderers.NotAvailableValue() if cred is None else cred + def _generator( self, pid_filter: Callable[[Any], bool], @@ -212,16 +216,21 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): task_fields = self.get_task_fields(task, decorate_comm) + task_uid = self._format_cred(task_fields.uid) + task_gid = self._format_cred(task_fields.gid) + task_euid = self._format_cred(task_fields.euid) + task_egid = self._format_cred(task_fields.egid) + yield 0, ( format_hints.Hex(task_fields.offset), task_fields.user_pid, task_fields.user_tid, task_fields.user_ppid, task_fields.name, - task_fields.uid or renderers.NotAvailableValue(), - task_fields.gid or renderers.NotAvailableValue(), - task_fields.euid or renderers.NotAvailableValue(), - task_fields.egid or renderers.NotAvailableValue(), + task_uid, + task_gid, + task_euid, + task_egid, task_fields.creation_time or renderers.NotAvailableValue(), file_output, ) From 66084878627f636d0bebabbecc79842ac954d209 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 15 Jan 2025 15:08:57 +1100 Subject: [PATCH 078/137] Linux: pagecache: Fix issue with incosistent inode page caches --- volatility3/framework/plugins/linux/pagecache.py | 6 +++++- volatility3/framework/symbols/linux/__init__.py | 14 ++++++++++++-- .../framework/symbols/linux/extensions/__init__.py | 9 +++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 9aff0e4f9..b2766be8d 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -520,7 +520,11 @@ class InodePages(plugins.PluginInterface): page_mapping_addr = page_obj.mapping page_index = int(page_obj.index) page_file_offset = page_index * vmlinux_layer.page_size - dump_safe = page_file_offset < inode_size + dump_safe = ( + page_file_offset < inode_size + and page_mapping_addr + and page_mapping_addr.is_readable() + ) page_flags_list = page_obj.get_flags_list() page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) fields = ( diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 3265dfd37..08f69c326 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -3,6 +3,7 @@ # import math import contextlib +import logging from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union @@ -12,6 +13,8 @@ from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions +vollog = logging.getLogger(__name__) + class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): provides = {"type": "interface"} @@ -612,7 +615,7 @@ class IDStorage(ABC): raise NotImplementedError def nodep_to_node(self, nodep) -> interfaces.objects.ObjectInterface: - """Instanciates a tree node from its pointer + """Instantiates a tree node from its pointer Args: nodep: Pointer to the XArray/RadixTree node @@ -846,4 +849,11 @@ class PageCache: if not layer.is_valid(page_addr): continue - yield self.vmlinux.object("page", offset=page_addr, absolute=True) + page = self.vmlinux.object("page", offset=page_addr, absolute=True) + if not page.is_valid(): + vollog.error( + f"Invalid cached page at {page.vol.offset:#x}, aborting", + ) + break + + yield page diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 187b1e280..6ae022923 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2534,6 +2534,15 @@ class address_space(objects.StructType): class page(objects.StructType): + def is_valid(self) -> bool: + if self.mapping and not self.mapping.is_readable(): + return False + + if self.to_paddr() < 0: + return False + + return True + @functools.cached_property def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values From ce40659d8728f57f9b6e63709af4b7a17588a309 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:19:47 +1100 Subject: [PATCH 079/137] linux: radix_tree: Fix various issues, enhance early inconsistency detection, and improve compatibility with older kernel versions --- volatility3/framework/exceptions.py | 4 ++ .../framework/symbols/linux/__init__.py | 62 ++++++++++++------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index c44fb4f2e..41c67b88d 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -130,3 +130,7 @@ class OfflineException(VolatilityException): class RenderException(VolatilityException): """Thrown if there is an error during rendering""" + + +class LinuxPageCacheException(VolatilityException): + """Thrown if there is an error during Linux Page Cache processing""" diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 08f69c326..4c9934439 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -3,6 +3,7 @@ # import math import contextlib +import functools import logging from abc import ABC, abstractmethod from typing import Iterator, List, Tuple, Optional, Union @@ -662,7 +663,7 @@ class IDStorage(ABC): height = self.get_tree_height(root.vol.offset) nodep = self.get_head_node(root) - if not nodep: + if not (nodep and nodep.is_readable()): return # Keep the internal flag before untagging it @@ -697,7 +698,7 @@ class XArray(IDStorage): def get_node_height(self, nodep) -> int: node = self.nodep_to_node(nodep) - return (node.shift / self.CHUNK_SHIFT) + 1 + return (node.shift // self.CHUNK_SHIFT) + 1 def get_head_node(self, tree) -> int: return tree.xa_head @@ -720,6 +721,7 @@ class RadixTree(IDStorage): RADIX_TREE_INTERNAL_NODE = 1 RADIX_TREE_EXCEPTIONAL_ENTRY = 2 RADIX_TREE_ENTRY_MASK = 3 + RADIX_TREE_MAP_SHIFT = 6 # CONFIG_BASE_FULL # Dynamic values. These will be initialized later RADIX_TREE_INDEX_BITS = None @@ -756,43 +758,57 @@ class RadixTree(IDStorage): def get_tree_height(self, treep) -> int: with contextlib.suppress(exceptions.SymbolError): if self.vmlinux.get_type("radix_tree_root").has_member("height"): - # kernels < 4.7.10 + # kernels < 4.7 d0891265bbc988dc91ed8580b38eb3dac128581b radix_tree_root = self.vmlinux.object( "radix_tree_root", offset=treep, absolute=True ) return radix_tree_root.height - # kernels >= 4.7.10 + # kernels >= 4.7 return 0 + @functools.cached_property + def _max_height_array(self): + if self.vmlinux.has_symbol("height_to_maxindex"): + # 2.6.24 26fb1589cb0aaec3a0b4418c54f30c1a2b1781f6 <= Kernels < 4.7 d0891265bbc988dc91ed8580b38eb3dac128581b + return self.vmlinux.object_from_symbol("height_to_maxindex") + elif self.vmlinux.has_symbol("height_to_maxnodes"): + # 4.8 c78c66d1ddfdbd2353f3fcfeba0268524537b096 <= kernels < 4.20 8cf2f98411e3a0865026a1061af637161b16d32b + return self.vmlinux.object_from_symbol("height_to_maxnodes") + + return None + def _radix_tree_maxindex(self, node, height) -> int: """Return the maximum key which can be store into a radix tree with this height.""" - if not self.vmlinux.has_symbol("height_to_maxindex"): - # Kernels >= 4.7 - return (self.CHUNK_SIZE << node.shift) - 1 + if self._max_height_array: + # 2.6.24 <= kernels <= 4.20 See _max_height_array() + return self._max_height_array[height] else: - # Kernels < 4.7 - height_to_maxindex_array = self.vmlinux.object_from_symbol( - "height_to_maxindex" - ) - maxindex = height_to_maxindex_array[height] - return maxindex + # Kernels >= 4.20 + return (self.CHUNK_SIZE << node.shift) - 1 def get_node_height(self, nodep) -> int: node = self.nodep_to_node(nodep) if hasattr(node, "shift"): # 4.7 <= Kernels < 4.20 - return (node.shift / self.CHUNK_SHIFT) + 1 + height = (node.shift // self.CHUNK_SHIFT) + 1 elif hasattr(node, "path"): # 3.15 <= Kernels < 4.7 - return node.path & self.RADIX_TREE_HEIGHT_MASK + height = node.path & self.RADIX_TREE_HEIGHT_MASK elif hasattr(node, "height"): # Kernels < 3.15 - return node.height + height = node.height else: raise exceptions.VolatilityException("Cannot find radix-tree node height") + if self._max_height_array and not (0 <= height < self._max_height_array.count): + error_msg = f"Radix Tree node {node.vol.offset:#x} height {height} exceeds max height of {self._max_height_array.count}" + vollog.error(error_msg) + raise exceptions.LinuxPageCacheException(error_msg) + + return height + def get_head_node(self, tree) -> int: return tree.rnode @@ -805,14 +821,16 @@ class RadixTree(IDStorage): def untag_node(self, nodep) -> int: return nodep & (~self.RADIX_TREE_ENTRY_MASK) - def is_valid_node(self, nodep) -> bool: + def _is_exceptional_node(self, nodep) -> bool: # In kernels 4.20, exceptional nodes were removed and internal entries took their bitmask - if self.vmlinux.has_type("radix_tree_root"): - return ( - nodep & self.RADIX_TREE_ENTRY_MASK - ) != self.RADIX_TREE_EXCEPTIONAL_ENTRY + return ( + self.vmlinux.has_type("radix_tree_root") + and (nodep & self.RADIX_TREE_ENTRY_MASK) + == self.RADIX_TREE_EXCEPTIONAL_ENTRY + ) - return True + def is_valid_node(self, nodep) -> bool: + return not self._is_exceptional_node(nodep) class PageCache: From 3c92d7f7b7d3d7d9548275e960571d4ac37a6a21 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:22:46 +1100 Subject: [PATCH 080/137] linux: page_cache: enhance early inconsistency detection --- volatility3/framework/symbols/linux/__init__.py | 14 ++++++-------- .../framework/symbols/linux/extensions/__init__.py | 7 ++++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 4c9934439..a7e6ef405 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -861,17 +861,15 @@ class PageCache: """ layer = self.vmlinux.context.layers[self.vmlinux.layer_name] for page_addr in self._idstorage.get_entries(self._page_cache.i_pages): - if not page_addr: - continue - if not layer.is_valid(page_addr): - continue + error_msg = f"Invalid cached page address at {page_addr:#x}, aborting" + vollog.error(error_msg) + raise exceptions.LinuxPageCacheException(error_msg) page = self.vmlinux.object("page", offset=page_addr, absolute=True) if not page.is_valid(): - vollog.error( - f"Invalid cached page at {page.vol.offset:#x}, aborting", - ) - break + error_msg = f"Invalid cached page at {page_addr:#x}, aborting" + vollog.error(error_msg) + raise exceptions.LinuxPageCacheException(error_msg) yield page diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 6ae022923..997370bbb 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2513,6 +2513,11 @@ class inode(objects.StructType): page_content (bytes): The page content """ for page_obj in self.get_pages(): + if page_obj.mapping != self.i_mapping: + vollog.warning( + f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" + ) + continue page_index = int(page_obj.index) page_content = page_obj.get_content() if page_content: @@ -2524,7 +2529,7 @@ class address_space(objects.StructType): def i_pages(self): """Returns the appropriate member containing the page cache tree""" if self.has_member("i_pages"): - # Kernel >= 4.17 + # Kernel >= 4.17 b93b016313b3ba8003c3b8bb71f569af91f19fc7 return self.member("i_pages") elif self.has_member("page_tree"): # Kernel < 4.17 From c1497410b4721703ebbb9f30f2ab3db497a4438b Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:28:25 +1100 Subject: [PATCH 081/137] linux: page_cache plugin: lazy file initialization and avoid redundant inode page cache walk during dumps not showing output with dumping to file. --- .../framework/plugins/linux/pagecache.py | 89 ++++++++++++------- 1 file changed, 57 insertions(+), 32 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index b2766be8d..7ad0f5a8f 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -8,7 +8,7 @@ import datetime from dataclasses import dataclass, astuple from typing import List, Set, Type, Iterable -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins from volatility3.framework.configuration import requirements @@ -453,16 +453,18 @@ class InodePages(plugins.PluginInterface): # created, saving both disk space and I/O time. # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. + inode_size = inode.i_size try: - with open_method(filename) as f: - inode_size = inode.i_size - f.truncate(inode_size) - + file_initialized = False + with open_method(filename) as file_obj: for page_idx, page_content in inode.get_contents(): current_fp = page_idx * vmlinux_layer.page_size max_length = inode_size - current_fp - page_bytes = page_content[:max_length] - if current_fp + len(page_bytes) > inode_size: + page_bytes_len = min(max_length, len(page_content)) + if ( + current_fp >= inode_size + or current_fp + page_bytes_len > inode_size + ): vollog.error( "Page out of file bounds: inode 0x%x, inode size %d, page index %d", inode.vol.offset, @@ -470,10 +472,20 @@ class InodePages(plugins.PluginInterface): page_idx, ) continue + page_bytes = page_content[:page_bytes_len] - f.seek(current_fp) - f.write(page_bytes) + if not file_initialized: + # Lazy initialization to avoid truncating the file until we are + # certain there is something to write + file_obj.truncate(inode_size) + file_initialized = True + file_obj.seek(current_fp) + file_obj.write(page_bytes) + except exceptions.LinuxPageCacheException: + vollog.error( + f"Error dumping cached pages for inode at {inode.vol.offset:#x}" + ) except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) @@ -514,31 +526,44 @@ class InodePages(plugins.PluginInterface): return None inode_size = inode.i_size - for page_obj in inode.get_pages(): - page_vaddr = page_obj.vol.offset - page_paddr = page_obj.to_paddr() - page_mapping_addr = page_obj.mapping - page_index = int(page_obj.index) - page_file_offset = page_index * vmlinux_layer.page_size - dump_safe = ( - page_file_offset < inode_size - and page_mapping_addr - and page_mapping_addr.is_readable() - ) - page_flags_list = page_obj.get_flags_list() - page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) - fields = ( - page_vaddr, - page_paddr, - page_mapping_addr, - page_index, - dump_safe, - page_flags, - ) + if not self.config["dump"]: + try: + for page_obj in inode.get_pages(): + if page_obj.mapping != inode.i_mapping: + vollog.warning( + f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" + ) + continue + page_vaddr = page_obj.vol.offset + page_paddr = page_obj.to_paddr() + page_mapping_addr = page_obj.mapping + page_index = int(page_obj.index) + page_file_offset = page_index * vmlinux_layer.page_size + dump_safe = ( + page_file_offset < inode_size + and page_mapping_addr + and page_mapping_addr.is_readable() + ) + page_flags_list = page_obj.get_flags_list() + page_flags = ",".join( + [x.replace("PG_", "") for x in page_flags_list] + ) + fields = ( + page_vaddr, + page_paddr, + page_mapping_addr, + page_index, + dump_safe, + page_flags, + ) - yield 0, fields + yield 0, fields + except exceptions.LinuxPageCacheException: + vollog.warning( + f"Page cache for inode at {inode.vol.offset:#x} is corrupt" + ) - if self.config["dump"]: + else: open_method = self.open inode_address = inode.vol.offset filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") From 5502a54198fc7617bb22e2e96519114c59704ab0 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 19:38:41 +1100 Subject: [PATCH 082/137] linux: page_cache plugin: Refactor to make _generator more readable --- .../framework/plugins/linux/pagecache.py | 83 ++++++++++--------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 7ad0f5a8f..39ed60486 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type, Iterable +from typing import List, Set, Type, Iterable, Tuple from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.renderers import format_hints @@ -489,6 +489,44 @@ class InodePages(plugins.PluginInterface): except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) + def _generate_inode_fields( + self, + inode: interfaces.objects.ObjectInterface, + vmlinux_layer: interfaces.layers.TranslationLayerInterface, + ) -> Iterable[Tuple[int, int, int, int, bool, str]]: + inode_size = inode.i_size + try: + for page_obj in inode.get_pages(): + if page_obj.mapping != inode.i_mapping: + vollog.warning( + f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" + ) + continue + page_vaddr = page_obj.vol.offset + page_paddr = page_obj.to_paddr() + page_mapping_addr = page_obj.mapping + page_index = int(page_obj.index) + page_file_offset = page_index * vmlinux_layer.page_size + dump_safe = ( + page_file_offset < inode_size + and page_mapping_addr + and page_mapping_addr.is_readable() + ) + page_flags_list = page_obj.get_flags_list() + page_flags = ",".join([x.replace("PG_", "") for x in page_flags_list]) + fields = ( + page_vaddr, + page_paddr, + page_mapping_addr, + page_index, + dump_safe, + page_flags, + ) + + yield 0, fields + except exceptions.LinuxPageCacheException: + vollog.warning(f"Page cache for inode at {inode.vol.offset:#x} is corrupt") + def _generator(self): vmlinux_module_name = self.config["kernel"] vmlinux = self.context.modules[vmlinux_module_name] @@ -510,7 +548,6 @@ class InodePages(plugins.PluginInterface): else: vollog.error("Unable to find inode with path %s", self.config["find"]) return None - elif self.config["inode"]: inode = vmlinux.object("inode", self.config["inode"], absolute=True) else: @@ -525,45 +562,7 @@ class InodePages(plugins.PluginInterface): vollog.error("The inode is not a regular file") return None - inode_size = inode.i_size - if not self.config["dump"]: - try: - for page_obj in inode.get_pages(): - if page_obj.mapping != inode.i_mapping: - vollog.warning( - f"Cached page at {page_obj.vol.offset:#x} has a mismatched address space with the inode. Skipping page" - ) - continue - page_vaddr = page_obj.vol.offset - page_paddr = page_obj.to_paddr() - page_mapping_addr = page_obj.mapping - page_index = int(page_obj.index) - page_file_offset = page_index * vmlinux_layer.page_size - dump_safe = ( - page_file_offset < inode_size - and page_mapping_addr - and page_mapping_addr.is_readable() - ) - page_flags_list = page_obj.get_flags_list() - page_flags = ",".join( - [x.replace("PG_", "") for x in page_flags_list] - ) - fields = ( - page_vaddr, - page_paddr, - page_mapping_addr, - page_index, - dump_safe, - page_flags, - ) - - yield 0, fields - except exceptions.LinuxPageCacheException: - vollog.warning( - f"Page cache for inode at {inode.vol.offset:#x} is corrupt" - ) - - else: + if self.config["dump"]: open_method = self.open inode_address = inode.vol.offset filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") @@ -571,6 +570,8 @@ class InodePages(plugins.PluginInterface): self.write_inode_content_to_file( inode, filename, open_method, vmlinux_layer ) + else: + yield from self._generate_inode_fields(inode, vmlinux_layer) def run(self): headers = [ From 9944fcc61f179a0d949266a90d3e0e290d5017c6 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 21:15:00 +1100 Subject: [PATCH 083/137] linux: page_cache test case: Since the --dump no longer generate output, we test both modes, listing and dumping. --- test/test_volatility.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index bb7c9a851..e25e8278b 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -714,7 +714,7 @@ def test_linux_page_cache_inodepages(image, volatility, python): image, volatility, python, - pluginargs=["--inode", inode_address, "--dump"], + pluginargs=["--inode", inode_address], ) assert rc == 0 @@ -725,6 +725,14 @@ def test_linux_page_cache_inodepages(image, volatility, python): rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", out, ) + + rc, out, _err = runvol_plugin( + "linux.pagecache.InodePages", + image, + volatility, + python, + pluginargs=["--inode", inode_address, "--dump"], + ) assert os.path.exists(inode_dump_filename) with open(inode_dump_filename, "rb") as fp: inode_contents = fp.read() From d8254b63735388b9ef6be27009474ccea5726650 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 16 Jan 2025 21:39:37 +1100 Subject: [PATCH 084/137] linux: page_cache test case: Improve test --- test/test_volatility.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/test/test_volatility.py b/test/test_volatility.py index e25e8278b..8676d1f3e 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -708,24 +708,25 @@ def test_linux_page_cache_inodepages(image, volatility, python): inode_address = hex(0x88001AB5C270) inode_dump_filename = f"inode_{inode_address}.dmp" + + rc, out, _err = runvol_plugin( + "linux.pagecache.InodePages", + image, + volatility, + python, + pluginargs=["--inode", inode_address], + ) + + assert rc == 0 + assert out.count(b"\n") > 4 + + # PageVAddr PagePAddr MappingAddr .. DumpSafe + assert re.search( + rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", + out, + ) + try: - rc, out, _err = runvol_plugin( - "linux.pagecache.InodePages", - image, - volatility, - python, - pluginargs=["--inode", inode_address], - ) - - assert rc == 0 - assert out.count(b"\n") > 4 - - # PageVAddr PagePAddr MappingAddr .. DumpSafe - assert re.search( - rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", - out, - ) - rc, out, _err = runvol_plugin( "linux.pagecache.InodePages", image, @@ -733,6 +734,10 @@ def test_linux_page_cache_inodepages(image, volatility, python): python, pluginargs=["--inode", inode_address, "--dump"], ) + + assert rc == 0 + assert out.count(b"\n") >= 4 + assert os.path.exists(inode_dump_filename) with open(inode_dump_filename, "rb") as fp: inode_contents = fp.read() From 749a0f9c656d8da0f5087faf25f225a6097ac8b4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 16 Jan 2025 12:06:22 +0000 Subject: [PATCH 085/137] Core: Fix up ISFinfo looking for fastjsonschema --- volatility3/framework/plugins/isfinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 34b0a5653..1c2ac52e9 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -97,7 +97,7 @@ class IsfInfo(plugins.PluginInterface): if filter_item in isf_file: filtered_list.append(isf_file) - if find_spec("fastjsonschema") and self.config["validate"]: + if find_spec("jsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False" From b447bfa81c36e91c3cf30bdc432e6eba48afbc53 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:24:38 +0100 Subject: [PATCH 086/137] remove module tainting proxies --- .../framework/plugins/linux/modxview.py | 16 ++++++++++-- .../symbols/linux/extensions/__init__.py | 25 ------------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 3655200e8..69c6ac8bb 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -140,9 +140,21 @@ class Modxview(interfaces.plugins.PluginInterface): seen_addresses.add(module.vol.offset) if self.config.get("plain_taints"): - taints = module.get_taints_as_plain_string() + taints = tainting.Tainting.get_taints_as_plain_string( + self.context, + kernel_name, + module.taints, + True, + ) else: - taints = ",".join(module.get_taints_parsed()) + taints = ",".join( + tainting.Tainting.get_taints_parsed( + self.context, + kernel_name, + module.taints, + True, + ) + ) yield ( 0, diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index ac2f87df0..289d6c0a4 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -19,7 +19,6 @@ from volatility3.framework.layers import linear, intel from volatility3.framework.objects import utility from volatility3.framework.symbols import generic, linux, intermed from volatility3.framework.symbols.linux.extensions import elf -from volatility3.framework.symbols.linux.utilities import tainting vollog = logging.getLogger(__name__) @@ -279,30 +278,6 @@ class module(generic.GenericIntelProcess): return None - def get_taints_as_plain_string(self) -> str: - """Convert the module's taints value to a 1-1 character mapping. - Convenient wrapper around framework's Tainting capabilities. - - Returns: - The raw taints string. - """ - return tainting.Tainting( - self._context, - linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, - ).get_taints_as_plain_string(self.taints, True) - - def get_taints_parsed(self) -> List[str]: - """Convert the module's taints string to a 1-1 descriptor mapping. - Convenient wrapper around framework's Tainting capabilities. - - Returns: - A comprehensive (user-friendly) taint descriptor list. - """ - return tainting.Tainting( - self._context, - linux.LinuxUtilities.get_module_from_volobj_type(self._context, self).name, - ).get_taints_parsed(self.taints, True) - @property def section_symtab(self): if self.has_member("kallsyms"): From 94704c6674d7f5fb9d57698faa0d9ed943c6158c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:26:57 +0100 Subject: [PATCH 087/137] 2.16.0 -> 2.17.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 832b2a5ba..3d68ab810 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 18 # Number of changes that only add to the interface +VERSION_MINOR = 17 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From cd8690a8059836f7e216c211c4397924ae311c84 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 16 Jan 2025 16:27:22 +0100 Subject: [PATCH 088/137] require framework version 2.17.0 --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 69c6ac8bb..3c2c5f05e 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -19,7 +19,7 @@ class Modxview(interfaces.plugins.PluginInterface): spot modules presence and taints.""" _version = (1, 0, 0) - _required_framework_version = (2, 18, 0) + _required_framework_version = (2, 17, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From adf81bc74a388d6ff5bffabe588bc5ca72147506 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 16 Jan 2025 19:59:31 +0000 Subject: [PATCH 089/137] Update copyright dates --- README.md | 2 +- doc/source/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cc33d3cc4..b74bdab0b 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ The latest generated copy of the documentation can be found at: Date: Fri, 17 Jan 2025 16:03:47 +0000 Subject: [PATCH 090/137] Core: Correct version dependencies to avoid conflicts Fixes #1546 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86e3921d2..542a1480a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ cloud = [ dev = [ "volatility3[full,cloud]", "jsonschema>=4.23.0,<5", - "pyinstaller>=6.11.0,<7", + "pyinstaller>=6.5.0,<7", "pyinstaller-hooks-contrib>=2024.9", "types-jsonschema>=4.23.0,<5", ] @@ -48,7 +48,7 @@ test = [ docs = [ "volatility3[dev]", - "sphinx>=8.0.0,<7", + "sphinx>=8.0.0,<9", "sphinx-autodoc-typehints>=2.5.0,<3", "sphinx-rtd-theme>=3.0.1,<4", ] From c4430cda8d6b13b0d69a787fe32e8158ae471c3a Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 17 Jan 2025 16:09:35 +0000 Subject: [PATCH 091/137] Core: Try to maintain python-3.8 support for the documentation --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 542a1480a..8944bb058 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ test = [ docs = [ "volatility3[dev]", "sphinx>=8.0.0,<9", - "sphinx-autodoc-typehints>=2.5.0,<3", + "sphinx-autodoc-typehints>=2.0.0,<3", "sphinx-rtd-theme>=3.0.1,<4", ] From 13a8c53f7b64bc7180b665e363b2a3f0348e8b04 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Fri, 17 Jan 2025 16:13:35 +0000 Subject: [PATCH 092/137] Core: There was no clear reason to stop supporting older versions of sphinx --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8944bb058..3f16eeece 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ test = [ docs = [ "volatility3[dev]", - "sphinx>=8.0.0,<9", + "sphinx>=4.0.0,<9", "sphinx-autodoc-typehints>=2.0.0,<3", "sphinx-rtd-theme>=3.0.1,<4", ] From cc9486cf03f6f8b8035069f02702ce30a589cb7e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 02:19:11 +0100 Subject: [PATCH 093/137] pre-process module triaging to improve readability --- .../framework/plugins/linux/modxview.py | 71 ++++++++++--------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 3c2c5f05e..125c1cc33 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -82,7 +82,8 @@ class Modxview(interfaces.plugins.PluginInterface): kernel_name: str, run_hidden_modules: bool = True, ) -> Dict[str, List[extensions.module]]: - """Run module scanning plugins and aggregate the results. + """Run module scanning plugins and aggregate the results. It is designed + to not operate any inter-plugin results triage. Args: run_hidden_modules: specify if the hidden_modules plugin should be run @@ -128,46 +129,50 @@ class Modxview(interfaces.plugins.PluginInterface): def _generator(self): kernel_name = self.config["kernel"] run_results = self.run_modules_scanners(self.context, kernel_name) - modules_offsets = {} - for key in ["lsmod", "check_modules", "hidden_modules"]: - modules_offsets[key] = set(module.vol.offset for module in run_results[key]) + aggregated_modules = {} + # We want to be explicit on the plugins results we are interested in + for plugin_name in ["lsmod", "check_modules", "hidden_modules"]: + # Iterate over each recovered module + for module in run_results[plugin_name]: + # Use offsets as unique keys, whether a module + # appears in many plugin runs or not + if aggregated_modules.get(module.vol.offset): + # Append the plugin to the list of originating plugins + aggregated_modules[module.vol.offset][1].append(plugin_name) + else: + aggregated_modules[module.vol.offset] = (module, [plugin_name]) - seen_addresses = set() - for modules_list in run_results.values(): - for module in modules_list: - if module.vol.offset in seen_addresses: - continue - seen_addresses.add(module.vol.offset) - - if self.config.get("plain_taints"): - taints = tainting.Tainting.get_taints_as_plain_string( + for module_offset, (module, originating_plugins) in aggregated_modules.items(): + # Tainting parsing capabilities applied to the module + if self.config.get("plain_taints"): + taints = tainting.Tainting.get_taints_as_plain_string( + self.context, + kernel_name, + module.taints, + True, + ) + else: + taints = ",".join( + tainting.Tainting.get_taints_parsed( self.context, kernel_name, module.taints, True, ) - else: - taints = ",".join( - tainting.Tainting.get_taints_parsed( - self.context, - kernel_name, - module.taints, - True, - ) - ) - - yield ( - 0, - ( - module.get_name() or NotAvailableValue(), - format_hints.Hex(module.vol.offset), - module.vol.offset in modules_offsets["lsmod"], - module.vol.offset in modules_offsets["check_modules"], - module.vol.offset in modules_offsets["hidden_modules"], - taints or NotAvailableValue(), - ), ) + yield ( + 0, + ( + module.get_name() or NotAvailableValue(), + format_hints.Hex(module_offset), + "lsmod" in originating_plugins, + "check_modules" in originating_plugins, + "hidden_modules" in originating_plugins, + taints or NotAvailableValue(), + ), + ) + def run(self): columns = [ ("Name", str), From 3b679cbafbb50a2c986a63efd223cf9088bbc330 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:26:43 +0100 Subject: [PATCH 094/137] explicit None check --- volatility3/framework/plugins/linux/modxview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/modxview.py b/volatility3/framework/plugins/linux/modxview.py index 125c1cc33..c74bf28e8 100644 --- a/volatility3/framework/plugins/linux/modxview.py +++ b/volatility3/framework/plugins/linux/modxview.py @@ -136,7 +136,7 @@ class Modxview(interfaces.plugins.PluginInterface): for module in run_results[plugin_name]: # Use offsets as unique keys, whether a module # appears in many plugin runs or not - if aggregated_modules.get(module.vol.offset): + if aggregated_modules.get(module.vol.offset, None) is not None: # Append the plugin to the list of originating plugins aggregated_modules[module.vol.offset][1].append(plugin_name) else: From bb6556dbc0145682866d56bb2608b5b841e381e8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:37:52 +0100 Subject: [PATCH 095/137] correct arguments for pre_4_10_rc1 --- volatility3/framework/symbols/linux/utilities/tainting.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 552f51b98..14b69d3d6 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -117,9 +117,7 @@ class Tainting(interfaces.configuration.VersionableInterface): return cls._module_flags_taint_post_4_10_rc1( context, kernel_module_name, taints, is_module ) - return cls._module_flags_taint_pre_4_10_rc1( - context, kernel_module_name, taints, is_module - ) + return cls._module_flags_taint_pre_4_10_rc1(taints, is_module) @classmethod def get_taints_parsed( From 0b82f731375583076abdfabd332ce067612d69f5 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:45:30 +0100 Subject: [PATCH 096/137] functools caching and doc. --- .../framework/symbols/linux/utilities/tainting.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index 14b69d3d6..c1136436e 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -1,3 +1,5 @@ +import functools + from volatility3 import framework from volatility3.framework import interfaces from volatility3.framework.constants import linux as linux_constants @@ -18,11 +20,18 @@ class Tainting(interfaces.configuration.VersionableInterface): framework.require_interface_version(*_required_framework_version) @classmethod + @functools.lru_cache def _get_kernel_taint_flags_list( cls, context: interfaces.context.ContextInterface, kernel_module_name: str, ) -> Optional[List[interfaces.objects.ObjectInterface]]: + """Determine whether the kernel embeds taint flags definition + in-memory or not. + + Returns: + A list of "taint_flag" kernel objects if taint_flags symbok exists + """ kernel = context.modules[kernel_module_name] if kernel.has_symbol("taint_flags"): return list(kernel.object_from_symbol("taint_flags")) From 8095924e8a926990f6002f16d2c7259c5c750980 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 13:46:13 +0100 Subject: [PATCH 097/137] typo --- volatility3/framework/symbols/linux/utilities/tainting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/utilities/tainting.py b/volatility3/framework/symbols/linux/utilities/tainting.py index c1136436e..2360401d5 100644 --- a/volatility3/framework/symbols/linux/utilities/tainting.py +++ b/volatility3/framework/symbols/linux/utilities/tainting.py @@ -30,7 +30,7 @@ class Tainting(interfaces.configuration.VersionableInterface): in-memory or not. Returns: - A list of "taint_flag" kernel objects if taint_flags symbok exists + A list of "taint_flag" kernel objects if taint_flags symbol exists """ kernel = context.modules[kernel_module_name] if kernel.has_symbol("taint_flags"): From 2fe8ee5983bd5faf8a89db5712512aa411329dbc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 18 Jan 2025 13:18:53 +0000 Subject: [PATCH 098/137] Layers: Update LeechCore RawIO with better error handling for readlines Fixes #1419 --- volatility3/framework/layers/leechcore.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index eeede1673..06c359203 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -129,6 +129,8 @@ if HAS_LEECHCORE: def readline(self, __size: Optional[int] = ...) -> bytes: data = b"" + if not __size: + __size = 0 while __size > self._chunk_size or __size < 0: data += self.read(self._chunk_size) index = data.find(b"\n") From 0849c163a1c517fa8595f9cc7610a737d1904fc2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:48:37 +0100 Subject: [PATCH 099/137] appropriate symbols type hinting --- volatility3/framework/contexts/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index f527544c0..17a91e827 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -337,7 +337,7 @@ class Module(interfaces.context.ModuleInterface): ) @property - def symbols(self): + def symbols(self) -> Iterable[str]: return self.context.symbol_space[self.symbol_table_name].symbols get_symbol = get_module_wrapper("get_symbol") From d46cb3328d07ae2216045ba3fc33679c2ab13fbc Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:48:46 +0100 Subject: [PATCH 100/137] appropriate symbols type hinting --- volatility3/framework/interfaces/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index a87e0f1e8..2b95a18ad 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -303,8 +303,8 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface): """Determines whether an enumeration is present in the module's symbol table.""" @abstractmethod - def symbols(self) -> List: - """Lists the symbols contained in the symbol table for this module""" + def symbols(self) -> Iterable[str]: + """Returns an iterable of the symbols contained in the symbol table for this module""" @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: From fb93d2333b8d3854d348548decc76ac67f358699 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:50:14 +0100 Subject: [PATCH 101/137] improve comments --- volatility3/framework/interfaces/symbols.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index b8712e38d..c0bebe1e2 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -122,7 +122,7 @@ class BaseSymbolTableInterface: @property def symbols(self) -> Iterable[str]: - """Returns an iterator of the Symbol names.""" + """Returns an iterable of the available symbol names.""" raise NotImplementedError( "Abstract property symbols not implemented by subclass." ) @@ -131,7 +131,7 @@ class BaseSymbolTableInterface: @property def types(self) -> Iterable[str]: - """Returns an iterator of the Symbol type names.""" + """Returns an iterable of the available symbol type names.""" raise NotImplementedError( "Abstract property types not implemented by subclass." ) @@ -149,7 +149,7 @@ class BaseSymbolTableInterface: @property def enumerations(self) -> Iterable[Any]: - """Returns an iterator of the Enumeration names.""" + """Returns an iterable of the available enumerations names.""" raise NotImplementedError( "Abstract property enumerations not implemented by subclass." ) @@ -366,6 +366,7 @@ class NativeTableInterface(BaseSymbolTableInterface): @property def symbols(self) -> Iterable[str]: + """Returns an iterable of the available symbol names.""" return [] def get_enumeration(self, name: str) -> objects.Template: From 1e9551b0530be824ab8d9a40db57cbd813d48136 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:50:28 +0100 Subject: [PATCH 102/137] types base class and comments improvements --- volatility3/framework/interfaces/symbols.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index c0bebe1e2..752d288f7 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -375,7 +375,13 @@ class NativeTableInterface(BaseSymbolTableInterface): ) @property - def enumerations(self) -> Iterable[str]: + def enumerations(self) -> Iterable[Any]: + """Returns an iterable of the available enumerations.""" + return [] + + @property + def types(self) -> Iterable[str]: + """Returns an iterable of the available symbol type names.""" return [] From aa99410dd2c50ee556293db959c469739c882684 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:51:31 +0100 Subject: [PATCH 103/137] prefer KeysView iterable to lists --- volatility3/framework/symbols/intermed.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 6802af7d6..0a30148aa 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -411,18 +411,23 @@ class Version1Format(ISFormatTable): @property def symbols(self) -> Iterable[str]: - """Returns an iterator of the symbol names.""" - return list(self._json_object.get("symbols", {})) + """Returns an iterable (KeysView) of the available symbol names.""" + return self._json_object.get("symbols", {}).keys() @property - def enumerations(self) -> Iterable[str]: - """Returns an iterator of the available enumerations.""" - return list(self._json_object.get("enums", {})) + def enumerations(self) -> Iterable[Any]: + """Returns an iterable (KeysView) of the available enumerations.""" + return self._json_object.get("enums", {}).keys() @property - def types(self) -> Iterable[str]: - """Returns an iterator of the symbol type names.""" - return list(self._json_object.get("user_types", {})) + list(self.natives.types) + def types(self): + """Returns an iterable (KeysView) of the available symbol type names.""" + # self.natives.types (set) is generally very small compared to user_types, + # so the dict conversion overhead can be neglected + return { + **self._json_object.get("user_types", {}), + **dict.fromkeys(self.natives.types), + }.keys() def get_type_class(self, name: str) -> Type[interfaces.objects.ObjectInterface]: return self._overrides.get(name, objects.AggregateType) From 3a2933155b6f92a8585666f611cd2069424ce5a9 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:51:52 +0100 Subject: [PATCH 104/137] improve comments --- volatility3/framework/symbols/native.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/native.py b/volatility3/framework/symbols/native.py index 7c3e1b312..61417532e 100644 --- a/volatility3/framework/symbols/native.py +++ b/volatility3/framework/symbols/native.py @@ -30,7 +30,7 @@ class NativeTable(interfaces.symbols.NativeTableInterface): @property def types(self) -> Iterable[str]: - """Returns an iterator of the symbol type names.""" + """Returns an iterable (set) of the available symbol type names.""" return self._types def get_type(self, type_name: str) -> interfaces.objects.Template: From ba09db6952d37590f62a647265c2bb5bb903ec3c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 14:54:04 +0100 Subject: [PATCH 105/137] improve comments --- volatility3/framework/interfaces/symbols.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index 752d288f7..2d142de9a 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -149,7 +149,7 @@ class BaseSymbolTableInterface: @property def enumerations(self) -> Iterable[Any]: - """Returns an iterable of the available enumerations names.""" + """Returns an iterable of the available enumerations.""" raise NotImplementedError( "Abstract property enumerations not implemented by subclass." ) From 4b3d93b0f0637c7d41acc545f397e3522a913978 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:00:03 +0100 Subject: [PATCH 106/137] 2.17.0 -> 2.18.0 bump --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 3d68ab810..832b2a5ba 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 17 # Number of changes that only add to the interface +VERSION_MINOR = 18 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From c5628c5d79496ae051942598bab08c19d3632a18 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:48:44 +0100 Subject: [PATCH 107/137] revert the mistakenly removed types type hinting --- volatility3/framework/symbols/intermed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 0a30148aa..9ece69d8b 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -420,7 +420,7 @@ class Version1Format(ISFormatTable): return self._json_object.get("enums", {}).keys() @property - def types(self): + def types(self) -> Iterable[str]: """Returns an iterable (KeysView) of the available symbol type names.""" # self.natives.types (set) is generally very small compared to user_types, # so the dict conversion overhead can be neglected From 0ee016e65554539318705a5b7c292864fcc2f436 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:50:28 +0100 Subject: [PATCH 108/137] 2.17.0 -> 2.17.1 bump --- volatility3/framework/constants/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 832b2a5ba..041439909 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 18 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_MINOR = 17 # Number of changes that only add to the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From c2ef3c2fe575f2c3ea49541b7e1207e7d93884f1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 18 Jan 2025 15:59:51 +0100 Subject: [PATCH 109/137] add fixme about merge operator --- volatility3/framework/symbols/intermed.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 9ece69d8b..cb0b67969 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -422,8 +422,12 @@ class Version1Format(ISFormatTable): @property def types(self) -> Iterable[str]: """Returns an iterable (KeysView) of the available symbol type names.""" - # self.natives.types (set) is generally very small compared to user_types, - # so the dict conversion overhead can be neglected + # We use ** instead of + # `set(self._json_object.get("user_types", {}).keys()).union(self.natives.types)` + # because converting user_types dict to a set is costly. + # It is more efficient to convert the (very small) self.natives.types set to a dict. + # FIXME: On Python3.8 support drop, merge the two dicts using the merge operator: + # (self._json_object.get("user_types", {}) | dict.fromkeys(self.natives.types)).keys() return { **self._json_object.get("user_types", {}), **dict.fromkeys(self.natives.types), From 726fbe6ccec8480c1baa74fadcc7fc4e87377bcd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 13:01:14 +0100 Subject: [PATCH 110/137] 2.17.1 -> 2.18.0 bump --- volatility3/framework/constants/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index 041439909..832b2a5ba 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,7 +1,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 17 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_MINOR = 18 # Number of changes that only add to the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( From 08830fee05c30adfc5ea4e997ce317f6f6927033 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:06:42 +0100 Subject: [PATCH 111/137] use architectures.LINUX_ARCHS --- volatility3/framework/plugins/linux/pagecache.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 382268515..408a9b98a 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -8,6 +8,7 @@ import datetime from dataclasses import dataclass, astuple from typing import List, Set, Type, Iterable +from volatility3.framework.constants import architectures from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins @@ -112,7 +113,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=["Intel32", "Intel64"], + architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 2, 0) @@ -397,7 +398,7 @@ class InodePages(plugins.PluginInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=["Intel32", "Intel64"], + architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( name="files", plugin=Files, version=(1, 0, 0) From d325e1ca176b55f68de03f462ca855631736ef6c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:08:27 +0100 Subject: [PATCH 112/137] add inode_size and format_symlink to Inode* dataclasses --- volatility3/framework/plugins/linux/pagecache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 408a9b98a..c86664f3d 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -38,6 +38,11 @@ class InodeUser: modification_time: str change_time: str path: str + inode_size: int + + @staticmethod + def format_symlink(symlink_source: str, symlink_dest: str): + return f"{symlink_source} -> {symlink_dest}" @dataclass @@ -81,6 +86,7 @@ class InodeInternal: access_time_dt = self.inode.get_access_time() modification_time_dt = self.inode.get_modification_time() change_time_dt = self.inode.get_change_time() + inode_size = int(self.inode.i_size) inode_user = InodeUser( superblock_addr=superblock_addr, @@ -96,6 +102,7 @@ class InodeInternal: modification_time=modification_time_dt, change_time=change_time_dt, path=self.path, + inode_size=inode_size, ) return inode_user From 1d0159325fbf04bb5029a9ed4ae2ea1acc770cee Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:09:20 +0100 Subject: [PATCH 113/137] switch to InodeUser.format_symlink --- volatility3/framework/plugins/linux/pagecache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index c86664f3d..a0dd8efe4 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -156,10 +156,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): """ # i_link (fast symlinks) were introduced in 4.2 if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: - i_link_str = inode.i_link.dereference().cast( + symlink_dest = inode.i_link.dereference().cast( "string", max_length=255, encoding="utf-8", errors="replace" ) - symlink_path = f"{symlink_path} -> {i_link_str}" + symlink_path = InodeUser.format_symlink(symlink_path, symlink_dest) return symlink_path From 48a8f3929bbe992ff687a494e3ea0e6a7446f42c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:10:32 +0100 Subject: [PATCH 114/137] add and leverage follow_symlinks parameter --- volatility3/framework/plugins/linux/pagecache.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index a0dd8efe4..4871d0d0f 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -220,12 +220,14 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, + follow_symlinks: bool = True, ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: context: The context that the plugin will operate within vmlinux_module_name: The name of the kernel module on which to operate + follow_symlinks: Whether to follow symlinks or not Yields: An InodeInternal object @@ -297,7 +299,9 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): continue seen_inodes.add(file_inode_ptr) - file_path = cls._follow_symlink(file_inode_ptr, file_path) + if follow_symlinks: + file_path = cls._follow_symlink(file_inode_ptr, file_path) + inode_in = InodeInternal( superblock=superblock, mountpoint=mountpoint, From 6f2ff4f7c6f702c11bf7d75b5ab78cb8f5881a20 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:11:03 +0100 Subject: [PATCH 115/137] add InodeSize column to Files --- volatility3/framework/plugins/linux/pagecache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 4871d0d0f..770c6391a 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -389,6 +389,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): ("ModificationTime", datetime.datetime), ("ChangeTime", datetime.datetime), ("FilePath", str), + ("InodeSize", int), ] return renderers.TreeGrid( From 85941060051b530350ad8e770de4c1f2d3edefee Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:12:10 +0100 Subject: [PATCH 116/137] 1.0.1 -> 1.2.0 Files bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 770c6391a..d57bd77f8 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -112,7 +112,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (1, 2, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From e8b44efc17dbfd6e412436d324702ed3e4cc7c7f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:13:26 +0100 Subject: [PATCH 117/137] add and leverage write_inode_content_to_stream --- .../framework/plugins/linux/pagecache.py | 60 ++++++++++++------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index d57bd77f8..42fa7538f 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,7 +6,7 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type, Iterable +from typing import List, Set, Type, Iterable, IO from volatility3.framework.constants import architectures from volatility3.framework import renderers, interfaces @@ -452,31 +452,47 @@ class InodePages(plugins.PluginInterface): vollog.error("The inode is not a regular file") return None - # By using truncate/seek, provided the filesystem supports it, a sparse file will be + try: + with open_method(filename) as f: + InodePages.write_inode_content_to_stream(inode, f, vmlinux_layer) + except OSError as e: + vollog.error("Unable to write to file (%s): %s", filename, e) + + @staticmethod + def write_inode_content_to_stream( + inode: interfaces.objects.ObjectInterface, + stream: IO, + vmlinux_layer: interfaces.layers.TranslationLayerInterface, + ) -> None: + """Extracts the inode's contents from the page cache and saves them to a stream + + Args: + inode: The inode to dump + stream: A IO steam to write to, typically FileHandlerInterface or BytesIO + vmlinux_layer: The kernel layer to obtain the page size + """ + + # By using truncate/seek, provided the filesystem supports it, and the + # stream is a File interface, a sparse file will be # created, saving both disk space and I/O time. # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. - try: - with open_method(filename) as f: - inode_size = inode.i_size - f.truncate(inode_size) + inode_size = inode.i_size + stream.truncate(inode_size) - for page_idx, page_content in inode.get_contents(): - current_fp = page_idx * vmlinux_layer.page_size - max_length = inode_size - current_fp - page_bytes = page_content[:max_length] - if current_fp + len(page_bytes) > inode_size: - vollog.error( - "Page out of file bounds: inode 0x%x, inode size %d, page index %d", - inode.vol.offset, - inode_size, - page_idx, - ) - f.seek(current_fp) - f.write(page_bytes) - - except OSError as e: - vollog.error("Unable to write to file (%s): %s", filename, e) + for page_idx, page_content in inode.get_contents(): + current_fp = page_idx * vmlinux_layer.page_size + max_length = inode_size - current_fp + page_bytes = page_content[:max_length] + if current_fp + len(page_bytes) > inode_size: + vollog.error( + "Page out of file bounds: inode 0x%x, inode size %d, page index %d", + inode.vol.offset, + inode_size, + page_idx, + ) + stream.seek(current_fp) + stream.write(page_bytes) def _generator(self): vmlinux_module_name = self.config["kernel"] From 818ddb746bb5bcbf45fe22fb54e8022d75d5fedd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:15:55 +0100 Subject: [PATCH 118/137] 2.0.0 -> 2.1.0 InodePages bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 42fa7538f..feac31bb7 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -402,7 +402,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From da068676d13eeefae033e46f8a916d91881bc2a6 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 15:16:09 +0100 Subject: [PATCH 119/137] typo --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index feac31bb7..e87bc2c9d 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -468,7 +468,7 @@ class InodePages(plugins.PluginInterface): Args: inode: The inode to dump - stream: A IO steam to write to, typically FileHandlerInterface or BytesIO + stream: An IO steam to write to, typically FileHandlerInterface or BytesIO vmlinux_layer: The kernel layer to obtain the page size """ From 452d6e705b973213238becd22d736f6ee1cb45e0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 16:26:25 +0100 Subject: [PATCH 120/137] 1.0.1 -> 1.1.0 Files bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index e87bc2c9d..89d30a9eb 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -112,7 +112,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 2, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From ecae545f5897e7c9a59d231d0b3f04b195cbf169 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:05:52 +0100 Subject: [PATCH 121/137] typo --- volatility3/framework/symbols/linux/extensions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 7d211150a..850045244 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -2948,7 +2948,7 @@ class scatterlist(objects.StructType): Returns: An iterator of bytes """ - # Either "physical" is layer-1 because this is a module layer, either "physical" is the current layer + # Either "physical" is layer-1 because this is a module layer, or "physical" is the current layer physical_layer_name = self._context.layers[self.vol.layer_name].config.get( "memory_layer", self.vol.layer_name ) From eddba98ef7c8c72162233dd19332e770d2a916d7 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:09:59 +0100 Subject: [PATCH 122/137] type hint format_symlink --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 89d30a9eb..d98103368 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -41,7 +41,7 @@ class InodeUser: inode_size: int @staticmethod - def format_symlink(symlink_source: str, symlink_dest: str): + def format_symlink(symlink_source: str, symlink_dest: str) -> str: return f"{symlink_source} -> {symlink_dest}" From 59703045c78941a12087a295b18cc8bc98414a06 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:26:42 +0100 Subject: [PATCH 123/137] switch calling convention to context and layer name --- .../framework/plugins/linux/pagecache.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index d98103368..d5297ab10 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -435,18 +435,20 @@ class InodePages(plugins.PluginInterface): @staticmethod def write_inode_content_to_file( + context: interfaces.context.ContextInterface, + layer_name: str, inode: interfaces.objects.ObjectInterface, filename: str, open_method: Type[interfaces.plugins.FileHandlerInterface], - vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a file Args: + context: The context on which to operate + layer_name: The name of the layer on which to operate inode: The inode to dump filename: Filename for writing the inode content open_method: class for constructing output files - vmlinux_layer: The kernel layer to obtain the page size """ if not inode.is_reg: vollog.error("The inode is not a regular file") @@ -454,24 +456,26 @@ class InodePages(plugins.PluginInterface): try: with open_method(filename) as f: - InodePages.write_inode_content_to_stream(inode, f, vmlinux_layer) + InodePages.write_inode_content_to_stream(context, layer_name, inode, f) except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) @staticmethod def write_inode_content_to_stream( + context: interfaces.context.ContextInterface, + layer_name: str, inode: interfaces.objects.ObjectInterface, stream: IO, - vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a stream Args: + context: The context on which to operate + layer_name: The name of the layer on which to operate inode: The inode to dump stream: An IO steam to write to, typically FileHandlerInterface or BytesIO - vmlinux_layer: The kernel layer to obtain the page size """ - + layer = context.layers[layer_name] # By using truncate/seek, provided the filesystem supports it, and the # stream is a File interface, a sparse file will be # created, saving both disk space and I/O time. @@ -481,7 +485,7 @@ class InodePages(plugins.PluginInterface): stream.truncate(inode_size) for page_idx, page_content in inode.get_contents(): - current_fp = page_idx * vmlinux_layer.page_size + current_fp = page_idx * layer.page_size max_length = inode_size - current_fp page_bytes = page_content[:max_length] if current_fp + len(page_bytes) > inode_size: @@ -557,7 +561,7 @@ class InodePages(plugins.PluginInterface): filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") vollog.info("[*] Writing inode at 0x%x to '%s'", inode_address, filename) self.write_inode_content_to_file( - inode, filename, open_method, vmlinux_layer + self.context, vmlinux_layer.name, inode, filename, open_method ) def run(self): From a02243bb3c4a14076cda7a516c7499e1734f19d0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:27:27 +0100 Subject: [PATCH 124/137] 2.1.0 -> 3.0.0 InodePages bump --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index d5297ab10..a86c1b936 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -402,7 +402,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 1, 0) + _version = (3, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: From d42ffc01e22672602a346b21f90b3733cd02db3e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 19 Jan 2025 17:33:47 +0100 Subject: [PATCH 125/137] typo --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index a86c1b936..77aa42338 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -473,7 +473,7 @@ class InodePages(plugins.PluginInterface): context: The context on which to operate layer_name: The name of the layer on which to operate inode: The inode to dump - stream: An IO steam to write to, typically FileHandlerInterface or BytesIO + stream: An IO stream to write to, typically FileHandlerInterface or BytesIO """ layer = context.layers[layer_name] # By using truncate/seek, provided the filesystem supports it, and the From 74b98e62c7aa4ca721f48651243122fa222548a5 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 19 Jan 2025 23:42:41 +0000 Subject: [PATCH 126/137] Revert "Add missing exception handling in env var recovery. Prevent backtraces" --- volatility3/framework/plugins/linux/envars.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 04b75c8a8..8cdbfe493 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -5,7 +5,7 @@ import logging from typing import Iterable, Tuple -from volatility3.framework import renderers, interfaces, exceptions +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -58,16 +58,10 @@ class Envars(plugins.PluginInterface): Tuples of (key, value) representing each environment variable. """ - # This ensures the `task` is valid as well as its - # memory mapping structures - try: - task_name = utility.array_to_string(task.comm) - env_start = task.mm.env_start - env_end = task.mm.env_end - except exceptions.InvalidAddressException: - return None - + task_name = utility.array_to_string(task.comm) task_pid = task.pid + env_start = task.mm.env_start + env_end = task.mm.env_end env_area_size = env_end - env_start if not (0 < env_area_size <= env_area_max_size): vollog.debug( From e18bffdde95e4adb0ee89aeb28b3845c53b37687 Mon Sep 17 00:00:00 2001 From: ikelos Date: Sun, 19 Jan 2025 23:51:28 +0000 Subject: [PATCH 127/137] Revert "Pre linux.pagecache.recoverfs support" --- .../framework/plugins/linux/pagecache.py | 97 ++++++------------- 1 file changed, 32 insertions(+), 65 deletions(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 77aa42338..382268515 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -6,9 +6,8 @@ import math import logging import datetime from dataclasses import dataclass, astuple -from typing import List, Set, Type, Iterable, IO +from typing import List, Set, Type, Iterable -from volatility3.framework.constants import architectures from volatility3.framework import renderers, interfaces from volatility3.framework.renderers import format_hints from volatility3.framework.interfaces import plugins @@ -38,11 +37,6 @@ class InodeUser: modification_time: str change_time: str path: str - inode_size: int - - @staticmethod - def format_symlink(symlink_source: str, symlink_dest: str) -> str: - return f"{symlink_source} -> {symlink_dest}" @dataclass @@ -86,7 +80,6 @@ class InodeInternal: access_time_dt = self.inode.get_access_time() modification_time_dt = self.inode.get_modification_time() change_time_dt = self.inode.get_change_time() - inode_size = int(self.inode.i_size) inode_user = InodeUser( superblock_addr=superblock_addr, @@ -102,7 +95,6 @@ class InodeInternal: modification_time=modification_time_dt, change_time=change_time_dt, path=self.path, - inode_size=inode_size, ) return inode_user @@ -112,7 +104,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -120,7 +112,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=architectures.LINUX_ARCHS, + architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( name="mountinfo", plugin=mountinfo.MountInfo, version=(1, 2, 0) @@ -156,10 +148,10 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): """ # i_link (fast symlinks) were introduced in 4.2 if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: - symlink_dest = inode.i_link.dereference().cast( + i_link_str = inode.i_link.dereference().cast( "string", max_length=255, encoding="utf-8", errors="replace" ) - symlink_path = InodeUser.format_symlink(symlink_path, symlink_dest) + symlink_path = f"{symlink_path} -> {i_link_str}" return symlink_path @@ -220,14 +212,12 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str, - follow_symlinks: bool = True, ) -> Iterable[InodeInternal]: """Retrieves the inodes from the superblocks Args: context: The context that the plugin will operate within vmlinux_module_name: The name of the kernel module on which to operate - follow_symlinks: Whether to follow symlinks or not Yields: An InodeInternal object @@ -299,9 +289,7 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): continue seen_inodes.add(file_inode_ptr) - if follow_symlinks: - file_path = cls._follow_symlink(file_inode_ptr, file_path) - + file_path = cls._follow_symlink(file_inode_ptr, file_path) inode_in = InodeInternal( superblock=superblock, mountpoint=mountpoint, @@ -389,7 +377,6 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): ("ModificationTime", datetime.datetime), ("ChangeTime", datetime.datetime), ("FilePath", str), - ("InodeSize", int), ] return renderers.TreeGrid( @@ -402,7 +389,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -410,7 +397,7 @@ class InodePages(plugins.PluginInterface): requirements.ModuleRequirement( name="kernel", description="Linux kernel", - architectures=architectures.LINUX_ARCHS, + architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( name="files", plugin=Files, version=(1, 0, 0) @@ -435,68 +422,48 @@ class InodePages(plugins.PluginInterface): @staticmethod def write_inode_content_to_file( - context: interfaces.context.ContextInterface, - layer_name: str, inode: interfaces.objects.ObjectInterface, filename: str, open_method: Type[interfaces.plugins.FileHandlerInterface], + vmlinux_layer: interfaces.layers.TranslationLayerInterface, ) -> None: """Extracts the inode's contents from the page cache and saves them to a file Args: - context: The context on which to operate - layer_name: The name of the layer on which to operate inode: The inode to dump filename: Filename for writing the inode content open_method: class for constructing output files + vmlinux_layer: The kernel layer to obtain the page size """ if not inode.is_reg: vollog.error("The inode is not a regular file") return None - try: - with open_method(filename) as f: - InodePages.write_inode_content_to_stream(context, layer_name, inode, f) - except OSError as e: - vollog.error("Unable to write to file (%s): %s", filename, e) - - @staticmethod - def write_inode_content_to_stream( - context: interfaces.context.ContextInterface, - layer_name: str, - inode: interfaces.objects.ObjectInterface, - stream: IO, - ) -> None: - """Extracts the inode's contents from the page cache and saves them to a stream - - Args: - context: The context on which to operate - layer_name: The name of the layer on which to operate - inode: The inode to dump - stream: An IO stream to write to, typically FileHandlerInterface or BytesIO - """ - layer = context.layers[layer_name] - # By using truncate/seek, provided the filesystem supports it, and the - # stream is a File interface, a sparse file will be + # By using truncate/seek, provided the filesystem supports it, a sparse file will be # created, saving both disk space and I/O time. # Additionally, using the page index will guarantee that each page is written at the # appropriate file position. - inode_size = inode.i_size - stream.truncate(inode_size) + try: + with open_method(filename) as f: + inode_size = inode.i_size + f.truncate(inode_size) - for page_idx, page_content in inode.get_contents(): - current_fp = page_idx * layer.page_size - max_length = inode_size - current_fp - page_bytes = page_content[:max_length] - if current_fp + len(page_bytes) > inode_size: - vollog.error( - "Page out of file bounds: inode 0x%x, inode size %d, page index %d", - inode.vol.offset, - inode_size, - page_idx, - ) - stream.seek(current_fp) - stream.write(page_bytes) + for page_idx, page_content in inode.get_contents(): + current_fp = page_idx * vmlinux_layer.page_size + max_length = inode_size - current_fp + page_bytes = page_content[:max_length] + if current_fp + len(page_bytes) > inode_size: + vollog.error( + "Page out of file bounds: inode 0x%x, inode size %d, page index %d", + inode.vol.offset, + inode_size, + page_idx, + ) + f.seek(current_fp) + f.write(page_bytes) + + except OSError as e: + vollog.error("Unable to write to file (%s): %s", filename, e) def _generator(self): vmlinux_module_name = self.config["kernel"] @@ -561,7 +528,7 @@ class InodePages(plugins.PluginInterface): filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") vollog.info("[*] Writing inode at 0x%x to '%s'", inode_address, filename) self.write_inode_content_to_file( - self.context, vmlinux_layer.name, inode, filename, open_method + inode, filename, open_method, vmlinux_layer ) def run(self): From bf76aad1e2367e2db4d561d9cc1cd76a42162420 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 20 Jan 2025 10:28:46 +1100 Subject: [PATCH 128/137] linux: page_cache.Files plugin: Ensure the inode's i_link pointer is readable --- volatility3/framework/plugins/linux/pagecache.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 39ed60486..f265241b6 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -147,7 +147,13 @@ class Files(plugins.PluginInterface, timeliner.TimeLinerInterface): Otherwise, it returns the same symlink_path """ # i_link (fast symlinks) were introduced in 4.2 - if inode and inode.is_link and inode.has_member("i_link") and inode.i_link: + if ( + inode + and inode.is_link + and inode.has_member("i_link") + and inode.i_link + and inode.i_link.is_readable() + ): i_link_str = inode.i_link.dereference().cast( "string", max_length=255, encoding="utf-8", errors="replace" ) From ec7a101eb92499741cafeb4dea1ab1ef1683a27c Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Mon, 20 Jan 2025 11:05:09 +1100 Subject: [PATCH 129/137] linux: page_cache.InodePages plugin: Remove unnecesary casting --- volatility3/framework/plugins/linux/pagecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index f265241b6..32b176b72 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -511,7 +511,7 @@ class InodePages(plugins.PluginInterface): page_vaddr = page_obj.vol.offset page_paddr = page_obj.to_paddr() page_mapping_addr = page_obj.mapping - page_index = int(page_obj.index) + page_index = page_obj.index page_file_offset = page_index * vmlinux_layer.page_size dump_safe = ( page_file_offset < inode_size From dfe3d255c064b9c78edf4f5f58eff6c15cc56486 Mon Sep 17 00:00:00 2001 From: Odysseas Stavrou Date: Mon, 20 Jan 2025 22:25:01 +0200 Subject: [PATCH 130/137] Volshell: Update Process retrieval methods with virtual/physical offsets --- volatility3/cli/volshell/linux.py | 58 +++++++++++++++++++++++++++++ volatility3/cli/volshell/windows.py | 47 +++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index cc58fa1c2..9ea3ea1f5 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -3,6 +3,7 @@ # from typing import Any, List, Optional, Tuple, Union +from enum import Enum from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -10,6 +11,16 @@ from volatility3.framework.configuration import requirements from volatility3.plugins.linux import pslist +# Could import the enum from psscan.py to avoid code duplication +class DescExitStateEnum(Enum): + """Enum for linux task exit_state as defined in include/linux/sched.h""" + + TASK_RUNNING = 0x00000000 + EXIT_DEAD = 0x00000010 + EXIT_ZOMBIE = 0x00000020 + EXIT_TRACE = EXIT_ZOMBIE | EXIT_DEAD + + class Volshell(generic.Volshell): """Shell environment to directly interact with a linux memory image.""" @@ -40,6 +51,52 @@ class Volshell(generic.Volshell): return None print(f"No task with task ID {pid} found") + def get_process(self, pid=None, offset=None): + """Get Task based on a process ID. Does not retrieve the layer, to change layer use the .pid attribute. The offset argument can be used both for physical or virtual offsets""" + + if pid is not None and offset is not None: + print("Only one parameter is accepted") + return None + + if offset is not None: + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] + + kernel_layer_name = vmlinux.layer_name + kernel_layer = self.context.layers[kernel_layer_name] + + memory_layer_name = kernel_layer.dependencies[0] + + ptask = self.context.object( + vmlinux.symbol_table_name + constants.BANG + "task_struct", + layer_name=memory_layer_name, + offset=offset, + native_layer_name=kernel_layer_name, + ) + + try: + DescExitStateEnum(ptask.exit_state) + except ValueError: + print( + f"task_struct @ {hex(ptask.vol.offset)} as exit_state {ptask.exit_state} is likely not valid" + ) + + if not (0 < ptask.pid < 65535): + print( + f"task_struct @ {hex(ptask.vol.offset)} as pid {ptask.pid} is likely not valid" + ) + + return ptask + + if pid is not None: + tasks = self.list_tasks() + for task in tasks: + if task.pid == pid: + return task + print(f"No task with task ID {pid} found") + + return None + def list_tasks(self): """Returns a list of task objects from the primary layer""" # We always use the main kernel memory and associated symbols @@ -50,6 +107,7 @@ class Volshell(generic.Volshell): result += [ (["ct", "change_task", "cp"], self.change_task), (["lt", "list_tasks", "ps"], self.list_tasks), + (["gp", "get_process"], self.get_process), (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get("pid", None) is not None: diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 303d4d5c3..a77392561 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -44,11 +44,58 @@ class Volshell(generic.Volshell): ) ) + def get_process(self, pid=None, v_offset=None, p_offset=None): + """Returns the EPROCESS object that matches the pid. If v_offset/p_offset is provided, construct the EPROCESS object at the provided address. Only one parameter is allowed.""" + + if sum(1 if x is not None else 0 for x in [pid, v_offset, p_offset]) != 1: + print("Only one parameter is accepted") + return None + + kernel_name = self.config["kernel"] + kernel = self.context.modules[kernel_name] + + kernel_layer_name = kernel.layer_name + + kernel_layer = self.context.layers[kernel_layer_name] + memory_layer_name = kernel_layer.dependencies[0] + + eprocess_symbol = kernel.symbol_table_name + constants.BANG + "_EPROCESS" + + if v_offset is not None: + eproc = self.context.object( + eprocess_symbol, + layer_name=kernel_layer_name, + offset=v_offset, + ) + + return eproc + + if p_offset is not None: + eproc = self.context.object( + eprocess_symbol, + layer_name=memory_layer_name, + offset=p_offset, + native_layer_name=kernel_layer_name, + ) + + return eproc + + if pid is not None: + processes = self.list_processes() + for process in processes: + if process.UniqueProcessId == pid: + return process + print(f"No process with process ID {pid} found") + return None + + return None + def construct_locals(self) -> List[Tuple[List[str], Any]]: result = super().construct_locals() result += [ (["cp", "change_process"], self.change_process), (["lp", "list_processes", "ps"], self.list_processes), + (["gp", "get_process"], self.get_process), (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get("pid", None) is not None: From b59f051353cd58b7d6e4bfda2f07746820f4f32a Mon Sep 17 00:00:00 2001 From: Odysseas Stavrou Date: Wed, 22 Jan 2025 02:39:35 +0200 Subject: [PATCH 131/137] Volshell: Updates to the get_process() methods --- volatility3/cli/volshell/linux.py | 55 +++++++++++++++++++---------- volatility3/cli/volshell/windows.py | 23 ++++++++---- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 9ea3ea1f5..b3689c3ae 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -51,42 +51,61 @@ class Volshell(generic.Volshell): return None print(f"No task with task ID {pid} found") - def get_process(self, pid=None, offset=None): - """Get Task based on a process ID. Does not retrieve the layer, to change layer use the .pid attribute. The offset argument can be used both for physical or virtual offsets""" + def get_process(self, pid=None, virtaddr=None, physaddr=None): + """Return the task_struct object that matches the pid. If a physical or a virtual address is provided, construct the task_struct object at said address. Only one parameter is allowed. - if pid is not None and offset is not None: + Args: + pid (int, optional): PID to search for + virtaddr (int, optional): Virtual address to construct object at + physaddr (int, optional): Physical address to construct object at + + Returns: + ObjectInterface: task_struct Object + """ + + if sum(1 if x is not None else 0 for x in [pid, virtaddr, physaddr]) != 1: print("Only one parameter is accepted") return None - if offset is not None: - vmlinux_module_name = self.config["kernel"] - vmlinux = self.context.modules[vmlinux_module_name] + vmlinux_module_name = self.config["kernel"] + vmlinux = self.context.modules[vmlinux_module_name] - kernel_layer_name = vmlinux.layer_name - kernel_layer = self.context.layers[kernel_layer_name] + kernel_layer_name = vmlinux.layer_name + kernel_layer = self.context.layers[kernel_layer_name] - memory_layer_name = kernel_layer.dependencies[0] + memory_layer_name = kernel_layer.dependencies[0] - ptask = self.context.object( - vmlinux.symbol_table_name + constants.BANG + "task_struct", + task_struct_symbol = vmlinux.symbol_table_name + constants.BANG + "task_struct" + + if virtaddr is not None: + task = self.context.object( + task_struct_symbol, + layer_name=kernel_layer_name, + offset=virtaddr, + ) + + if physaddr is not None: + task = self.context.object( + task_struct_symbol, layer_name=memory_layer_name, - offset=offset, + offset=physaddr, native_layer_name=kernel_layer_name, ) + if physaddr is not None or virtaddr is not None: try: - DescExitStateEnum(ptask.exit_state) + DescExitStateEnum(task.exit_state) except ValueError: print( - f"task_struct @ {hex(ptask.vol.offset)} as exit_state {ptask.exit_state} is likely not valid" + f"task_struct @ {hex(task.vol.offset)} as exit_state {task.exit_state} is likely not valid" ) - if not (0 < ptask.pid < 65535): + if not (0 < task.pid < 65535): print( - f"task_struct @ {hex(ptask.vol.offset)} as pid {ptask.pid} is likely not valid" + f"task_struct @ {hex(task.vol.offset)} as pid {task.pid} is likely not valid" ) - return ptask + return task if pid is not None: tasks = self.list_tasks() @@ -107,7 +126,7 @@ class Volshell(generic.Volshell): result += [ (["ct", "change_task", "cp"], self.change_task), (["lt", "list_tasks", "ps"], self.list_tasks), - (["gp", "get_process"], self.get_process), + (["gp", "get_process", "get_task"], self.get_process), (["symbols"], self.context.symbol_space[self.current_symbol_table]), ] if self.config.get("pid", None) is not None: diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index a77392561..9b89a8b81 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -44,10 +44,19 @@ class Volshell(generic.Volshell): ) ) - def get_process(self, pid=None, v_offset=None, p_offset=None): - """Returns the EPROCESS object that matches the pid. If v_offset/p_offset is provided, construct the EPROCESS object at the provided address. Only one parameter is allowed.""" + def get_process(self, pid=None, virtaddr=None, physaddr=None): + """Returns the _EPROCESS object that matches the pid. If a physical or a virtual address is provided, construct the _EPROCESS object at said address. Only one parameter is allowed. - if sum(1 if x is not None else 0 for x in [pid, v_offset, p_offset]) != 1: + Args: + pid (int, optional): PID / UniqueProcessId to search for. + virtaddr (int, optional): Virtual address to construct object at + physaddr (int, optional): Physical address to construct object at + + Returns: + ObjectInterface: _EPROCESS Object + """ + + if sum(1 if x is not None else 0 for x in [pid, virtaddr, physaddr]) != 1: print("Only one parameter is accepted") return None @@ -61,20 +70,20 @@ class Volshell(generic.Volshell): eprocess_symbol = kernel.symbol_table_name + constants.BANG + "_EPROCESS" - if v_offset is not None: + if virtaddr is not None: eproc = self.context.object( eprocess_symbol, layer_name=kernel_layer_name, - offset=v_offset, + offset=virtaddr, ) return eproc - if p_offset is not None: + if physaddr is not None: eproc = self.context.object( eprocess_symbol, layer_name=memory_layer_name, - offset=p_offset, + offset=physaddr, native_layer_name=kernel_layer_name, ) From b8d9c7b88311016cea97b8b60afeec4d47558af0 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 23 Jan 2025 11:57:43 -0600 Subject: [PATCH 132/137] #1473 - add missing exception handling for get_key --- volatility3/framework/plugins/windows/envars.py | 12 ++++++------ .../framework/plugins/windows/getservicesids.py | 5 +++-- .../framework/plugins/windows/getsids.py | 2 +- .../plugins/windows/registry/userassist.py | 17 ++++++++++++----- .../framework/plugins/windows/svcscan.py | 6 +++--- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index cac4ecf40..48e1ef671 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -76,14 +76,14 @@ class Envars(interfaces.plugins.PluginInterface): "CurrentControlSet\\Control\\Session Manager\\Environment" ) sys = True - except KeyError: - with contextlib.suppress(KeyError): + except (KeyError, registry.RegistryFormatException): + with contextlib.suppress(KeyError, registry.RegistryFormatException): key = hive.get_key( "ControlSet001\\Control\\Session Manager\\Environment" ) sys = True if sys: - with contextlib.suppress(KeyError): + with contextlib.suppress(KeyError, registry.RegistryFormatException): for node in key.get_values(): try: value_node_name = node.get_name() @@ -100,11 +100,11 @@ class Envars(interfaces.plugins.PluginInterface): continue ## The user-specific variables - with contextlib.suppress(KeyError): + with contextlib.suppress(KeyError, registry.RegistryFormatException): key = hive.get_key("Environment") ntuser = True if ntuser: - with contextlib.suppress(KeyError): + with contextlib.suppress(KeyError, registry.RegistryFormatException): for node in key.get_values(): try: value_node_name = node.get_name() @@ -123,7 +123,7 @@ class Envars(interfaces.plugins.PluginInterface): ## The volatile user variables try: key = hive.get_key("Volatile Environment") - except KeyError: + except (KeyError, registry.RegistryFormatException): continue try: for node in key.get_values(): diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index eece7fb6c..b97d2bb46 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -10,6 +10,7 @@ from typing import List from volatility3.framework import renderers, interfaces, constants, exceptions from volatility3.framework.configuration import requirements +from volatility3.framework.layers import registry from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) @@ -86,10 +87,10 @@ class GetServiceSIDs(interfaces.plugins.PluginInterface): # Get ControlSet\Services. try: services = hive.get_key(r"CurrentControlSet\Services") - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): try: services = hive.get_key(r"ControlSet001\Services") - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): continue if services: diff --git a/volatility3/framework/plugins/windows/getsids.py b/volatility3/framework/plugins/windows/getsids.py index df0c7a835..00c78e1cf 100644 --- a/volatility3/framework/plugins/windows/getsids.py +++ b/volatility3/framework/plugins/windows/getsids.py @@ -158,7 +158,7 @@ class GetSIDs(interfaces.plugins.PluginInterface): layers.registry.RegistryFormatException, ): continue - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, layers.registry.RegistryFormatException): continue return sids diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 932ee9d6f..646fb1d7f 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -13,7 +13,7 @@ from typing import Any, Generator, List, Tuple from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.layers.physical import BufferDataLayer -from volatility3.framework.layers.registry import RegistryHive +from volatility3.framework.layers.registry import RegistryHive, RegistryFormatException from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed from volatility3.plugins.windows.registry import hivelist @@ -167,10 +167,17 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac self._determine_userassist_type() - userassist_node_path = hive.get_key( - "software\\microsoft\\windows\\currentversion\\explorer\\userassist", - return_list=True, - ) + try: + userassist_node_path = hive.get_key( + "software\\microsoft\\windows\\currentversion\\explorer\\userassist", + return_list=True, + ) + except RegistryFormatException as e: + vollog.warning(f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}") + return None + except KeyError: + vollog.warning(f"UserAssist key not found in {hive_name} at {hive.hive_offset:#x}") + return None if not userassist_node_path: vollog.warning("list_userassist did not find a valid node_path (or None)") diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index bd477ba27..93087f352 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -15,7 +15,7 @@ from volatility3.framework import ( symbols, ) from volatility3.framework.configuration import requirements -from volatility3.framework.layers import scanners +from volatility3.framework.layers import scanners, registry from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions @@ -159,12 +159,12 @@ class SvcScan(interfaces.plugins.PluginInterface): return cast( objects.StructType, hive.get_key(r"CurrentControlSet\Services") ) - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): try: return cast( objects.StructType, hive.get_key(r"ControlSet001\Services") ) - except (KeyError, exceptions.InvalidAddressException): + except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): vollog.log( constants.LOGLEVEL_VVVV, "Could not retrieve any control set from SYSTEM hive", From 6c4cafa64f68e6b001cd1ed32e8e5fb3d9993f30 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 23 Jan 2025 11:59:37 -0600 Subject: [PATCH 133/137] #1473 - black fixes --- .../framework/plugins/windows/getservicesids.py | 12 ++++++++++-- volatility3/framework/plugins/windows/getsids.py | 6 +++++- .../framework/plugins/windows/registry/userassist.py | 8 ++++++-- volatility3/framework/plugins/windows/svcscan.py | 12 ++++++++++-- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index b97d2bb46..207d0e2ad 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -87,10 +87,18 @@ class GetServiceSIDs(interfaces.plugins.PluginInterface): # Get ControlSet\Services. try: services = hive.get_key(r"CurrentControlSet\Services") - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): try: services = hive.get_key(r"ControlSet001\Services") - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): continue if services: diff --git a/volatility3/framework/plugins/windows/getsids.py b/volatility3/framework/plugins/windows/getsids.py index 00c78e1cf..a75bbe7ea 100644 --- a/volatility3/framework/plugins/windows/getsids.py +++ b/volatility3/framework/plugins/windows/getsids.py @@ -158,7 +158,11 @@ class GetSIDs(interfaces.plugins.PluginInterface): layers.registry.RegistryFormatException, ): continue - except (KeyError, exceptions.InvalidAddressException, layers.registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + layers.registry.RegistryFormatException, + ): continue return sids diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index 646fb1d7f..d50b5216e 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -173,10 +173,14 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac return_list=True, ) except RegistryFormatException as e: - vollog.warning(f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}") + vollog.warning( + f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}" + ) return None except KeyError: - vollog.warning(f"UserAssist key not found in {hive_name} at {hive.hive_offset:#x}") + vollog.warning( + f"UserAssist key not found in {hive_name} at {hive.hive_offset:#x}" + ) return None if not userassist_node_path: diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 93087f352..17baac5b0 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -159,12 +159,20 @@ class SvcScan(interfaces.plugins.PluginInterface): return cast( objects.StructType, hive.get_key(r"CurrentControlSet\Services") ) - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): try: return cast( objects.StructType, hive.get_key(r"ControlSet001\Services") ) - except (KeyError, exceptions.InvalidAddressException, registry.RegistryFormatException): + except ( + KeyError, + exceptions.InvalidAddressException, + registry.RegistryFormatException, + ): vollog.log( constants.LOGLEVEL_VVVV, "Could not retrieve any control set from SYSTEM hive", From 81ba89eef663727608886e66595dfd7b3dcd9831 Mon Sep 17 00:00:00 2001 From: Dave Lassalle Date: Thu, 23 Jan 2025 12:20:14 -0600 Subject: [PATCH 134/137] #1473 - update exception message --- volatility3/framework/plugins/windows/registry/userassist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index d50b5216e..87016553a 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -174,7 +174,7 @@ class UserAssist(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterfac ) except RegistryFormatException as e: vollog.warning( - f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}" + f"Error accessing UserAssist key in {hive_name} at {hive.hive_offset:#x}: {e}" ) return None except KeyError: From a68be50798254cbadc490393721e74180b4117cc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 23 Jan 2025 20:30:21 +0000 Subject: [PATCH 135/137] Revert "Typing fix" This reverts commit c82d432b10258136ff0777dfec1fbf5844316132. --- volatility3/framework/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index a1925faef..754939460 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -5,6 +5,7 @@ # Check the python version to ensure it's suitable import glob import sys +from volatility3.framework import check_python_version as check_python_version import zipfile import importlib import inspect @@ -57,7 +58,7 @@ class NonInheritable: self.default_value = value self.cls = cls - def __get__(self, obj: Any, get_type: Optional[Type] = None) -> Any: + def __get__(self, obj: Any, get_type: Type = Optional[None]) -> Any: if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) From e9088be0d86fa2f68774d47371ebd2736287f1c5 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Sat, 28 Dec 2024 22:46:39 +0000 Subject: [PATCH 136/137] Prevent infinite looping and out of memory errors #1482 --- .../framework/symbols/windows/extensions/registry.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index c9544a8ba..b282b13cf 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -133,8 +133,17 @@ class CM_KEY_BODY(objects.StructType): def get_full_key_name(self) -> str: output = [] + seen = set() + kcb = self.KeyControlBlock while kcb.ParentKcb: + if kcb.ParentKcb.vol.offset in seen: + return "" + seen.add(kcb.ParentKcb.vol.offset) + + if len(output) > 128: + return "" + if kcb.NameBlock.Name is None: break From 506a61d8846e6a3399ab1f964d41b09a197592a6 Mon Sep 17 00:00:00 2001 From: Andrew Case Date: Fri, 24 Jan 2025 22:09:24 +0000 Subject: [PATCH 137/137] Address feedback --- volatility3/framework/plugins/windows/handles.py | 4 ++-- volatility3/framework/symbols/windows/extensions/registry.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 38ccfbfbc..6a391fe35 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -341,7 +341,7 @@ class Handles(interfaces.plugins.PluginInterface): try: obj_name = entry.NameInfo.Name.String except (ValueError, exceptions.InvalidAddressException): - obj_name = "" + obj_name = None except exceptions.InvalidAddressException: vollog.log( @@ -359,7 +359,7 @@ class Handles(interfaces.plugins.PluginInterface): format_hints.Hex(entry.HandleValue), obj_type, format_hints.Hex(entry.GrantedAccess), - obj_name, + obj_name or renderers.NotAvailableValue(), ), ) diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index b282b13cf..a8cc7703c 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -138,11 +138,11 @@ class CM_KEY_BODY(objects.StructType): kcb = self.KeyControlBlock while kcb.ParentKcb: if kcb.ParentKcb.vol.offset in seen: - return "" + return None seen.add(kcb.ParentKcb.vol.offset) if len(output) > 128: - return "" + return None if kcb.NameBlock.Name is None: break